Mercurial > hg > CbC > CbC_gcc
diff gcc/config/i386/i386.c @ 132:d34655255c78
update gcc-8.2
| author | mir3636 |
|---|---|
| date | Thu, 25 Oct 2018 10:21:07 +0900 |
| parents | ab0bcb71f44d 84e7813d76e9 |
| children | 351920fa3827 |
line wrap: on
line diff
--- a/gcc/config/i386/i386.c Thu Oct 25 08:08:40 2018 +0900 +++ b/gcc/config/i386/i386.c Thu Oct 25 10:21:07 2018 +0900 @@ -1,5 +1,5 @@ /* Subroutines used for code generation on IA-32. - Copyright (C) 1988-2017 Free Software Foundation, Inc. + Copyright (C) 1988-2018 Free Software Foundation, Inc. This file is part of GCC. @@ -17,6 +17,8 @@ along with GCC; see the file COPYING3. If not see <http://www.gnu.org/licenses/>. */ +#define IN_TARGET_CODE 1 + #include "config.h" #include "system.h" #include "coretypes.h" @@ -71,11 +73,8 @@ #include "builtins.h" #include "rtl-iter.h" #include "tree-iterator.h" -#include "tree-chkp.h" -#include "rtl-chkp.h" #include "dbgcnt.h" #include "case-cfn-macros.h" -#include "regrename.h" #include "dojump.h" #include "fold-const-call.h" #include "tree-vrp.h" @@ -88,6 +87,10 @@ #include "symbol-summary.h" #include "ipa-prop.h" #include "ipa-fnsummary.h" +#include "wide-int-bitmask.h" +#include "tree-vector-builder.h" +#include "debug.h" +#include "dwarf2out.h" /* This file should be included last. */ #include "target-def.h" @@ -101,6 +104,8 @@ static bool ix86_save_reg (unsigned int, bool, bool); static bool ix86_function_naked (const_tree); static bool ix86_notrack_prefixed_insn_p (rtx); +static void ix86_emit_restore_reg_using_pop (rtx); + #ifndef CHECK_STACK_LIMIT #define CHECK_STACK_LIMIT (-1) @@ -122,46 +127,56 @@ const struct processor_costs *ix86_cost = NULL; /* Processor feature/optimization bitmasks. */ -#define m_386 (1U<<PROCESSOR_I386) -#define m_486 (1U<<PROCESSOR_I486) -#define m_PENT (1U<<PROCESSOR_PENTIUM) -#define m_LAKEMONT (1U<<PROCESSOR_LAKEMONT) -#define m_PPRO (1U<<PROCESSOR_PENTIUMPRO) -#define m_PENT4 (1U<<PROCESSOR_PENTIUM4) -#define m_NOCONA (1U<<PROCESSOR_NOCONA) +#define m_386 (HOST_WIDE_INT_1U<<PROCESSOR_I386) +#define m_486 (HOST_WIDE_INT_1U<<PROCESSOR_I486) +#define m_PENT (HOST_WIDE_INT_1U<<PROCESSOR_PENTIUM) +#define m_LAKEMONT (HOST_WIDE_INT_1U<<PROCESSOR_LAKEMONT) +#define m_PPRO (HOST_WIDE_INT_1U<<PROCESSOR_PENTIUMPRO) +#define m_PENT4 (HOST_WIDE_INT_1U<<PROCESSOR_PENTIUM4) +#define m_NOCONA (HOST_WIDE_INT_1U<<PROCESSOR_NOCONA) #define m_P4_NOCONA (m_PENT4 | m_NOCONA) -#define m_CORE2 (1U<<PROCESSOR_CORE2) -#define m_NEHALEM (1U<<PROCESSOR_NEHALEM) -#define m_SANDYBRIDGE (1U<<PROCESSOR_SANDYBRIDGE) -#define m_HASWELL (1U<<PROCESSOR_HASWELL) -#define m_CORE_ALL (m_CORE2 | m_NEHALEM | m_SANDYBRIDGE | m_HASWELL) -#define m_BONNELL (1U<<PROCESSOR_BONNELL) -#define m_SILVERMONT (1U<<PROCESSOR_SILVERMONT) -#define m_KNL (1U<<PROCESSOR_KNL) -#define m_KNM (1U<<PROCESSOR_KNM) -#define m_SKYLAKE_AVX512 (1U<<PROCESSOR_SKYLAKE_AVX512) -#define m_INTEL (1U<<PROCESSOR_INTEL) - -#define m_GEODE (1U<<PROCESSOR_GEODE) -#define m_K6 (1U<<PROCESSOR_K6) +#define m_CORE2 (HOST_WIDE_INT_1U<<PROCESSOR_CORE2) +#define m_NEHALEM (HOST_WIDE_INT_1U<<PROCESSOR_NEHALEM) +#define m_SANDYBRIDGE (HOST_WIDE_INT_1U<<PROCESSOR_SANDYBRIDGE) +#define m_HASWELL (HOST_WIDE_INT_1U<<PROCESSOR_HASWELL) +#define m_BONNELL (HOST_WIDE_INT_1U<<PROCESSOR_BONNELL) +#define m_SILVERMONT (HOST_WIDE_INT_1U<<PROCESSOR_SILVERMONT) +#define m_KNL (HOST_WIDE_INT_1U<<PROCESSOR_KNL) +#define m_KNM (HOST_WIDE_INT_1U<<PROCESSOR_KNM) +#define m_SKYLAKE (HOST_WIDE_INT_1U<<PROCESSOR_SKYLAKE) +#define m_SKYLAKE_AVX512 (HOST_WIDE_INT_1U<<PROCESSOR_SKYLAKE_AVX512) +#define m_CANNONLAKE (HOST_WIDE_INT_1U<<PROCESSOR_CANNONLAKE) +#define m_ICELAKE_CLIENT (HOST_WIDE_INT_1U<<PROCESSOR_ICELAKE_CLIENT) +#define m_ICELAKE_SERVER (HOST_WIDE_INT_1U<<PROCESSOR_ICELAKE_SERVER) +#define m_CORE_AVX512 (m_SKYLAKE_AVX512 | m_CANNONLAKE \ + | m_ICELAKE_CLIENT | m_ICELAKE_SERVER) +#define m_CORE_AVX2 (m_HASWELL | m_SKYLAKE | m_CORE_AVX512) +#define m_CORE_ALL (m_CORE2 | m_NEHALEM | m_SANDYBRIDGE | m_CORE_AVX2) +#define m_GOLDMONT (HOST_WIDE_INT_1U<<PROCESSOR_GOLDMONT) +#define m_GOLDMONT_PLUS (HOST_WIDE_INT_1U<<PROCESSOR_GOLDMONT_PLUS) +#define m_TREMONT (HOST_WIDE_INT_1U<<PROCESSOR_TREMONT) +#define m_INTEL (HOST_WIDE_INT_1U<<PROCESSOR_INTEL) + +#define m_GEODE (HOST_WIDE_INT_1U<<PROCESSOR_GEODE) +#define m_K6 (HOST_WIDE_INT_1U<<PROCESSOR_K6) #define m_K6_GEODE (m_K6 | m_GEODE) -#define m_K8 (1U<<PROCESSOR_K8) -#define m_ATHLON (1U<<PROCESSOR_ATHLON) +#define m_K8 (HOST_WIDE_INT_1U<<PROCESSOR_K8) +#define m_ATHLON (HOST_WIDE_INT_1U<<PROCESSOR_ATHLON) #define m_ATHLON_K8 (m_K8 | m_ATHLON) -#define m_AMDFAM10 (1U<<PROCESSOR_AMDFAM10) -#define m_BDVER1 (1U<<PROCESSOR_BDVER1) -#define m_BDVER2 (1U<<PROCESSOR_BDVER2) -#define m_BDVER3 (1U<<PROCESSOR_BDVER3) -#define m_BDVER4 (1U<<PROCESSOR_BDVER4) -#define m_ZNVER1 (1U<<PROCESSOR_ZNVER1) -#define m_BTVER1 (1U<<PROCESSOR_BTVER1) -#define m_BTVER2 (1U<<PROCESSOR_BTVER2) +#define m_AMDFAM10 (HOST_WIDE_INT_1U<<PROCESSOR_AMDFAM10) +#define m_BDVER1 (HOST_WIDE_INT_1U<<PROCESSOR_BDVER1) +#define m_BDVER2 (HOST_WIDE_INT_1U<<PROCESSOR_BDVER2) +#define m_BDVER3 (HOST_WIDE_INT_1U<<PROCESSOR_BDVER3) +#define m_BDVER4 (HOST_WIDE_INT_1U<<PROCESSOR_BDVER4) +#define m_ZNVER1 (HOST_WIDE_INT_1U<<PROCESSOR_ZNVER1) +#define m_BTVER1 (HOST_WIDE_INT_1U<<PROCESSOR_BTVER1) +#define m_BTVER2 (HOST_WIDE_INT_1U<<PROCESSOR_BTVER2) #define m_BDVER (m_BDVER1 | m_BDVER2 | m_BDVER3 | m_BDVER4) #define m_BTVER (m_BTVER1 | m_BTVER2) #define m_AMD_MULTIPLE (m_ATHLON_K8 | m_AMDFAM10 | m_BDVER | m_BTVER \ | m_ZNVER1) -#define m_GENERIC (1U<<PROCESSOR_GENERIC) +#define m_GENERIC (HOST_WIDE_INT_1U<<PROCESSOR_GENERIC) const char* ix86_tune_feature_names[X86_TUNE_LAST] = { #undef DEF_TUNE @@ -175,7 +190,7 @@ /* Feature tests against the various tunings used to create ix86_tune_features based on the processor mask. */ -static unsigned int initial_ix86_tune_features[X86_TUNE_LAST] = { +static unsigned HOST_WIDE_INT initial_ix86_tune_features[X86_TUNE_LAST] = { #undef DEF_TUNE #define DEF_TUNE(tune, name, selector) selector, #include "x86-tune.def" @@ -187,7 +202,7 @@ /* Feature tests against the various architecture variations, used to create ix86_arch_features based on the processor mask. */ -static unsigned int initial_ix86_arch_features[X86_ARCH_LAST] = { +static unsigned HOST_WIDE_INT initial_ix86_arch_features[X86_ARCH_LAST] = { /* X86_ARCH_CMOV: Conditional move was added for pentiumpro. */ ~(m_386 | m_486 | m_PENT | m_LAKEMONT | m_K6), @@ -226,32 +241,28 @@ /* FP registers */ FP_TOP_REG, FP_SECOND_REG, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, FLOAT_REGS, - /* arg pointer */ - NON_Q_REGS, - /* flags, fpsr, fpcr, frame */ - NO_REGS, NO_REGS, NO_REGS, NON_Q_REGS, + /* arg pointer, flags, fpsr, frame */ + NON_Q_REGS, NO_REGS, NO_REGS, NON_Q_REGS, /* SSE registers */ - SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, - SSE_REGS, SSE_REGS, + SSE_FIRST_REG, SSE_REGS, SSE_REGS, SSE_REGS, + SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, /* MMX registers */ - MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, - MMX_REGS, MMX_REGS, + MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, + MMX_REGS, MMX_REGS, MMX_REGS, MMX_REGS, /* REX registers */ - NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, - NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, NON_Q_REGS, + GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, + GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, GENERAL_REGS, /* SSE REX registers */ - SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, - SSE_REGS, SSE_REGS, + SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, + SSE_REGS, SSE_REGS, SSE_REGS, SSE_REGS, /* AVX-512 SSE registers */ - EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, - EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, - EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, - EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, EVEX_SSE_REGS, + ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, + ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, + ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, + ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, ALL_SSE_REGS, /* Mask registers. */ - MASK_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, - MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, MASK_EVEX_REGS, - /* MPX bound registers */ - BND_REGS, BND_REGS, BND_REGS, BND_REGS, + ALL_MASK_REGS, MASK_REGS, MASK_REGS, MASK_REGS, + MASK_REGS, MASK_REGS, MASK_REGS, MASK_REGS }; /* The "default" register map used in 32bit mode. */ @@ -260,15 +271,14 @@ { 0, 2, 1, 3, 6, 7, 4, 5, /* general regs */ 12, 13, 14, 15, 16, 17, 18, 19, /* fp regs */ - -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */ + -1, -1, -1, -1, /* arg, flags, fpsr, frame */ 21, 22, 23, 24, 25, 26, 27, 28, /* SSE */ 29, 30, 31, 32, 33, 34, 35, 36, /* MMX */ -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */ -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */ -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 16-23*/ -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 24-31*/ - 93, 94, 95, 96, 97, 98, 99, 100, /* Mask registers */ - 101, 102, 103, 104, /* bound registers */ + 93, 94, 95, 96, 97, 98, 99, 100 /* Mask registers */ }; /* The "default" register map used in 64bit mode. */ @@ -277,15 +287,14 @@ { 0, 1, 2, 3, 4, 5, 6, 7, /* general regs */ 33, 34, 35, 36, 37, 38, 39, 40, /* fp regs */ - -1, -1, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */ + -1, -1, -1, -1, /* arg, flags, fpsr, frame */ 17, 18, 19, 20, 21, 22, 23, 24, /* SSE */ 41, 42, 43, 44, 45, 46, 47, 48, /* MMX */ 8,9,10,11,12,13,14,15, /* extended integer registers */ 25, 26, 27, 28, 29, 30, 31, 32, /* extended SSE registers */ 67, 68, 69, 70, 71, 72, 73, 74, /* AVX-512 registers 16-23 */ 75, 76, 77, 78, 79, 80, 81, 82, /* AVX-512 registers 24-31 */ - 118, 119, 120, 121, 122, 123, 124, 125, /* Mask registers */ - 126, 127, 128, 129, /* bound registers */ + 118, 119, 120, 121, 122, 123, 124, 125 /* Mask registers */ }; /* Define the register numbers to be used in Dwarf debugging information. @@ -301,7 +310,7 @@ 7 for %edi (gcc regno = 5) The following three DWARF register numbers are never generated by the SVR4 C compiler or by the GNU compilers, but SDB on x86/svr4 - believes these numbers have these meanings. + believed these numbers have these meanings. 8 for %eip (no gcc equivalent) 9 for %eflags (gcc regno = 17) 10 for %trapno (no gcc equivalent) @@ -309,20 +318,20 @@ for the x86 architecture. If the version of SDB on x86/svr4 were a bit less brain dead with respect to floating-point then we would have a precedent to follow with respect to DWARF register numbers - for x86 FP registers, but the SDB on x86/svr4 is so completely + for x86 FP registers, but the SDB on x86/svr4 was so completely broken with respect to FP registers that it is hardly worth thinking of it as something to strive for compatibility with. - The version of x86/svr4 SDB I have at the moment does (partially) + The version of x86/svr4 SDB I had does (partially) seem to believe that DWARF register number 11 is associated with the x86 register %st(0), but that's about all. Higher DWARF register numbers don't seem to be associated with anything in - particular, and even for DWARF regno 11, SDB only seems to under- + particular, and even for DWARF regno 11, SDB only seemed to under- stand that it should say that a variable lives in %st(0) (when asked via an `=' command) if we said it was in DWARF regno 11, - but SDB still prints garbage when asked for the value of the + but SDB still printed garbage when asked for the value of the variable in question (via a `/' command). - (Also note that the labels SDB prints for various FP stack regs - when doing an `x' command are all wrong.) + (Also note that the labels SDB printed for various FP stack regs + when doing an `x' command were all wrong.) Note that these problems generally don't affect the native SVR4 C compiler because it doesn't allow the use of -O with -g and because when it is *not* optimizing, it allocates a memory @@ -346,15 +355,14 @@ { 0, 2, 1, 3, 6, 7, 5, 4, /* general regs */ 11, 12, 13, 14, 15, 16, 17, 18, /* fp regs */ - -1, 9, -1, -1, -1, /* arg, flags, fpsr, fpcr, frame */ + -1, 9, -1, -1, /* arg, flags, fpsr, frame */ 21, 22, 23, 24, 25, 26, 27, 28, /* SSE registers */ 29, 30, 31, 32, 33, 34, 35, 36, /* MMX registers */ -1, -1, -1, -1, -1, -1, -1, -1, /* extended integer registers */ -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */ -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 16-23*/ -1, -1, -1, -1, -1, -1, -1, -1, /* AVX-512 registers 24-31*/ - 93, 94, 95, 96, 97, 98, 99, 100, /* Mask registers */ - 101, 102, 103, 104, /* bound registers */ + 93, 94, 95, 96, 97, 98, 99, 100 /* Mask registers */ }; /* Define parameter passing and return registers. */ @@ -820,51 +828,46 @@ static tree ix86_veclibabi_svml (combined_fn, tree, tree); static tree ix86_veclibabi_acml (combined_fn, tree, tree); -/* Processor target table, indexed by processor number */ -struct ptt -{ - const char *const name; /* processor name */ - const struct processor_costs *cost; /* Processor costs */ - const int align_loop; /* Default alignments. */ - const int align_loop_max_skip; - const int align_jump; - const int align_jump_max_skip; - const int align_func; -}; - /* This table must be in sync with enum processor_type in i386.h. */ -static const struct ptt processor_target_table[PROCESSOR_max] = -{ - {"generic", &generic_cost, 16, 10, 16, 10, 16}, - {"i386", &i386_cost, 4, 3, 4, 3, 4}, - {"i486", &i486_cost, 16, 15, 16, 15, 16}, - {"pentium", &pentium_cost, 16, 7, 16, 7, 16}, - {"lakemont", &lakemont_cost, 16, 7, 16, 7, 16}, - {"pentiumpro", &pentiumpro_cost, 16, 15, 16, 10, 16}, - {"pentium4", &pentium4_cost, 0, 0, 0, 0, 0}, - {"nocona", &nocona_cost, 0, 0, 0, 0, 0}, - {"core2", &core_cost, 16, 10, 16, 10, 16}, - {"nehalem", &core_cost, 16, 10, 16, 10, 16}, - {"sandybridge", &core_cost, 16, 10, 16, 10, 16}, - {"haswell", &core_cost, 16, 10, 16, 10, 16}, - {"bonnell", &atom_cost, 16, 15, 16, 7, 16}, - {"silvermont", &slm_cost, 16, 15, 16, 7, 16}, - {"knl", &slm_cost, 16, 15, 16, 7, 16}, - {"knm", &slm_cost, 16, 15, 16, 7, 16}, - {"skylake-avx512", &core_cost, 16, 10, 16, 10, 16}, - {"intel", &intel_cost, 16, 15, 16, 7, 16}, - {"geode", &geode_cost, 0, 0, 0, 0, 0}, - {"k6", &k6_cost, 32, 7, 32, 7, 32}, - {"athlon", &athlon_cost, 16, 7, 16, 7, 16}, - {"k8", &k8_cost, 16, 7, 16, 7, 16}, - {"amdfam10", &amdfam10_cost, 32, 24, 32, 7, 32}, - {"bdver1", &bdver1_cost, 16, 10, 16, 7, 11}, - {"bdver2", &bdver2_cost, 16, 10, 16, 7, 11}, - {"bdver3", &bdver3_cost, 16, 10, 16, 7, 11}, - {"bdver4", &bdver4_cost, 16, 10, 16, 7, 11}, - {"btver1", &btver1_cost, 16, 10, 16, 7, 11}, - {"btver2", &btver2_cost, 16, 10, 16, 7, 11}, - {"znver1", &znver1_cost, 16, 15, 16, 15, 16} +static const struct processor_costs *processor_cost_table[PROCESSOR_max] = +{ + &generic_cost, + &i386_cost, + &i486_cost, + &pentium_cost, + &lakemont_cost, + &pentiumpro_cost, + &pentium4_cost, + &nocona_cost, + &core_cost, + &core_cost, + &core_cost, + &core_cost, + &atom_cost, + &slm_cost, + &slm_cost, + &slm_cost, + &slm_cost, + &slm_cost, + &slm_cost, + &skylake_cost, + &skylake_cost, + &skylake_cost, + &skylake_cost, + &skylake_cost, + &intel_cost, + &geode_cost, + &k6_cost, + &athlon_cost, + &k8_cost, + &amdfam10_cost, + &bdver_cost, + &bdver_cost, + &bdver_cost, + &bdver_cost, + &btver1_cost, + &btver2_cost, + &znver1_cost, }; static unsigned int @@ -873,7 +876,7 @@ int i; /* vzeroupper instructions are inserted immediately after reload to - account for possible spills from 256bit registers. The pass + account for possible spills from 256bit or 512bit registers. The pass reuses mode switching infrastructure by re-running mode insertion pass, so disable entities that have already been processed. */ for (i = 0; i < MAX_386_ENTITIES; i++) @@ -2495,7 +2498,7 @@ /* opt_pass methods: */ virtual bool gate (function *) { - return TARGET_AVX && !TARGET_AVX512F + return TARGET_AVX && TARGET_VZEROUPPER && flag_expensive_optimizations && !optimize_size; } @@ -2590,11 +2593,17 @@ TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl))) && !cgraph_node::get (cfun->decl)->only_called_directly_p ()) { - cet_eb = gen_nop_endbr (); - - bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb; - insn = BB_HEAD (bb); - emit_insn_before (cet_eb, insn); + /* Queue ENDBR insertion to x86_function_profiler. */ + if (crtl->profile && flag_fentry) + cfun->machine->endbr_queued_at_entrance = true; + else + { + cet_eb = gen_nop_endbr (); + + bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb; + insn = BB_HEAD (bb); + emit_insn_before (cet_eb, insn); + } } bb = 0; @@ -2603,27 +2612,45 @@ for (insn = BB_HEAD (bb); insn != NEXT_INSN (BB_END (bb)); insn = NEXT_INSN (insn)) { - if (INSN_P (insn) && GET_CODE (insn) == CALL_INSN) - { - rtx_insn *next_insn = insn; - - while ((next_insn != BB_END (bb)) - && (DEBUG_INSN_P (NEXT_INSN (next_insn)) - || NOTE_P (NEXT_INSN (next_insn)) - || BARRIER_P (NEXT_INSN (next_insn)))) - next_insn = NEXT_INSN (next_insn); - + if (CALL_P (insn)) + { + bool need_endbr; + need_endbr = find_reg_note (insn, REG_SETJMP, NULL) != NULL; + if (!need_endbr && !SIBLING_CALL_P (insn)) + { + rtx call = get_call_rtx_from (insn); + rtx fnaddr = XEXP (call, 0); + tree fndecl = NULL_TREE; + + /* Also generate ENDBRANCH for non-tail call which + may return via indirect branch. */ + if (GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF) + fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0)); + if (fndecl == NULL_TREE) + fndecl = MEM_EXPR (fnaddr); + if (fndecl + && TREE_CODE (TREE_TYPE (fndecl)) != FUNCTION_TYPE + && TREE_CODE (TREE_TYPE (fndecl)) != METHOD_TYPE) + fndecl = NULL_TREE; + if (fndecl && TYPE_ARG_TYPES (TREE_TYPE (fndecl))) + { + tree fntype = TREE_TYPE (fndecl); + if (lookup_attribute ("indirect_return", + TYPE_ATTRIBUTES (fntype))) + need_endbr = true; + } + } + if (!need_endbr) + continue; /* Generate ENDBRANCH after CALL, which can return more than twice, setjmp-like functions. */ - if (find_reg_note (insn, REG_SETJMP, NULL) != NULL) - { - cet_eb = gen_nop_endbr (); - emit_insn_after (cet_eb, next_insn); - } + + cet_eb = gen_nop_endbr (); + emit_insn_after_setloc (cet_eb, insn, INSN_LOCATION (insn)); continue; } - if (INSN_P (insn) && JUMP_P (insn) && flag_cet_switch) + if (JUMP_P (insn) && flag_cet_switch) { rtx target = JUMP_LABEL (insn); if (target == NULL_RTX || ANY_RETURN_P (target)) @@ -2658,7 +2685,7 @@ if ((LABEL_P (insn) && LABEL_PRESERVE_P (insn)) || (NOTE_P (insn) && NOTE_KIND (insn) == NOTE_INSN_DELETED_LABEL)) -/* TODO. Check /s bit also. */ + /* TODO. Check /s bit also. */ { cet_eb = gen_nop_endbr (); emit_insn_after (cet_eb, insn); @@ -2696,7 +2723,7 @@ /* opt_pass methods: */ virtual bool gate (function *) { - return ((flag_cf_protection & CF_BRANCH) && TARGET_IBT); + return ((flag_cf_protection & CF_BRANCH)); } virtual unsigned int execute (function *) @@ -2714,12 +2741,23 @@ return new pass_insert_endbranch (ctxt); } -/* Return true if a red-zone is in use. */ +/* Return true if a red-zone is in use. We can't use red-zone when + there are local indirect jumps, like "indirect_jump" or "tablejump", + which jumps to another place in the function, since "call" in the + indirect thunk pushes the return address onto stack, destroying + red-zone. + + TODO: If we can reserve the first 2 WORDs, for PUSH and, another + for CALL, in red-zone, we can allow local indirect jumps with + indirect thunk. */ bool ix86_using_red_zone (void) { - return TARGET_RED_ZONE && !TARGET_64BIT_MS_ABI; + return (TARGET_RED_ZONE + && !TARGET_64BIT_MS_ABI + && (!cfun->machine->has_local_indirect_jump + || cfun->machine->indirect_branch_type == indirect_branch_keep)); } /* Return a string that documents the current -m options. The caller is @@ -2741,17 +2779,30 @@ ISAs come first. Target string will be displayed in the same order. */ static struct ix86_target_opts isa2_opts[] = { - { "-mgfni", OPTION_MASK_ISA_GFNI }, + { "-mcx16", OPTION_MASK_ISA_CX16 }, + { "-mvaes", OPTION_MASK_ISA_VAES }, { "-mrdpid", OPTION_MASK_ISA_RDPID }, + { "-mpconfig", OPTION_MASK_ISA_PCONFIG }, + { "-mwbnoinvd", OPTION_MASK_ISA_WBNOINVD }, { "-msgx", OPTION_MASK_ISA_SGX }, { "-mavx5124vnniw", OPTION_MASK_ISA_AVX5124VNNIW }, { "-mavx5124fmaps", OPTION_MASK_ISA_AVX5124FMAPS }, - { "-mavx512vpopcntdq", OPTION_MASK_ISA_AVX512VPOPCNTDQ }, - { "-mibt", OPTION_MASK_ISA_IBT }, - { "-mshstk", OPTION_MASK_ISA_SHSTK } + { "-mhle", OPTION_MASK_ISA_HLE }, + { "-mmovbe", OPTION_MASK_ISA_MOVBE }, + { "-mclzero", OPTION_MASK_ISA_CLZERO }, + { "-mmwaitx", OPTION_MASK_ISA_MWAITX }, + { "-mmovdir64b", OPTION_MASK_ISA_MOVDIR64B }, + { "-mwaitpkg", OPTION_MASK_ISA_WAITPKG }, + { "-mcldemote", OPTION_MASK_ISA_CLDEMOTE } }; static struct ix86_target_opts isa_opts[] = { + { "-mavx512vpopcntdq", OPTION_MASK_ISA_AVX512VPOPCNTDQ }, + { "-mavx512bitalg", OPTION_MASK_ISA_AVX512BITALG }, + { "-mvpclmulqdq", OPTION_MASK_ISA_VPCLMULQDQ }, + { "-mgfni", OPTION_MASK_ISA_GFNI }, + { "-mavx512vnni", OPTION_MASK_ISA_AVX512VNNI }, + { "-mavx512vbmi2", OPTION_MASK_ISA_AVX512VBMI2 }, { "-mavx512vbmi", OPTION_MASK_ISA_AVX512VBMI }, { "-mavx512ifma", OPTION_MASK_ISA_AVX512IFMA }, { "-mavx512vl", OPTION_MASK_ISA_AVX512VL }, @@ -2797,20 +2848,16 @@ { "-mlzcnt", OPTION_MASK_ISA_LZCNT }, { "-mtbm", OPTION_MASK_ISA_TBM }, { "-mpopcnt", OPTION_MASK_ISA_POPCNT }, - { "-mcx16", OPTION_MASK_ISA_CX16 }, { "-msahf", OPTION_MASK_ISA_SAHF }, - { "-mmovbe", OPTION_MASK_ISA_MOVBE }, { "-mcrc32", OPTION_MASK_ISA_CRC32 }, { "-mfsgsbase", OPTION_MASK_ISA_FSGSBASE }, { "-mrdrnd", OPTION_MASK_ISA_RDRND }, - { "-mmwaitx", OPTION_MASK_ISA_MWAITX }, - { "-mclzero", OPTION_MASK_ISA_CLZERO }, { "-mpku", OPTION_MASK_ISA_PKU }, { "-mlwp", OPTION_MASK_ISA_LWP }, - { "-mhle", OPTION_MASK_ISA_HLE }, { "-mfxsr", OPTION_MASK_ISA_FXSR }, - { "-mmpx", OPTION_MASK_ISA_MPX }, - { "-mclwb", OPTION_MASK_ISA_CLWB } + { "-mclwb", OPTION_MASK_ISA_CLWB }, + { "-mshstk", OPTION_MASK_ISA_SHSTK }, + { "-mmovdiri", OPTION_MASK_ISA_MOVDIRI } }; /* Flag options. */ @@ -2844,15 +2891,13 @@ { "-mstv", MASK_STV }, { "-mavx256-split-unaligned-load", MASK_AVX256_SPLIT_UNALIGNED_LOAD }, { "-mavx256-split-unaligned-store", MASK_AVX256_SPLIT_UNALIGNED_STORE }, - { "-mprefer-avx128", MASK_PREFER_AVX128 }, { "-mcall-ms2sysv-xlogues", MASK_CALL_MS2SYSV_XLOGUES } }; /* Additional flag options. */ static struct ix86_target_opts flag2_opts[] = { - { "-mgeneral-regs-only", OPTION_MASK_GENERAL_REGS_ONLY }, - { "-mprefer-avx256", OPTION_MASK_PREFER_AVX256 }, + { "-mgeneral-regs-only", OPTION_MASK_GENERAL_REGS_ONLY } }; const char *opts[ARRAY_SIZE (isa_opts) + ARRAY_SIZE (isa2_opts) @@ -3075,15 +3120,6 @@ return; } -/* Return true if T is one of the bytes we should avoid with - -fmitigate-rop. */ - -static bool -ix86_rop_should_change_byte_p (int t) -{ - return t == 0xc2 || t == 0xc3 || t == 0xca || t == 0xcb; -} - static const char *stringop_alg_names[] = { #define DEF_ENUM #define DEF_ALG(alg, name) #name, @@ -3149,8 +3185,8 @@ if (next_range_str) *next_range_str++ = '\0'; - if (3 != sscanf (curr_range_str, "%20[^:]:%d:%10s", - alg_name, &maxs, align)) + if (sscanf (curr_range_str, "%20[^:]:%d:%10s", alg_name, &maxs, + align) != 3) { error ("wrong argument %qs to option %qs", curr_range_str, opt); return; @@ -3277,7 +3313,7 @@ } } if (i == X86_TUNE_LAST) - error ("Unknown parameter to option -mtune-ctrl: %s", + error ("unknown parameter to option -mtune-ctrl: %s", clear ? curr_feature_string - 1 : curr_feature_string); curr_feature_string = next_feature_string; } @@ -3291,7 +3327,7 @@ static void set_ix86_tune_features (enum processor_type ix86_tune, bool dump) { - unsigned int ix86_tune_mask = 1u << ix86_tune; + unsigned HOST_WIDE_INT ix86_tune_mask = HOST_WIDE_INT_1U << ix86_tune; int i; for (i = 0; i < X86_TUNE_LAST; ++i) @@ -3299,7 +3335,8 @@ if (ix86_tune_no_default) ix86_tune_features[i] = 0; else - ix86_tune_features[i] = !!(initial_ix86_tune_features[i] & ix86_tune_mask); + ix86_tune_features[i] + = !!(initial_ix86_tune_features[i] & ix86_tune_mask); } if (dump) @@ -3319,20 +3356,15 @@ static void ix86_default_align (struct gcc_options *opts) { - if (opts->x_align_loops == 0) - { - opts->x_align_loops = processor_target_table[ix86_tune].align_loop; - align_loops_max_skip = processor_target_table[ix86_tune].align_loop_max_skip; - } - if (opts->x_align_jumps == 0) - { - opts->x_align_jumps = processor_target_table[ix86_tune].align_jump; - align_jumps_max_skip = processor_target_table[ix86_tune].align_jump_max_skip; - } - if (opts->x_align_functions == 0) - { - opts->x_align_functions = processor_target_table[ix86_tune].align_func; - } + /* -falign-foo without argument: supply one. */ + if (opts->x_flag_align_loops && !opts->x_str_align_loops) + opts->x_str_align_loops = processor_cost_table[ix86_tune]->align_loop; + if (opts->x_flag_align_jumps && !opts->x_str_align_jumps) + opts->x_str_align_jumps = processor_cost_table[ix86_tune]->align_jump; + if (opts->x_flag_align_labels && !opts->x_str_align_labels) + opts->x_str_align_labels = processor_cost_table[ix86_tune]->align_label; + if (opts->x_flag_align_functions && !opts->x_str_align_functions) + opts->x_str_align_functions = processor_cost_table[ix86_tune]->align_func; } /* Implement TARGET_OVERRIDE_OPTIONS_AFTER_CHANGE hook. */ @@ -3343,6 +3375,8 @@ ix86_default_align (&global_options); } + + /* Override various settings based on options. If MAIN_ARGS_P, the options are from the command line, otherwise they are from attributes. Return true if there's an error related to march @@ -3354,293 +3388,9 @@ struct gcc_options *opts_set) { int i; - unsigned int ix86_arch_mask; + unsigned HOST_WIDE_INT ix86_arch_mask; const bool ix86_tune_specified = (opts->x_ix86_tune_string != NULL); -#define PTA_3DNOW (HOST_WIDE_INT_1 << 0) -#define PTA_3DNOW_A (HOST_WIDE_INT_1 << 1) -#define PTA_64BIT (HOST_WIDE_INT_1 << 2) -#define PTA_ABM (HOST_WIDE_INT_1 << 3) -#define PTA_AES (HOST_WIDE_INT_1 << 4) -#define PTA_AVX (HOST_WIDE_INT_1 << 5) -#define PTA_BMI (HOST_WIDE_INT_1 << 6) -#define PTA_CX16 (HOST_WIDE_INT_1 << 7) -#define PTA_F16C (HOST_WIDE_INT_1 << 8) -#define PTA_FMA (HOST_WIDE_INT_1 << 9) -#define PTA_FMA4 (HOST_WIDE_INT_1 << 10) -#define PTA_FSGSBASE (HOST_WIDE_INT_1 << 11) -#define PTA_LWP (HOST_WIDE_INT_1 << 12) -#define PTA_LZCNT (HOST_WIDE_INT_1 << 13) -#define PTA_MMX (HOST_WIDE_INT_1 << 14) -#define PTA_MOVBE (HOST_WIDE_INT_1 << 15) -#define PTA_NO_SAHF (HOST_WIDE_INT_1 << 16) -#define PTA_PCLMUL (HOST_WIDE_INT_1 << 17) -#define PTA_POPCNT (HOST_WIDE_INT_1 << 18) -#define PTA_PREFETCH_SSE (HOST_WIDE_INT_1 << 19) -#define PTA_RDRND (HOST_WIDE_INT_1 << 20) -#define PTA_SSE (HOST_WIDE_INT_1 << 21) -#define PTA_SSE2 (HOST_WIDE_INT_1 << 22) -#define PTA_SSE3 (HOST_WIDE_INT_1 << 23) -#define PTA_SSE4_1 (HOST_WIDE_INT_1 << 24) -#define PTA_SSE4_2 (HOST_WIDE_INT_1 << 25) -#define PTA_SSE4A (HOST_WIDE_INT_1 << 26) -#define PTA_SSSE3 (HOST_WIDE_INT_1 << 27) -#define PTA_TBM (HOST_WIDE_INT_1 << 28) -#define PTA_XOP (HOST_WIDE_INT_1 << 29) -#define PTA_AVX2 (HOST_WIDE_INT_1 << 30) -#define PTA_BMI2 (HOST_WIDE_INT_1 << 31) -#define PTA_RTM (HOST_WIDE_INT_1 << 32) -#define PTA_HLE (HOST_WIDE_INT_1 << 33) -#define PTA_PRFCHW (HOST_WIDE_INT_1 << 34) -#define PTA_RDSEED (HOST_WIDE_INT_1 << 35) -#define PTA_ADX (HOST_WIDE_INT_1 << 36) -#define PTA_FXSR (HOST_WIDE_INT_1 << 37) -#define PTA_XSAVE (HOST_WIDE_INT_1 << 38) -#define PTA_XSAVEOPT (HOST_WIDE_INT_1 << 39) -#define PTA_AVX512F (HOST_WIDE_INT_1 << 40) -#define PTA_AVX512ER (HOST_WIDE_INT_1 << 41) -#define PTA_AVX512PF (HOST_WIDE_INT_1 << 42) -#define PTA_AVX512CD (HOST_WIDE_INT_1 << 43) -#define PTA_MPX (HOST_WIDE_INT_1 << 44) -#define PTA_SHA (HOST_WIDE_INT_1 << 45) -#define PTA_PREFETCHWT1 (HOST_WIDE_INT_1 << 46) -#define PTA_CLFLUSHOPT (HOST_WIDE_INT_1 << 47) -#define PTA_XSAVEC (HOST_WIDE_INT_1 << 48) -#define PTA_XSAVES (HOST_WIDE_INT_1 << 49) -#define PTA_AVX512DQ (HOST_WIDE_INT_1 << 50) -#define PTA_AVX512BW (HOST_WIDE_INT_1 << 51) -#define PTA_AVX512VL (HOST_WIDE_INT_1 << 52) -#define PTA_AVX512IFMA (HOST_WIDE_INT_1 << 53) -#define PTA_AVX512VBMI (HOST_WIDE_INT_1 << 54) -#define PTA_CLWB (HOST_WIDE_INT_1 << 55) -#define PTA_MWAITX (HOST_WIDE_INT_1 << 56) -#define PTA_CLZERO (HOST_WIDE_INT_1 << 57) -#define PTA_NO_80387 (HOST_WIDE_INT_1 << 58) -#define PTA_PKU (HOST_WIDE_INT_1 << 59) -#define PTA_AVX5124VNNIW (HOST_WIDE_INT_1 << 60) -#define PTA_AVX5124FMAPS (HOST_WIDE_INT_1 << 61) -#define PTA_AVX512VPOPCNTDQ (HOST_WIDE_INT_1 << 62) -#define PTA_SGX (HOST_WIDE_INT_1 << 63) - -#define PTA_CORE2 \ - (PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSSE3 \ - | PTA_CX16 | PTA_FXSR) -#define PTA_NEHALEM \ - (PTA_CORE2 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_POPCNT) -#define PTA_WESTMERE \ - (PTA_NEHALEM | PTA_AES | PTA_PCLMUL) -#define PTA_SANDYBRIDGE \ - (PTA_WESTMERE | PTA_AVX | PTA_XSAVE | PTA_XSAVEOPT) -#define PTA_IVYBRIDGE \ - (PTA_SANDYBRIDGE | PTA_FSGSBASE | PTA_RDRND | PTA_F16C) -#define PTA_HASWELL \ - (PTA_IVYBRIDGE | PTA_AVX2 | PTA_BMI | PTA_BMI2 | PTA_LZCNT \ - | PTA_FMA | PTA_MOVBE | PTA_HLE) -#define PTA_BROADWELL \ - (PTA_HASWELL | PTA_ADX | PTA_PRFCHW | PTA_RDSEED) -#define PTA_SKYLAKE \ - (PTA_BROADWELL | PTA_CLFLUSHOPT | PTA_XSAVEC | PTA_XSAVES) -#define PTA_SKYLAKE_AVX512 \ - (PTA_SKYLAKE | PTA_AVX512F | PTA_AVX512CD | PTA_AVX512VL \ - | PTA_AVX512BW | PTA_AVX512DQ | PTA_PKU) -#define PTA_KNL \ - (PTA_BROADWELL | PTA_AVX512PF | PTA_AVX512ER | PTA_AVX512F | PTA_AVX512CD) -#define PTA_BONNELL \ - (PTA_CORE2 | PTA_MOVBE) -#define PTA_SILVERMONT \ - (PTA_WESTMERE | PTA_MOVBE) -#define PTA_KNM \ - (PTA_KNL | PTA_AVX5124VNNIW | PTA_AVX5124FMAPS | PTA_AVX512VPOPCNTDQ) - -/* if this reaches 64, need to widen struct pta flags below */ - - static struct pta - { - const char *const name; /* processor name or nickname. */ - const enum processor_type processor; - const enum attr_cpu schedule; - const unsigned HOST_WIDE_INT flags; - } - const processor_alias_table[] = - { - {"i386", PROCESSOR_I386, CPU_NONE, 0}, - {"i486", PROCESSOR_I486, CPU_NONE, 0}, - {"i586", PROCESSOR_PENTIUM, CPU_PENTIUM, 0}, - {"pentium", PROCESSOR_PENTIUM, CPU_PENTIUM, 0}, - {"lakemont", PROCESSOR_LAKEMONT, CPU_PENTIUM, PTA_NO_80387}, - {"pentium-mmx", PROCESSOR_PENTIUM, CPU_PENTIUM, PTA_MMX}, - {"winchip-c6", PROCESSOR_I486, CPU_NONE, PTA_MMX}, - {"winchip2", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW}, - {"c3", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW}, - {"samuel-2", PROCESSOR_I486, CPU_NONE, PTA_MMX | PTA_3DNOW}, - {"c3-2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, - PTA_MMX | PTA_SSE | PTA_FXSR}, - {"nehemiah", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, - PTA_MMX | PTA_SSE | PTA_FXSR}, - {"c7", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, - PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR}, - {"esther", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, - PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR}, - {"i686", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0}, - {"pentiumpro", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, 0}, - {"pentium2", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, PTA_MMX | PTA_FXSR}, - {"pentium3", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, - PTA_MMX | PTA_SSE | PTA_FXSR}, - {"pentium3m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, - PTA_MMX | PTA_SSE | PTA_FXSR}, - {"pentium-m", PROCESSOR_PENTIUMPRO, CPU_PENTIUMPRO, - PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_FXSR}, - {"pentium4", PROCESSOR_PENTIUM4, CPU_NONE, - PTA_MMX |PTA_SSE | PTA_SSE2 | PTA_FXSR}, - {"pentium4m", PROCESSOR_PENTIUM4, CPU_NONE, - PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_FXSR}, - {"prescott", PROCESSOR_NOCONA, CPU_NONE, - PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR}, - {"nocona", PROCESSOR_NOCONA, CPU_NONE, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_CX16 | PTA_NO_SAHF | PTA_FXSR}, - {"core2", PROCESSOR_CORE2, CPU_CORE2, PTA_CORE2}, - {"nehalem", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_NEHALEM}, - {"corei7", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_NEHALEM}, - {"westmere", PROCESSOR_NEHALEM, CPU_NEHALEM, PTA_WESTMERE}, - {"sandybridge", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM, - PTA_SANDYBRIDGE}, - {"corei7-avx", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM, - PTA_SANDYBRIDGE}, - {"ivybridge", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM, - PTA_IVYBRIDGE}, - {"core-avx-i", PROCESSOR_SANDYBRIDGE, CPU_NEHALEM, - PTA_IVYBRIDGE}, - {"haswell", PROCESSOR_HASWELL, CPU_HASWELL, PTA_HASWELL}, - {"core-avx2", PROCESSOR_HASWELL, CPU_HASWELL, PTA_HASWELL}, - {"broadwell", PROCESSOR_HASWELL, CPU_HASWELL, PTA_BROADWELL}, - {"skylake", PROCESSOR_HASWELL, CPU_HASWELL, PTA_SKYLAKE}, - {"skylake-avx512", PROCESSOR_HASWELL, CPU_HASWELL, PTA_SKYLAKE_AVX512}, - {"bonnell", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL}, - {"atom", PROCESSOR_BONNELL, CPU_ATOM, PTA_BONNELL}, - {"silvermont", PROCESSOR_SILVERMONT, CPU_SLM, PTA_SILVERMONT}, - {"slm", PROCESSOR_SILVERMONT, CPU_SLM, PTA_SILVERMONT}, - {"knl", PROCESSOR_KNL, CPU_SLM, PTA_KNL}, - {"knm", PROCESSOR_KNM, CPU_SLM, PTA_KNM}, - {"intel", PROCESSOR_INTEL, CPU_SLM, PTA_NEHALEM}, - {"geode", PROCESSOR_GEODE, CPU_GEODE, - PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE}, - {"k6", PROCESSOR_K6, CPU_K6, PTA_MMX}, - {"k6-2", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW}, - {"k6-3", PROCESSOR_K6, CPU_K6, PTA_MMX | PTA_3DNOW}, - {"athlon", PROCESSOR_ATHLON, CPU_ATHLON, - PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE}, - {"athlon-tbird", PROCESSOR_ATHLON, CPU_ATHLON, - PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_PREFETCH_SSE}, - {"athlon-4", PROCESSOR_ATHLON, CPU_ATHLON, - PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_FXSR}, - {"athlon-xp", PROCESSOR_ATHLON, CPU_ATHLON, - PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_FXSR}, - {"athlon-mp", PROCESSOR_ATHLON, CPU_ATHLON, - PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_FXSR}, - {"x86-64", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR}, - {"eden-x2", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_FXSR}, - {"nano", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_FXSR}, - {"nano-1000", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_FXSR}, - {"nano-2000", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_FXSR}, - {"nano-3000", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_SSE4_1 | PTA_FXSR}, - {"nano-x2", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_SSE4_1 | PTA_FXSR}, - {"eden-x4", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_SSE4_1 | PTA_FXSR}, - {"nano-x4", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_SSE4_1 | PTA_FXSR}, - {"k8", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE - | PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR}, - {"k8-sse3", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE - | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_FXSR}, - {"opteron", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE - | PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR}, - {"opteron-sse3", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE - | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_FXSR}, - {"athlon64", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE - | PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR}, - {"athlon64-sse3", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE - | PTA_SSE2 | PTA_SSE3 | PTA_NO_SAHF | PTA_FXSR}, - {"athlon-fx", PROCESSOR_K8, CPU_K8, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE - | PTA_SSE2 | PTA_NO_SAHF | PTA_FXSR}, - {"amdfam10", PROCESSOR_AMDFAM10, CPU_AMDFAM10, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2 - | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_PRFCHW | PTA_FXSR}, - {"barcelona", PROCESSOR_AMDFAM10, CPU_AMDFAM10, - PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2 - | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_PRFCHW | PTA_FXSR}, - {"bdver1", PROCESSOR_BDVER1, CPU_BDVER1, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1 - | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4 - | PTA_XOP | PTA_LWP | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE}, - {"bdver2", PROCESSOR_BDVER2, CPU_BDVER2, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1 - | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4 - | PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C - | PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE}, - {"bdver3", PROCESSOR_BDVER3, CPU_BDVER3, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1 - | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4 - | PTA_XOP | PTA_LWP | PTA_BMI | PTA_TBM | PTA_F16C - | PTA_FMA | PTA_PRFCHW | PTA_FXSR | PTA_XSAVE - | PTA_XSAVEOPT | PTA_FSGSBASE}, - {"bdver4", PROCESSOR_BDVER4, CPU_BDVER4, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1 - | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_AVX2 - | PTA_FMA4 | PTA_XOP | PTA_LWP | PTA_BMI | PTA_BMI2 - | PTA_TBM | PTA_F16C | PTA_FMA | PTA_PRFCHW | PTA_FXSR - | PTA_XSAVE | PTA_XSAVEOPT | PTA_FSGSBASE | PTA_RDRND - | PTA_MOVBE | PTA_MWAITX}, - {"znver1", PROCESSOR_ZNVER1, CPU_ZNVER1, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1 - | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_AVX2 - | PTA_BMI | PTA_BMI2 | PTA_F16C | PTA_FMA | PTA_PRFCHW - | PTA_FXSR | PTA_XSAVE | PTA_XSAVEOPT | PTA_FSGSBASE - | PTA_RDRND | PTA_MOVBE | PTA_MWAITX | PTA_ADX | PTA_RDSEED - | PTA_CLZERO | PTA_CLFLUSHOPT | PTA_XSAVEC | PTA_XSAVES - | PTA_SHA | PTA_LZCNT | PTA_POPCNT}, - {"btver1", PROCESSOR_BTVER1, CPU_GENERIC, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16 | PTA_PRFCHW - | PTA_FXSR | PTA_XSAVE}, - {"btver2", PROCESSOR_BTVER2, CPU_BTVER2, - PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 - | PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16 | PTA_SSE4_1 - | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX - | PTA_BMI | PTA_F16C | PTA_MOVBE | PTA_PRFCHW - | PTA_FXSR | PTA_XSAVE | PTA_XSAVEOPT}, - - {"generic", PROCESSOR_GENERIC, CPU_GENERIC, - PTA_64BIT - | PTA_HLE /* flags are only used for -march switch. */ }, - }; - /* -mrecip options. */ static struct { @@ -3657,7 +3407,6 @@ { "vec-sqrt", RECIP_MASK_VEC_SQRT }, }; - int const pta_size = ARRAY_SIZE (processor_alias_table); /* Turn off both OPTION_MASK_ABI_64 and OPTION_MASK_ABI_X32 if TARGET_64BIT_DEFAULT is true and TARGET_64BIT is false. */ @@ -3745,8 +3494,7 @@ opts->x_ix86_tune_string = opts->x_ix86_arch_string; if (!opts->x_ix86_tune_string) { - opts->x_ix86_tune_string - = processor_target_table[TARGET_CPU_DEFAULT].name; + opts->x_ix86_tune_string = processor_names[TARGET_CPU_DEFAULT]; ix86_tune_defaulted = 1; } @@ -3907,7 +3655,7 @@ } if (TARGET_64BIT_P (opts->x_ix86_isa_flags) - && !(processor_alias_table[i].flags & PTA_64BIT)) + && !((processor_alias_table[i].flags & PTA_64BIT) != 0)) { error ("CPU you selected does not support x86-64 " "instruction set"); @@ -3919,192 +3667,222 @@ /* Default cpu tuning to the architecture. */ ix86_tune = ix86_arch; - if (processor_alias_table[i].flags & PTA_MMX + if (((processor_alias_table[i].flags & PTA_MMX) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MMX)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MMX; - if (processor_alias_table[i].flags & PTA_3DNOW + if (((processor_alias_table[i].flags & PTA_3DNOW) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW; - if (processor_alias_table[i].flags & PTA_3DNOW_A + if (((processor_alias_table[i].flags & PTA_3DNOW_A) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_3DNOW_A)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_3DNOW_A; - if (processor_alias_table[i].flags & PTA_SSE + if (((processor_alias_table[i].flags & PTA_SSE) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE; - if (processor_alias_table[i].flags & PTA_SSE2 + if (((processor_alias_table[i].flags & PTA_SSE2) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE2)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE2; - if (processor_alias_table[i].flags & PTA_SSE3 + if (((processor_alias_table[i].flags & PTA_SSE3) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE3)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE3; - if (processor_alias_table[i].flags & PTA_SSSE3 + if (((processor_alias_table[i].flags & PTA_SSSE3) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSSE3)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSSE3; - if (processor_alias_table[i].flags & PTA_SSE4_1 + if (((processor_alias_table[i].flags & PTA_SSE4_1) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_1)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_1; - if (processor_alias_table[i].flags & PTA_SSE4_2 + if (((processor_alias_table[i].flags & PTA_SSE4_2) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4_2)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4_2; - if (processor_alias_table[i].flags & PTA_AVX + if (((processor_alias_table[i].flags & PTA_AVX) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX; - if (processor_alias_table[i].flags & PTA_AVX2 + if (((processor_alias_table[i].flags & PTA_AVX2) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX2)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX2; - if (processor_alias_table[i].flags & PTA_FMA + if (((processor_alias_table[i].flags & PTA_FMA) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA; - if (processor_alias_table[i].flags & PTA_SSE4A + if (((processor_alias_table[i].flags & PTA_SSE4A) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SSE4A)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SSE4A; - if (processor_alias_table[i].flags & PTA_FMA4 + if (((processor_alias_table[i].flags & PTA_FMA4) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FMA4)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FMA4; - if (processor_alias_table[i].flags & PTA_XOP + if (((processor_alias_table[i].flags & PTA_XOP) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XOP)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XOP; - if (processor_alias_table[i].flags & PTA_LWP + if (((processor_alias_table[i].flags & PTA_LWP) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LWP)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LWP; - if (processor_alias_table[i].flags & PTA_ABM + if (((processor_alias_table[i].flags & PTA_ABM) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ABM; - if (processor_alias_table[i].flags & PTA_BMI + if (((processor_alias_table[i].flags & PTA_BMI) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI; - if (processor_alias_table[i].flags & (PTA_LZCNT | PTA_ABM) + if (((processor_alias_table[i].flags & (PTA_LZCNT | PTA_ABM)) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_LZCNT)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_LZCNT; - if (processor_alias_table[i].flags & PTA_TBM + if (((processor_alias_table[i].flags & PTA_TBM) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_TBM)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_TBM; - if (processor_alias_table[i].flags & PTA_BMI2 + if (((processor_alias_table[i].flags & PTA_BMI2) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI2)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_BMI2; - if (processor_alias_table[i].flags & PTA_CX16 - && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16)) - opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CX16; - if (processor_alias_table[i].flags & (PTA_POPCNT | PTA_ABM) + if (((processor_alias_table[i].flags & PTA_CX16) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_CX16)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_CX16; + if (((processor_alias_table[i].flags & (PTA_POPCNT | PTA_ABM)) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_POPCNT)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_POPCNT; if (!(TARGET_64BIT_P (opts->x_ix86_isa_flags) - && (processor_alias_table[i].flags & PTA_NO_SAHF)) + && ((processor_alias_table[i].flags & PTA_NO_SAHF) != 0)) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_SAHF)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_SAHF; - if (processor_alias_table[i].flags & PTA_MOVBE - && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MOVBE)) - opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MOVBE; - if (processor_alias_table[i].flags & PTA_AES + if (((processor_alias_table[i].flags & PTA_MOVBE) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_MOVBE)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_MOVBE; + if (((processor_alias_table[i].flags & PTA_AES) != 0) && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_AES)) ix86_isa_flags |= OPTION_MASK_ISA_AES; - if (processor_alias_table[i].flags & PTA_SHA + if (((processor_alias_table[i].flags & PTA_SHA) != 0) && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_SHA)) ix86_isa_flags |= OPTION_MASK_ISA_SHA; - if (processor_alias_table[i].flags & PTA_PCLMUL + if (((processor_alias_table[i].flags & PTA_PCLMUL) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL; - if (processor_alias_table[i].flags & PTA_FSGSBASE + if (((processor_alias_table[i].flags & PTA_FSGSBASE) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FSGSBASE)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FSGSBASE; - if (processor_alias_table[i].flags & PTA_RDRND + if (((processor_alias_table[i].flags & PTA_RDRND) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDRND)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDRND; - if (processor_alias_table[i].flags & PTA_F16C + if (((processor_alias_table[i].flags & PTA_F16C) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_F16C)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_F16C; - if (processor_alias_table[i].flags & PTA_RTM + if (((processor_alias_table[i].flags & PTA_RTM) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RTM)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RTM; - if (processor_alias_table[i].flags & PTA_HLE - && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_HLE)) - opts->x_ix86_isa_flags |= OPTION_MASK_ISA_HLE; - if (processor_alias_table[i].flags & PTA_PRFCHW + if (((processor_alias_table[i].flags & PTA_HLE) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_HLE)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_HLE; + if (((processor_alias_table[i].flags & PTA_PRFCHW) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PRFCHW)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PRFCHW; - if (processor_alias_table[i].flags & PTA_RDSEED + if (((processor_alias_table[i].flags & PTA_RDSEED) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_RDSEED)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_RDSEED; - if (processor_alias_table[i].flags & PTA_ADX + if (((processor_alias_table[i].flags & PTA_ADX) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_ADX)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_ADX; - if (processor_alias_table[i].flags & PTA_FXSR + if (((processor_alias_table[i].flags & PTA_FXSR) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_FXSR)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_FXSR; - if (processor_alias_table[i].flags & PTA_XSAVE + if (((processor_alias_table[i].flags & PTA_XSAVE) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVE)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVE; - if (processor_alias_table[i].flags & PTA_XSAVEOPT + if (((processor_alias_table[i].flags & PTA_XSAVEOPT) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEOPT)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEOPT; - if (processor_alias_table[i].flags & PTA_AVX512F + if (((processor_alias_table[i].flags & PTA_AVX512F) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512F)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512F; - if (processor_alias_table[i].flags & PTA_AVX512ER + if (((processor_alias_table[i].flags & PTA_AVX512ER) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512ER)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512ER; - if (processor_alias_table[i].flags & PTA_AVX512PF + if (((processor_alias_table[i].flags & PTA_AVX512PF) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512PF)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512PF; - if (processor_alias_table[i].flags & PTA_AVX512CD + if (((processor_alias_table[i].flags & PTA_AVX512CD) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512CD)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512CD; - if (processor_alias_table[i].flags & PTA_PREFETCHWT1 + if (((processor_alias_table[i].flags & PTA_PREFETCHWT1) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PREFETCHWT1)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PREFETCHWT1; - if (processor_alias_table[i].flags & PTA_CLWB + if (((processor_alias_table[i].flags & PTA_CLWB) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLWB)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLWB; - if (processor_alias_table[i].flags & PTA_CLFLUSHOPT + if (((processor_alias_table[i].flags & PTA_CLFLUSHOPT) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLFLUSHOPT)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLFLUSHOPT; - if (processor_alias_table[i].flags & PTA_CLZERO - && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_CLZERO)) - opts->x_ix86_isa_flags |= OPTION_MASK_ISA_CLZERO; - if (processor_alias_table[i].flags & PTA_XSAVEC + if (((processor_alias_table[i].flags & PTA_CLZERO) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_CLZERO)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_CLZERO; + if (((processor_alias_table[i].flags & PTA_XSAVEC) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVEC)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVEC; - if (processor_alias_table[i].flags & PTA_XSAVES + if (((processor_alias_table[i].flags & PTA_XSAVES) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_XSAVES)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_XSAVES; - if (processor_alias_table[i].flags & PTA_AVX512DQ + if (((processor_alias_table[i].flags & PTA_AVX512DQ) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512DQ)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512DQ; - if (processor_alias_table[i].flags & PTA_AVX512BW + if (((processor_alias_table[i].flags & PTA_AVX512BW) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512BW)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512BW; - if (processor_alias_table[i].flags & PTA_AVX512VL + if (((processor_alias_table[i].flags & PTA_AVX512VL) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VL)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VL; - if (processor_alias_table[i].flags & PTA_MPX - && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MPX)) - opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MPX; - if (processor_alias_table[i].flags & PTA_AVX512VBMI + if (((processor_alias_table[i].flags & PTA_AVX512VBMI) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VBMI)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VBMI; - if (processor_alias_table[i].flags & PTA_AVX512IFMA + if (((processor_alias_table[i].flags & PTA_AVX512IFMA) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512IFMA)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512IFMA; - - if (processor_alias_table[i].flags & PTA_AVX5124VNNIW - && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_AVX5124VNNIW)) + if (((processor_alias_table[i].flags & PTA_AVX512VNNI) != 0) + && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_AVX512VNNI)) + opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VNNI; + if (((processor_alias_table[i].flags & PTA_GFNI) != 0) + && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_GFNI)) + opts->x_ix86_isa_flags |= OPTION_MASK_ISA_GFNI; + if (((processor_alias_table[i].flags & PTA_AVX512VBMI2) != 0) + && !(opts->x_ix86_isa_flags_explicit + & OPTION_MASK_ISA_AVX512VBMI2)) + opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VBMI2; + if (((processor_alias_table[i].flags & PTA_VPCLMULQDQ) != 0) + && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_VPCLMULQDQ)) + opts->x_ix86_isa_flags |= OPTION_MASK_ISA_VPCLMULQDQ; + if (((processor_alias_table[i].flags & PTA_AVX512BITALG) != 0) + && !(opts->x_ix86_isa_flags_explicit + & OPTION_MASK_ISA_AVX512BITALG)) + opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512BITALG; + + if (((processor_alias_table[i].flags & PTA_AVX5124VNNIW) != 0) + && !(opts->x_ix86_isa_flags2_explicit + & OPTION_MASK_ISA_AVX5124VNNIW)) opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_AVX5124VNNIW; - if (processor_alias_table[i].flags & PTA_AVX5124FMAPS - && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_AVX5124FMAPS)) + if (((processor_alias_table[i].flags & PTA_AVX5124FMAPS) != 0) + && !(opts->x_ix86_isa_flags2_explicit + & OPTION_MASK_ISA_AVX5124FMAPS)) opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_AVX5124FMAPS; - if (processor_alias_table[i].flags & PTA_AVX512VPOPCNTDQ - && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_AVX512VPOPCNTDQ)) - opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_AVX512VPOPCNTDQ; - if (processor_alias_table[i].flags & PTA_SGX + if (((processor_alias_table[i].flags & PTA_AVX512VPOPCNTDQ) != 0) + && !(opts->x_ix86_isa_flags_explicit + & OPTION_MASK_ISA_AVX512VPOPCNTDQ)) + opts->x_ix86_isa_flags |= OPTION_MASK_ISA_AVX512VPOPCNTDQ; + if (((processor_alias_table[i].flags & PTA_SGX) != 0) && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_SGX)) opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_SGX; - - if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)) + if (((processor_alias_table[i].flags & PTA_VAES) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_VAES)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_VAES; + if (((processor_alias_table[i].flags & PTA_RDPID) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_RDPID)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_RDPID; + if (((processor_alias_table[i].flags & PTA_PCONFIG) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_PCONFIG)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_PCONFIG; + if (((processor_alias_table[i].flags & PTA_WBNOINVD) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_WBNOINVD)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_WBNOINVD; + + if ((processor_alias_table[i].flags + & (PTA_PREFETCH_SSE | PTA_SSE)) != 0) x86_prefetch_sse = true; - if (processor_alias_table[i].flags & PTA_MWAITX - && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_MWAITX)) - opts->x_ix86_isa_flags |= OPTION_MASK_ISA_MWAITX; - if (processor_alias_table[i].flags & PTA_PKU + if (((processor_alias_table[i].flags & PTA_MWAITX) != 0) + && !(opts->x_ix86_isa_flags2_explicit & OPTION_MASK_ISA_MWAITX)) + opts->x_ix86_isa_flags2 |= OPTION_MASK_ISA_MWAITX; + if (((processor_alias_table[i].flags & PTA_PKU) != 0) && !(opts->x_ix86_isa_flags_explicit & OPTION_MASK_ISA_PKU)) opts->x_ix86_isa_flags |= OPTION_MASK_ISA_PKU; @@ -4113,7 +3891,7 @@ if (!(opts_set->x_ix86_target_flags & OPTION_MASK_GENERAL_REGS_ONLY) && !(opts_set->x_target_flags & MASK_80387)) { - if (processor_alias_table[i].flags & PTA_NO_80387) + if (((processor_alias_table[i].flags & PTA_NO_80387) != 0)) opts->x_target_flags &= ~MASK_80387; else opts->x_target_flags |= MASK_80387; @@ -4121,12 +3899,6 @@ break; } - if (TARGET_X32 && (opts->x_ix86_isa_flags & OPTION_MASK_ISA_MPX)) - error ("Intel MPX does not support x32"); - - if (TARGET_X32 && (ix86_isa_flags & OPTION_MASK_ISA_MPX)) - error ("Intel MPX does not support x32"); - if (i == pta_size) { error (main_args_p @@ -4139,9 +3911,14 @@ if (strcmp (processor_alias_table[i].name, "generic") && strcmp (processor_alias_table[i].name, "intel") && (!TARGET_64BIT_P (opts->x_ix86_isa_flags) - || (processor_alias_table[i].flags & PTA_64BIT))) + || ((processor_alias_table[i].flags & PTA_64BIT) != 0))) candidates.safe_push (processor_alias_table[i].name); +#ifdef HAVE_LOCAL_CPU_DETECT + /* Add also "native" as possible value. */ + candidates.safe_push ("native"); +#endif + char *s; const char *hint = candidates_list_and_hint (opts->x_ix86_arch_string, s, candidates); @@ -4161,7 +3938,7 @@ XDELETEVEC (s); } - ix86_arch_mask = 1u << ix86_arch; + ix86_arch_mask = HOST_WIDE_INT_1U << ix86_arch; for (i = 0; i < X86_ARCH_LAST; ++i) ix86_arch_features[i] = !!(initial_ix86_arch_features[i] & ix86_arch_mask); @@ -4172,7 +3949,7 @@ ix86_tune = processor_alias_table[i].processor; if (TARGET_64BIT_P (opts->x_ix86_isa_flags)) { - if (!(processor_alias_table[i].flags & PTA_64BIT)) + if (!((processor_alias_table[i].flags & PTA_64BIT) != 0)) { if (ix86_tune_defaulted) { @@ -4195,7 +3972,8 @@ However, the VIA C3 gives a SIGILL, so we only do that for i686 and higher processors. */ if (TARGET_CMOV - && (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE))) + && ((processor_alias_table[i].flags + & (PTA_PREFETCH_SSE | PTA_SSE)) != 0)) x86_prefetch_sse = true; break; } @@ -4210,9 +3988,14 @@ auto_vec <const char *> candidates; for (i = 0; i < pta_size; i++) if (!TARGET_64BIT_P (opts->x_ix86_isa_flags) - || (processor_alias_table[i].flags & PTA_64BIT)) + || ((processor_alias_table[i].flags & PTA_64BIT) != 0)) candidates.safe_push (processor_alias_table[i].name); +#ifdef HAVE_LOCAL_CPU_DETECT + /* Add also "native" as possible value. */ + candidates.safe_push ("native"); +#endif + char *s; const char *hint = candidates_list_and_hint (opts->x_ix86_tune_string, s, candidates); @@ -4278,7 +4061,7 @@ } } - ix86_tune_cost = processor_target_table[ix86_tune].cost; + ix86_tune_cost = processor_cost_table[ix86_tune]; /* TODO: ix86_cost should be chosen at instruction or function granuality so for cold code we use size_cost even in !optimize_size compilation. */ if (opts->x_optimize_size) @@ -4664,7 +4447,8 @@ if (TARGET_SEH && TARGET_CALL_MS2SYSV_XLOGUES) sorry ("-mcall-ms2sysv-xlogues isn%'t currently supported with SEH"); - if (!(opts_set->x_target_flags & MASK_VZEROUPPER)) + if (!(opts_set->x_target_flags & MASK_VZEROUPPER) + && TARGET_EMIT_VZEROUPPER) opts->x_target_flags |= MASK_VZEROUPPER; if (!(opts_set->x_target_flags & MASK_STV)) opts->x_target_flags |= MASK_STV; @@ -4682,11 +4466,18 @@ if (!ix86_tune_features[X86_TUNE_AVX256_UNALIGNED_STORE_OPTIMAL] && !(opts_set->x_target_flags & MASK_AVX256_SPLIT_UNALIGNED_STORE)) opts->x_target_flags |= MASK_AVX256_SPLIT_UNALIGNED_STORE; + /* Enable 128-bit AVX instruction generation for the auto-vectorizer. */ if (TARGET_AVX128_OPTIMAL - && !(opts_set->x_target_flags & MASK_PREFER_AVX128)) - opts->x_target_flags |= MASK_PREFER_AVX128; + && (opts_set->x_prefer_vector_width_type == PVW_NONE)) + opts->x_prefer_vector_width_type = PVW_AVX128; + + /* Use 256-bit AVX instruction generation + in the auto-vectorizer. */ + if (ix86_tune_features[X86_TUNE_AVX256_OPTIMAL] + && (opts_set->x_prefer_vector_width_type == PVW_NONE)) + opts->x_prefer_vector_width_type = PVW_AVX256; if (opts->x_ix86_recip_name) { @@ -4841,36 +4632,14 @@ target_option_default_node = target_option_current_node = build_target_option_node (opts); - /* Do not support control flow instrumentation if CET is not enabled. */ if (opts->x_flag_cf_protection != CF_NONE) - { - if (!(TARGET_IBT_P (opts->x_ix86_isa_flags2) - || TARGET_SHSTK_P (opts->x_ix86_isa_flags2))) - { - if (flag_cf_protection == CF_FULL) - { - error ("%<-fcf-protection=full%> requires CET support " - "on this target. Use -mcet or one of -mibt, " - "-mshstk options to enable CET"); - } - else if (flag_cf_protection == CF_BRANCH) - { - error ("%<-fcf-protection=branch%> requires CET support " - "on this target. Use -mcet or one of -mibt, " - "-mshstk options to enable CET"); - } - else if (flag_cf_protection == CF_RETURN) - { - error ("%<-fcf-protection=return%> requires CET support " - "on this target. Use -mcet or one of -mibt, " - "-mshstk options to enable CET"); - } - flag_cf_protection = CF_NONE; - return false; - } - opts->x_flag_cf_protection = - (cf_protection_level) (opts->x_flag_cf_protection | CF_SET); - } + opts->x_flag_cf_protection = + (cf_protection_level) (opts->x_flag_cf_protection | CF_SET); + + if (ix86_tune_features [X86_TUNE_AVOID_128FMA_CHAINS]) + maybe_set_param_value (PARAM_AVOID_FMA_MAX_BITS, 128, + opts->x_param_values, + opts_set->x_param_values); return true; } @@ -4965,11 +4734,6 @@ for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++) fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = ""; } - - /* If MPX is disabled, squash the registers. */ - if (! TARGET_MPX) - for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++) - fixed_regs[i] = call_used_regs[i] = 1, reg_names[i] = ""; } /* Canonicalize a comparison from one we don't have to one we do have. */ @@ -5054,7 +4818,7 @@ { enum processor_type old_tune = ix86_tune; enum processor_type old_arch = ix86_arch; - unsigned int ix86_arch_mask; + unsigned HOST_WIDE_INT ix86_arch_mask; int i; /* We don't change -fPIC. */ @@ -5094,7 +4858,7 @@ opts->x_ix86_tune_memset_strategy = ptr->x_ix86_tune_memset_strategy; opts->x_ix86_tune_no_default = ptr->x_ix86_tune_no_default; opts->x_ix86_veclibabi_type = ptr->x_ix86_veclibabi_type; - ix86_tune_cost = processor_target_table[ix86_tune].cost; + ix86_tune_cost = processor_cost_table[ix86_tune]; /* TODO: ix86_cost should be chosen at instruction or function granuality so for cold code we use size_cost even in !optimize_size compilation. */ if (opts->x_optimize_size) @@ -5105,7 +4869,7 @@ /* Recreate the arch feature tests if the arch changed */ if (old_arch != ix86_arch) { - ix86_arch_mask = 1u << ix86_arch; + ix86_arch_mask = HOST_WIDE_INT_1U << ix86_arch; for (i = 0; i < X86_ARCH_LAST; ++i) ix86_arch_features[i] = !!(initial_ix86_arch_features[i] & ix86_arch_mask); @@ -5181,12 +4945,12 @@ gcc_assert (ptr->arch < PROCESSOR_max); fprintf (file, "%*sarch = %d (%s)\n", indent, "", - ptr->arch, processor_target_table[ptr->arch].name); + ptr->arch, processor_names[ptr->arch]); gcc_assert (ptr->tune < PROCESSOR_max); fprintf (file, "%*stune = %d (%s)\n", indent, "", - ptr->tune, processor_target_table[ptr->tune].name); + ptr->tune, processor_names[ptr->tune]); fprintf (file, "%*sbranch_cost = %d\n", indent, "", ptr->branch_cost); @@ -5236,10 +5000,15 @@ int mask; } attrs[] = { /* isa options */ + IX86_ATTR_ISA ("pconfig", OPT_mpconfig), + IX86_ATTR_ISA ("wbnoinvd", OPT_mwbnoinvd), IX86_ATTR_ISA ("sgx", OPT_msgx), IX86_ATTR_ISA ("avx5124fmaps", OPT_mavx5124fmaps), IX86_ATTR_ISA ("avx5124vnniw", OPT_mavx5124vnniw), IX86_ATTR_ISA ("avx512vpopcntdq", OPT_mavx512vpopcntdq), + IX86_ATTR_ISA ("avx512vbmi2", OPT_mavx512vbmi2), + IX86_ATTR_ISA ("avx512vnni", OPT_mavx512vnni), + IX86_ATTR_ISA ("avx512bitalg", OPT_mavx512bitalg), IX86_ATTR_ISA ("avx512vbmi", OPT_mavx512vbmi), IX86_ATTR_ISA ("avx512ifma", OPT_mavx512ifma), @@ -5298,12 +5067,16 @@ IX86_ATTR_ISA ("lwp", OPT_mlwp), IX86_ATTR_ISA ("hle", OPT_mhle), IX86_ATTR_ISA ("fxsr", OPT_mfxsr), - IX86_ATTR_ISA ("mpx", OPT_mmpx), IX86_ATTR_ISA ("clwb", OPT_mclwb), IX86_ATTR_ISA ("rdpid", OPT_mrdpid), IX86_ATTR_ISA ("gfni", OPT_mgfni), - IX86_ATTR_ISA ("ibt", OPT_mibt), IX86_ATTR_ISA ("shstk", OPT_mshstk), + IX86_ATTR_ISA ("vaes", OPT_mvaes), + IX86_ATTR_ISA ("vpclmulqdq", OPT_mvpclmulqdq), + IX86_ATTR_ISA ("movdiri", OPT_mmovdiri), + IX86_ATTR_ISA ("movdir64b", OPT_mmovdir64b), + IX86_ATTR_ISA ("waitpkg", OPT_mwaitpkg), + IX86_ATTR_ISA ("cldemote", OPT_mcldemote), /* enum options */ IX86_ATTR_ENUM ("fpmath=", OPT_mfpmath_), @@ -5653,6 +5426,19 @@ { tree caller_tree = DECL_FUNCTION_SPECIFIC_TARGET (caller); tree callee_tree = DECL_FUNCTION_SPECIFIC_TARGET (callee); + + /* Changes of those flags can be tolerated for always inlines. Lets hope + user knows what he is doing. */ + const unsigned HOST_WIDE_INT always_inline_safe_mask + = (MASK_USE_8BIT_IDIV | MASK_ACCUMULATE_OUTGOING_ARGS + | MASK_NO_ALIGN_STRINGOPS | MASK_AVX256_SPLIT_UNALIGNED_LOAD + | MASK_AVX256_SPLIT_UNALIGNED_STORE | MASK_CLD + | MASK_NO_FANCY_MATH_387 | MASK_IEEE_FP | MASK_INLINE_ALL_STRINGOPS + | MASK_INLINE_STRINGOPS_DYNAMICALLY | MASK_RECIP | MASK_STACK_PROBE + | MASK_STV | MASK_TLS_DIRECT_SEG_REFS | MASK_VZEROUPPER + | MASK_NO_PUSH_ARGS | MASK_OMIT_LEAF_FRAME_POINTER); + + if (!callee_tree) callee_tree = target_option_default_node; if (!caller_tree) @@ -5663,7 +5449,12 @@ struct cl_target_option *caller_opts = TREE_TARGET_OPTION (caller_tree); struct cl_target_option *callee_opts = TREE_TARGET_OPTION (callee_tree); bool ret = false; - + bool always_inline = + (DECL_DISREGARD_INLINE_LIMITS (callee) + && lookup_attribute ("always_inline", + DECL_ATTRIBUTES (callee))); + + cgraph_node *callee_node = cgraph_node::get (callee); /* Callee's isa options should be a subset of the caller's, i.e. a SSE4 function can inline a SSE2 function but a SSE2 function can't inline a SSE4 function. */ @@ -5674,14 +5465,17 @@ ret = false; /* See if we have the same non-isa options. */ - else if (caller_opts->x_target_flags != callee_opts->x_target_flags) + else if ((!always_inline + && caller_opts->x_target_flags != callee_opts->x_target_flags) + || (caller_opts->x_target_flags & ~always_inline_safe_mask) + != (callee_opts->x_target_flags & ~always_inline_safe_mask)) ret = false; /* See if arch, tune, etc. are the same. */ else if (caller_opts->arch != callee_opts->arch) ret = false; - else if (caller_opts->tune != callee_opts->tune) + else if (!always_inline && caller_opts->tune != callee_opts->tune) ret = false; else if (caller_opts->x_ix86_fpmath != callee_opts->x_ix86_fpmath @@ -5690,11 +5484,12 @@ for multi-versioning call optimization, so beware of ipa_fn_summaries not available. */ && (! ipa_fn_summaries - || ipa_fn_summaries->get - (cgraph_node::get (callee))->fp_expressions)) + || ipa_fn_summaries->get (callee_node) == NULL + || ipa_fn_summaries->get (callee_node)->fp_expressions)) ret = false; - else if (caller_opts->branch_cost != callee_opts->branch_cost) + else if (!always_inline + && caller_opts->branch_cost != callee_opts->branch_cost) ret = false; else @@ -5764,6 +5559,88 @@ } } +/* Set the indirect_branch_type field from the function FNDECL. */ + +static void +ix86_set_indirect_branch_type (tree fndecl) +{ + if (cfun->machine->indirect_branch_type == indirect_branch_unset) + { + tree attr = lookup_attribute ("indirect_branch", + DECL_ATTRIBUTES (fndecl)); + if (attr != NULL) + { + tree args = TREE_VALUE (attr); + if (args == NULL) + gcc_unreachable (); + tree cst = TREE_VALUE (args); + if (strcmp (TREE_STRING_POINTER (cst), "keep") == 0) + cfun->machine->indirect_branch_type = indirect_branch_keep; + else if (strcmp (TREE_STRING_POINTER (cst), "thunk") == 0) + cfun->machine->indirect_branch_type = indirect_branch_thunk; + else if (strcmp (TREE_STRING_POINTER (cst), "thunk-inline") == 0) + cfun->machine->indirect_branch_type = indirect_branch_thunk_inline; + else if (strcmp (TREE_STRING_POINTER (cst), "thunk-extern") == 0) + cfun->machine->indirect_branch_type = indirect_branch_thunk_extern; + else + gcc_unreachable (); + } + else + cfun->machine->indirect_branch_type = ix86_indirect_branch; + + /* -mcmodel=large is not compatible with -mindirect-branch=thunk + nor -mindirect-branch=thunk-extern. */ + if ((ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC) + && ((cfun->machine->indirect_branch_type + == indirect_branch_thunk_extern) + || (cfun->machine->indirect_branch_type + == indirect_branch_thunk))) + error ("%<-mindirect-branch=%s%> and %<-mcmodel=large%> are not " + "compatible", + ((cfun->machine->indirect_branch_type + == indirect_branch_thunk_extern) + ? "thunk-extern" : "thunk")); + } + + if (cfun->machine->function_return_type == indirect_branch_unset) + { + tree attr = lookup_attribute ("function_return", + DECL_ATTRIBUTES (fndecl)); + if (attr != NULL) + { + tree args = TREE_VALUE (attr); + if (args == NULL) + gcc_unreachable (); + tree cst = TREE_VALUE (args); + if (strcmp (TREE_STRING_POINTER (cst), "keep") == 0) + cfun->machine->function_return_type = indirect_branch_keep; + else if (strcmp (TREE_STRING_POINTER (cst), "thunk") == 0) + cfun->machine->function_return_type = indirect_branch_thunk; + else if (strcmp (TREE_STRING_POINTER (cst), "thunk-inline") == 0) + cfun->machine->function_return_type = indirect_branch_thunk_inline; + else if (strcmp (TREE_STRING_POINTER (cst), "thunk-extern") == 0) + cfun->machine->function_return_type = indirect_branch_thunk_extern; + else + gcc_unreachable (); + } + else + cfun->machine->function_return_type = ix86_function_return; + + /* -mcmodel=large is not compatible with -mfunction-return=thunk + nor -mfunction-return=thunk-extern. */ + if ((ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC) + && ((cfun->machine->function_return_type + == indirect_branch_thunk_extern) + || (cfun->machine->function_return_type + == indirect_branch_thunk))) + error ("%<-mfunction-return=%s%> and %<-mcmodel=large%> are not " + "compatible", + ((cfun->machine->function_return_type + == indirect_branch_thunk_extern) + ? "thunk-extern" : "thunk")); + } +} + /* Establish appropriate back-end context for processing the function FNDECL. The argument might be NULL to indicate processing at top level, outside of any function scope. */ @@ -5779,7 +5656,10 @@ one is extern inline and one isn't. Call ix86_set_func_type to set the func_type field. */ if (fndecl != NULL_TREE) - ix86_set_func_type (fndecl); + { + ix86_set_func_type (fndecl); + ix86_set_indirect_branch_type (fndecl); + } return; } @@ -5799,6 +5679,7 @@ } ix86_set_func_type (fndecl); + ix86_set_indirect_branch_type (fndecl); tree new_tree = DECL_FUNCTION_SPECIFIC_TARGET (fndecl); if (new_tree == NULL_TREE) @@ -5834,12 +5715,10 @@ if (cfun->machine->func_type != TYPE_NORMAL || cfun->machine->no_caller_saved_registers) { - /* Don't allow MPX, SSE, MMX nor x87 instructions since they + /* Don't allow SSE, MMX nor x87 instructions since they may change processor state. */ const char *isa; - if (TARGET_MPX) - isa = "MPX"; - else if (TARGET_SSE) + if (TARGET_SSE) isa = "SSE"; else if (TARGET_MMX) isa = "MMX/3Dnow"; @@ -6222,7 +6101,8 @@ function via GOT slot are indirect. */ if (!decl || (bind_global && flag_pic && !flag_plt) - || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && DECL_DLLIMPORT_P (decl))) + || (TARGET_DLLIMPORT_DECL_ATTRIBUTES && DECL_DLLIMPORT_P (decl)) + || flag_force_indirect_call) { /* Check if regparm >= 3 since arg_reg_available is set to false if regparm == 0. If regparm is 1 or 2, there is @@ -6231,7 +6111,7 @@ ??? The symbol indirect call doesn't need a call-clobbered register. But we don't know if this is a symbol indirect call or not here. */ - if (ix86_function_regparm (type, NULL) >= 3 + if (ix86_function_regparm (type, decl) >= 3 && !cfun->machine->arg_reg_available) return false; } @@ -6246,10 +6126,8 @@ arguments as in struct attribute_spec.handler. */ static tree -ix86_handle_cconv_attribute (tree *node, tree name, - tree args, - int, - bool *no_add_attrs) +ix86_handle_cconv_attribute (tree *node, tree name, tree args, int, + bool *no_add_attrs) { if (TREE_CODE (*node) != FUNCTION_TYPE && TREE_CODE (*node) != METHOD_TYPE @@ -6709,8 +6587,8 @@ The attribute stdcall is equivalent to RTD on a per module basis. */ -static int -ix86_return_pops_args (tree fundecl, tree funtype, int size) +static poly_int64 +ix86_return_pops_args (tree fundecl, tree funtype, poly_int64 size) { unsigned int ccvt; @@ -6834,9 +6712,6 @@ enum calling_abi call_abi; const int *parm_regs; - if (TARGET_MPX && BND_REGNO_P (regno)) - return true; - if (!TARGET_64BIT) { if (TARGET_MACHO) @@ -7181,11 +7056,28 @@ ? (!prototype_p (fntype) || stdarg_p (fntype)) : !libname); - cum->bnd_regno = FIRST_BND_REG; - cum->bnds_in_bt = 0; - cum->force_bnd_pass = 0; cum->decl = fndecl; + cum->warn_empty = !warn_abi || cum->stdarg; + if (!cum->warn_empty && fntype) + { + function_args_iterator iter; + tree argtype; + bool seen_empty_type = false; + FOREACH_FUNCTION_ARGS (fntype, argtype, iter) + { + if (argtype == error_mark_node || VOID_TYPE_P (argtype)) + break; + if (TYPE_EMPTY_P (argtype)) + seen_empty_type = true; + else if (seen_empty_type) + { + cum->warn_empty = true; + break; + } + } + } + if (!TARGET_64BIT) { /* If there are variable arguments, then we won't pass anything @@ -7977,7 +7869,7 @@ case X86_64_SSEDF_CLASS: if (mode != BLKmode) return gen_reg_or_parallel (mode, orig_mode, - SSE_REGNO (sse_regno)); + GET_SSE_REGNO (sse_regno)); break; case X86_64_X87_CLASS: case X86_64_COMPLEX_X87_CLASS: @@ -7993,7 +7885,7 @@ && regclass[1] == X86_64_SSEUP_CLASS && mode != BLKmode) return gen_reg_or_parallel (mode, orig_mode, - SSE_REGNO (sse_regno)); + GET_SSE_REGNO (sse_regno)); if (n == 4 && regclass[0] == X86_64_SSE_CLASS && regclass[1] == X86_64_SSEUP_CLASS @@ -8001,7 +7893,7 @@ && regclass[3] == X86_64_SSEUP_CLASS && mode != BLKmode) return gen_reg_or_parallel (mode, orig_mode, - SSE_REGNO (sse_regno)); + GET_SSE_REGNO (sse_regno)); if (n == 8 && regclass[0] == X86_64_SSE_CLASS && regclass[1] == X86_64_SSEUP_CLASS @@ -8013,7 +7905,7 @@ && regclass[7] == X86_64_SSEUP_CLASS && mode != BLKmode) return gen_reg_or_parallel (mode, orig_mode, - SSE_REGNO (sse_regno)); + GET_SSE_REGNO (sse_regno)); if (n == 2 && regclass[0] == X86_64_X87_CLASS && regclass[1] == X86_64_X87UP_CLASS) @@ -8022,9 +7914,22 @@ if (n == 2 && regclass[0] == X86_64_INTEGER_CLASS && regclass[1] == X86_64_INTEGER_CLASS - && (mode == CDImode || mode == TImode) + && (mode == CDImode || mode == TImode || mode == BLKmode) && intreg[0] + 1 == intreg[1]) - return gen_rtx_REG (mode, intreg[0]); + { + if (mode == BLKmode) + { + /* Use TImode for BLKmode values in 2 integer registers. */ + exp[0] = gen_rtx_EXPR_LIST (VOIDmode, + gen_rtx_REG (TImode, intreg[0]), + GEN_INT (0)); + ret = gen_rtx_PARALLEL (mode, rtvec_alloc (1)); + XVECEXP (ret, 0, 0) = exp[0]; + return ret; + } + else + return gen_rtx_REG (mode, intreg[0]); + } /* Otherwise figure out the entries of the PARALLEL. */ for (i = 0; i < n; i++) @@ -8060,7 +7965,7 @@ exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode, gen_rtx_REG (SFmode, - SSE_REGNO (sse_regno)), + GET_SSE_REGNO (sse_regno)), GEN_INT (i*8)); sse_regno++; break; @@ -8068,7 +7973,7 @@ exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode, gen_rtx_REG (DFmode, - SSE_REGNO (sse_regno)), + GET_SSE_REGNO (sse_regno)), GEN_INT (i*8)); sse_regno++; break; @@ -8114,7 +8019,7 @@ exp [nexps++] = gen_rtx_EXPR_LIST (VOIDmode, gen_rtx_REG (tmpmode, - SSE_REGNO (sse_regno)), + GET_SSE_REGNO (sse_regno)), GEN_INT (pos*8)); sse_regno++; break; @@ -8336,36 +8241,6 @@ if (type) mode = type_natural_mode (type, NULL, false); - if ((type && POINTER_BOUNDS_TYPE_P (type)) - || POINTER_BOUNDS_MODE_P (mode)) - { - /* If we pass bounds in BT then just update remained bounds count. */ - if (cum->bnds_in_bt) - { - cum->bnds_in_bt--; - return; - } - - /* Update remained number of bounds to force. */ - if (cum->force_bnd_pass) - cum->force_bnd_pass--; - - cum->bnd_regno++; - - return; - } - - /* The first arg not going to Bounds Tables resets this counter. */ - cum->bnds_in_bt = 0; - /* For unnamed args we always pass bounds to avoid bounds mess when - passed and received types do not match. If bounds do not follow - unnamed arg, still pretend required number of bounds were passed. */ - if (cum->force_bnd_pass) - { - cum->bnd_regno += cum->force_bnd_pass; - cum->force_bnd_pass = 0; - } - if (TARGET_64BIT) { enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi; @@ -8378,10 +8253,6 @@ else nregs = function_arg_advance_32 (cum, mode, type, bytes, words); - /* For stdarg we expect bounds to be passed for each value passed - in register. */ - if (cum->stdarg) - cum->force_bnd_pass = nregs; /* For pointers passed in memory we expect bounds passed in Bounds Table. */ if (!nregs) @@ -8389,8 +8260,6 @@ /* Track if there are outgoing arguments on stack. */ if (cum->caller) cfun->machine->outgoing_args_on_stack = true; - - cum->bnds_in_bt = chkp_type_bounds_count (type); } } @@ -8678,23 +8547,6 @@ return arg; } - /* All pointer bounds arguments are handled separately here. */ - if ((type && POINTER_BOUNDS_TYPE_P (type)) - || POINTER_BOUNDS_MODE_P (mode)) - { - /* Return NULL if bounds are forced to go in Bounds Table. */ - if (cum->bnds_in_bt) - arg = NULL; - /* Return the next available bound reg if any. */ - else if (cum->bnd_regno <= LAST_BND_REG) - arg = gen_rtx_REG (BNDmode, cum->bnd_regno); - /* Return the next special slot number otherwise. */ - else - arg = GEN_INT (cum->bnd_regno - LAST_BND_REG - 1); - - return arg; - } - if (mode == BLKmode) bytes = int_size_in_bytes (type); else @@ -8737,11 +8589,6 @@ { CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v); - /* Bounds are never passed by reference. */ - if ((type && POINTER_BOUNDS_TYPE_P (type)) - || POINTER_BOUNDS_MODE_P (mode)) - return false; - if (TARGET_64BIT) { enum calling_abi call_abi = cum ? cum->call_abi : ix86_abi; @@ -8927,6 +8774,8 @@ the main variant type. */ type = TYPE_MAIN_VARIANT (type); align = TYPE_ALIGN (type); + if (TYPE_EMPTY_P (type)) + return PARM_BOUNDARY; } else align = GET_MODE_ALIGNMENT (mode); @@ -8983,10 +8832,6 @@ case SI_REG: return TARGET_64BIT && ix86_cfun_abi () != MS_ABI; - case BND0_REG: - case BND1_REG: - return chkp_function_instrumented_p (current_function_decl); - /* Complex values are returned in %st(0)/%st(1) pair. */ case ST0_REG: case ST1_REG: @@ -9170,10 +9015,7 @@ fn = fntype_or_decl; fntype = fn ? TREE_TYPE (fn) : fntype_or_decl; - if ((valtype && POINTER_BOUNDS_TYPE_P (valtype)) - || POINTER_BOUNDS_MODE_P (mode)) - return gen_rtx_REG (BNDmode, FIRST_BND_REG); - else if (TARGET_64BIT && ix86_function_type_abi (fntype) == MS_ABI) + if (TARGET_64BIT && ix86_function_type_abi (fntype) == MS_ABI) return function_value_ms_64 (orig_mode, mode, valtype); else if (TARGET_64BIT) return function_value_64 (orig_mode, mode, valtype); @@ -9191,57 +9033,6 @@ return ix86_function_value_1 (valtype, fntype_or_decl, orig_mode, mode); } -/* Return an RTX representing a place where a function returns - or recieves pointer bounds or NULL if no bounds are returned. - - VALTYPE is a data type of a value returned by the function. - - FN_DECL_OR_TYPE is a tree node representing FUNCTION_DECL - or FUNCTION_TYPE of the function. - - If OUTGOING is false, return a place in which the caller will - see the return value. Otherwise, return a place where a - function returns a value. */ - -static rtx -ix86_function_value_bounds (const_tree valtype, - const_tree fntype_or_decl ATTRIBUTE_UNUSED, - bool outgoing ATTRIBUTE_UNUSED) -{ - rtx res = NULL_RTX; - - if (BOUNDED_TYPE_P (valtype)) - res = gen_rtx_REG (BNDmode, FIRST_BND_REG); - else if (chkp_type_has_pointer (valtype)) - { - bitmap slots; - rtx bounds[2]; - bitmap_iterator bi; - unsigned i, bnd_no = 0; - - bitmap_obstack_initialize (NULL); - slots = BITMAP_ALLOC (NULL); - chkp_find_bound_slots (valtype, slots); - - EXECUTE_IF_SET_IN_BITMAP (slots, 0, i, bi) - { - rtx reg = gen_rtx_REG (BNDmode, FIRST_BND_REG + bnd_no); - rtx offs = GEN_INT (i * POINTER_SIZE / BITS_PER_UNIT); - gcc_assert (bnd_no < 2); - bounds[bnd_no++] = gen_rtx_EXPR_LIST (VOIDmode, reg, offs); - } - - res = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (bnd_no, bounds)); - - BITMAP_FREE (slots); - bitmap_obstack_release (NULL); - } - else - res = NULL_RTX; - - return res; -} - /* Pointer function arguments and return values are promoted to word_mode for normal functions. */ @@ -9290,9 +9081,6 @@ const machine_mode mode = type_natural_mode (type, NULL, true); HOST_WIDE_INT size; - if (POINTER_BOUNDS_TYPE_P (type)) - return false; - if (TARGET_64BIT) { if (ix86_function_type_abi (fntype) == MS_ABI) @@ -9547,7 +9335,7 @@ set_mem_alias_set (mem, set); set_mem_align (mem, GET_MODE_ALIGNMENT (smode)); - emit_move_insn (mem, gen_rtx_REG (smode, SSE_REGNO (i))); + emit_move_insn (mem, gen_rtx_REG (smode, GET_SSE_REGNO (i))); } emit_label (label); @@ -9620,8 +9408,7 @@ CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v); CUMULATIVE_ARGS next_cum; tree fntype; - rtx save_area; - int bnd_reg, i, max; + int max; gcc_assert (!no_rtl); @@ -9637,40 +9424,10 @@ if (stdarg_p (fntype)) ix86_function_arg_advance (pack_cumulative_args (&next_cum), mode, type, true); - save_area = frame_pointer_rtx; max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD; if (max > X86_64_REGPARM_MAX) max = X86_64_REGPARM_MAX; - - bnd_reg = cum->bnd_regno + cum->force_bnd_pass; - if (chkp_function_instrumented_p (current_function_decl)) - for (i = cum->regno; i < max; i++) - { - rtx addr = plus_constant (Pmode, save_area, i * UNITS_PER_WORD); - rtx ptr = gen_rtx_REG (Pmode, - x86_64_int_parameter_registers[i]); - rtx bounds; - - if (bnd_reg <= LAST_BND_REG) - bounds = gen_rtx_REG (BNDmode, bnd_reg); - else - { - rtx ldx_addr = - plus_constant (Pmode, arg_pointer_rtx, - (LAST_BND_REG - bnd_reg) * GET_MODE_SIZE (Pmode)); - bounds = gen_reg_rtx (BNDmode); - emit_insn (BNDmode == BND64mode - ? gen_bnd64_ldx (bounds, ldx_addr, ptr) - : gen_bnd32_ldx (bounds, ldx_addr, ptr)); - } - - emit_insn (BNDmode == BND64mode - ? gen_bnd64_stx (addr, ptr, bounds) - : gen_bnd32_stx (addr, ptr, bounds)); - - bnd_reg++; - } } @@ -9748,13 +9505,6 @@ crtl->args.arg_offset_rtx, NULL_RTX, 0, OPTAB_LIB_WIDEN); convert_move (va_r, next, 0); - - /* Store zero bounds for va_list. */ - if (chkp_function_instrumented_p (current_function_decl)) - chkp_expand_bounds_reset_for_mem (valist, - make_tree (TREE_TYPE (valist), - next)); - } return; } @@ -9809,10 +9559,6 @@ if (words != 0) t = fold_build_pointer_plus_hwi (t, words * UNITS_PER_WORD); - /* Store zero bounds for overflow area pointer. */ - if (chkp_function_instrumented_p (current_function_decl)) - chkp_expand_bounds_reset_for_mem (ovf, t); - t = build2 (MODIFY_EXPR, type, ovf, t); TREE_SIDE_EFFECTS (t) = 1; expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL); @@ -9826,10 +9572,6 @@ if (!ix86_varargs_gpr_size) t = fold_build_pointer_plus_hwi (t, -8 * X86_64_REGPARM_MAX); - /* Store zero bounds for save area pointer. */ - if (chkp_function_instrumented_p (current_function_decl)) - chkp_expand_bounds_reset_for_mem (sav, t); - t = build2 (MODIFY_EXPR, type, sav, t); TREE_SIDE_EFFECTS (t) = 1; expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL); @@ -9873,7 +9615,7 @@ indirect_p = pass_by_reference (NULL, TYPE_MODE (type), type, false); if (indirect_p) type = build_pointer_type (type); - size = int_size_in_bytes (type); + size = arg_int_size_in_bytes (type); rsize = CEIL (size, UNITS_PER_WORD); nat_mode = type_natural_mode (type, NULL, false); @@ -10344,12 +10086,13 @@ } /* Return the opcode of the special instruction to be used to load - the constant X. */ + the constant operands[1] into operands[0]. */ const char * -standard_sse_constant_opcode (rtx_insn *insn, rtx x) +standard_sse_constant_opcode (rtx_insn *insn, rtx *operands) { machine_mode mode; + rtx x = operands[1]; gcc_assert (TARGET_SSE); @@ -10359,34 +10102,51 @@ { switch (get_attr_mode (insn)) { + case MODE_TI: + if (!EXT_REX_SSE_REG_P (operands[0])) + return "%vpxor\t%0, %d0"; + /* FALLTHRU */ case MODE_XI: - return "vpxord\t%g0, %g0, %g0"; case MODE_OI: - return (TARGET_AVX512VL - ? "vpxord\t%x0, %x0, %x0" - : "vpxor\t%x0, %x0, %x0"); - case MODE_TI: - return (TARGET_AVX512VL - ? "vpxord\t%t0, %t0, %t0" - : "%vpxor\t%0, %d0"); - + if (EXT_REX_SSE_REG_P (operands[0])) + return (TARGET_AVX512VL + ? "vpxord\t%x0, %x0, %x0" + : "vpxord\t%g0, %g0, %g0"); + return "vpxor\t%x0, %x0, %x0"; + + case MODE_V2DF: + if (!EXT_REX_SSE_REG_P (operands[0])) + return "%vxorpd\t%0, %d0"; + /* FALLTHRU */ case MODE_V8DF: - return (TARGET_AVX512DQ - ? "vxorpd\t%g0, %g0, %g0" - : "vpxorq\t%g0, %g0, %g0"); case MODE_V4DF: - return "vxorpd\t%x0, %x0, %x0"; - case MODE_V2DF: - return "%vxorpd\t%0, %d0"; - + if (!EXT_REX_SSE_REG_P (operands[0])) + return "vxorpd\t%x0, %x0, %x0"; + else if (TARGET_AVX512DQ) + return (TARGET_AVX512VL + ? "vxorpd\t%x0, %x0, %x0" + : "vxorpd\t%g0, %g0, %g0"); + else + return (TARGET_AVX512VL + ? "vpxorq\t%x0, %x0, %x0" + : "vpxorq\t%g0, %g0, %g0"); + + case MODE_V4SF: + if (!EXT_REX_SSE_REG_P (operands[0])) + return "%vxorps\t%0, %d0"; + /* FALLTHRU */ case MODE_V16SF: - return (TARGET_AVX512DQ - ? "vxorps\t%g0, %g0, %g0" - : "vpxord\t%g0, %g0, %g0"); case MODE_V8SF: - return "vxorps\t%x0, %x0, %x0"; - case MODE_V4SF: - return "%vxorps\t%0, %d0"; + if (!EXT_REX_SSE_REG_P (operands[0])) + return "vxorps\t%x0, %x0, %x0"; + else if (TARGET_AVX512DQ) + return (TARGET_AVX512VL + ? "vxorps\t%x0, %x0, %x0" + : "vxorps\t%g0, %g0, %g0"); + else + return (TARGET_AVX512VL + ? "vpxord\t%x0, %x0, %x0" + : "vpxord\t%g0, %g0, %g0"); default: gcc_unreachable (); @@ -10413,9 +10173,14 @@ case MODE_V2DF: case MODE_V4SF: gcc_assert (TARGET_SSE2); - return (TARGET_AVX - ? "vpcmpeqd\t%0, %0, %0" - : "pcmpeqd\t%0, %0"); + if (!EXT_REX_SSE_REG_P (operands[0])) + return (TARGET_AVX + ? "vpcmpeqd\t%0, %0, %0" + : "pcmpeqd\t%0, %0"); + else if (TARGET_AVX512VL) + return "vpternlogd\t{$0xFF, %0, %0, %0|%0, %0, %0, 0xFF}"; + else + return "vpternlogd\t{$0xFF, %g0, %g0, %g0|%g0, %g0, %g0, 0xFF}"; default: gcc_unreachable (); @@ -10484,8 +10249,6 @@ bool ix86_can_use_return_insn_p (void) { - struct ix86_frame frame; - if (ix86_function_naked (current_function_decl)) return false; @@ -10500,7 +10263,7 @@ if (crtl->args.pops_args && crtl->args.size >= 32768) return 0; - frame = cfun->machine->frame; + struct ix86_frame &frame = cfun->machine->frame; return (frame.stack_pointer_offset == UNITS_PER_WORD && (frame.nregs + frame.nsseregs) == 0); } @@ -10565,6 +10328,274 @@ # endif #endif +/* Label count for call and return thunks. It is used to make unique + labels in call and return thunks. */ +static int indirectlabelno; + +/* True if call thunk function is needed. */ +static bool indirect_thunk_needed = false; + +/* Bit masks of integer registers, which contain branch target, used + by call thunk functions. */ +static int indirect_thunks_used; + +/* True if return thunk function is needed. */ +static bool indirect_return_needed = false; + +/* True if return thunk function via CX is needed. */ +static bool indirect_return_via_cx; + +#ifndef INDIRECT_LABEL +# define INDIRECT_LABEL "LIND" +#endif + +/* Indicate what prefix is needed for an indirect branch. */ +enum indirect_thunk_prefix +{ + indirect_thunk_prefix_none, + indirect_thunk_prefix_nt +}; + +/* Return the prefix needed for an indirect branch INSN. */ + +enum indirect_thunk_prefix +indirect_thunk_need_prefix (rtx_insn *insn) +{ + enum indirect_thunk_prefix need_prefix; + if ((cfun->machine->indirect_branch_type + == indirect_branch_thunk_extern) + && ix86_notrack_prefixed_insn_p (insn)) + { + /* NOTRACK prefix is only used with external thunk so that it + can be properly updated to support CET at run-time. */ + need_prefix = indirect_thunk_prefix_nt; + } + else + need_prefix = indirect_thunk_prefix_none; + return need_prefix; +} + +/* Fills in the label name that should be used for the indirect thunk. */ + +static void +indirect_thunk_name (char name[32], unsigned int regno, + enum indirect_thunk_prefix need_prefix, + bool ret_p) +{ + if (regno != INVALID_REGNUM && regno != CX_REG && ret_p) + gcc_unreachable (); + + if (USE_HIDDEN_LINKONCE) + { + const char *prefix; + + if (need_prefix == indirect_thunk_prefix_nt + && regno != INVALID_REGNUM) + { + /* NOTRACK prefix is only used with external thunk via + register so that NOTRACK prefix can be added to indirect + branch via register to support CET at run-time. */ + prefix = "_nt"; + } + else + prefix = ""; + + const char *ret = ret_p ? "return" : "indirect"; + + if (regno != INVALID_REGNUM) + { + const char *reg_prefix; + if (LEGACY_INT_REGNO_P (regno)) + reg_prefix = TARGET_64BIT ? "r" : "e"; + else + reg_prefix = ""; + sprintf (name, "__x86_%s_thunk%s_%s%s", + ret, prefix, reg_prefix, reg_names[regno]); + } + else + sprintf (name, "__x86_%s_thunk%s", ret, prefix); + } + else + { + if (regno != INVALID_REGNUM) + ASM_GENERATE_INTERNAL_LABEL (name, "LITR", regno); + else + { + if (ret_p) + ASM_GENERATE_INTERNAL_LABEL (name, "LRT", 0); + else + ASM_GENERATE_INTERNAL_LABEL (name, "LIT", 0); + } + } +} + +/* Output a call and return thunk for indirect branch. If REGNO != -1, + the function address is in REGNO and the call and return thunk looks like: + + call L2 + L1: + pause + lfence + jmp L1 + L2: + mov %REG, (%sp) + ret + + Otherwise, the function address is on the top of stack and the + call and return thunk looks like: + + call L2 + L1: + pause + lfence + jmp L1 + L2: + lea WORD_SIZE(%sp), %sp + ret + */ + +static void +output_indirect_thunk (unsigned int regno) +{ + char indirectlabel1[32]; + char indirectlabel2[32]; + + ASM_GENERATE_INTERNAL_LABEL (indirectlabel1, INDIRECT_LABEL, + indirectlabelno++); + ASM_GENERATE_INTERNAL_LABEL (indirectlabel2, INDIRECT_LABEL, + indirectlabelno++); + + /* Call */ + fputs ("\tcall\t", asm_out_file); + assemble_name_raw (asm_out_file, indirectlabel2); + fputc ('\n', asm_out_file); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1); + + /* AMD and Intel CPUs prefer each a different instruction as loop filler. + Usage of both pause + lfence is compromise solution. */ + fprintf (asm_out_file, "\tpause\n\tlfence\n"); + + /* Jump. */ + fputs ("\tjmp\t", asm_out_file); + assemble_name_raw (asm_out_file, indirectlabel1); + fputc ('\n', asm_out_file); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2); + + /* The above call insn pushed a word to stack. Adjust CFI info. */ + if (flag_asynchronous_unwind_tables && dwarf2out_do_frame ()) + { + if (! dwarf2out_do_cfi_asm ()) + { + dw_cfi_ref xcfi = ggc_cleared_alloc<dw_cfi_node> (); + xcfi->dw_cfi_opc = DW_CFA_advance_loc4; + xcfi->dw_cfi_oprnd1.dw_cfi_addr = ggc_strdup (indirectlabel2); + vec_safe_push (cfun->fde->dw_fde_cfi, xcfi); + } + dw_cfi_ref xcfi = ggc_cleared_alloc<dw_cfi_node> (); + xcfi->dw_cfi_opc = DW_CFA_def_cfa_offset; + xcfi->dw_cfi_oprnd1.dw_cfi_offset = 2 * UNITS_PER_WORD; + vec_safe_push (cfun->fde->dw_fde_cfi, xcfi); + dwarf2out_emit_cfi (xcfi); + } + + if (regno != INVALID_REGNUM) + { + /* MOV. */ + rtx xops[2]; + xops[0] = gen_rtx_MEM (word_mode, stack_pointer_rtx); + xops[1] = gen_rtx_REG (word_mode, regno); + output_asm_insn ("mov\t{%1, %0|%0, %1}", xops); + } + else + { + /* LEA. */ + rtx xops[2]; + xops[0] = stack_pointer_rtx; + xops[1] = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD); + output_asm_insn ("lea\t{%E1, %0|%0, %E1}", xops); + } + + fputs ("\tret\n", asm_out_file); +} + +/* Output a funtion with a call and return thunk for indirect branch. + If REGNO != INVALID_REGNUM, the function address is in REGNO. + Otherwise, the function address is on the top of stack. Thunk is + used for function return if RET_P is true. */ + +static void +output_indirect_thunk_function (enum indirect_thunk_prefix need_prefix, + unsigned int regno, bool ret_p) +{ + char name[32]; + tree decl; + + /* Create __x86_indirect_thunk. */ + indirect_thunk_name (name, regno, need_prefix, ret_p); + decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL, + get_identifier (name), + build_function_type_list (void_type_node, NULL_TREE)); + DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL, + NULL_TREE, void_type_node); + TREE_PUBLIC (decl) = 1; + TREE_STATIC (decl) = 1; + DECL_IGNORED_P (decl) = 1; + +#if TARGET_MACHO + if (TARGET_MACHO) + { + switch_to_section (darwin_sections[picbase_thunk_section]); + fputs ("\t.weak_definition\t", asm_out_file); + assemble_name (asm_out_file, name); + fputs ("\n\t.private_extern\t", asm_out_file); + assemble_name (asm_out_file, name); + putc ('\n', asm_out_file); + ASM_OUTPUT_LABEL (asm_out_file, name); + DECL_WEAK (decl) = 1; + } + else +#endif + if (USE_HIDDEN_LINKONCE) + { + cgraph_node::create (decl)->set_comdat_group (DECL_ASSEMBLER_NAME (decl)); + + targetm.asm_out.unique_section (decl, 0); + switch_to_section (get_named_section (decl, NULL, 0)); + + targetm.asm_out.globalize_label (asm_out_file, name); + fputs ("\t.hidden\t", asm_out_file); + assemble_name (asm_out_file, name); + putc ('\n', asm_out_file); + ASM_DECLARE_FUNCTION_NAME (asm_out_file, name, decl); + } + else + { + switch_to_section (text_section); + ASM_OUTPUT_LABEL (asm_out_file, name); + } + + DECL_INITIAL (decl) = make_node (BLOCK); + current_function_decl = decl; + allocate_struct_function (decl, false); + init_function_start (decl); + /* We're about to hide the function body from callees of final_* by + emitting it directly; tell them we're a thunk, if they care. */ + cfun->is_thunk = true; + first_function_block_is_cold = false; + /* Make sure unwind info is emitted for the thunk if needed. */ + final_start_function (emit_barrier (), asm_out_file, 1); + + output_indirect_thunk (regno); + + final_end_function (); + init_insn_lengths (); + free_after_compilation (cfun); + set_cfun (NULL); + current_function_decl = NULL; +} + static int pic_labels_used; /* Fills in the label name that should be used for a pc thunk for @@ -10589,13 +10620,35 @@ ix86_code_end (void) { rtx xops[2]; - int regno; + unsigned int regno; + + if (indirect_return_needed) + output_indirect_thunk_function (indirect_thunk_prefix_none, + INVALID_REGNUM, true); + if (indirect_return_via_cx) + output_indirect_thunk_function (indirect_thunk_prefix_none, + CX_REG, true); + if (indirect_thunk_needed) + output_indirect_thunk_function (indirect_thunk_prefix_none, + INVALID_REGNUM, false); + + for (regno = FIRST_REX_INT_REG; regno <= LAST_REX_INT_REG; regno++) + { + unsigned int i = regno - FIRST_REX_INT_REG + LAST_INT_REG + 1; + if ((indirect_thunks_used & (1 << i))) + output_indirect_thunk_function (indirect_thunk_prefix_none, + regno, false); + } for (regno = FIRST_INT_REG; regno <= LAST_INT_REG; regno++) { char name[32]; tree decl; + if ((indirect_thunks_used & (1 << regno))) + output_indirect_thunk_function (indirect_thunk_prefix_none, + regno, false); + if (!(pic_labels_used & (1 << regno))) continue; @@ -10841,15 +10894,6 @@ && df_regs_ever_live_p (regno))); } -/* Return true if register class CL should be an additional allocno - class. */ - -static bool -ix86_additional_allocno_class_p (reg_class_t cl) -{ - return cl == MOD4_SSE_REGS; -} - /* Return TRUE if we need to save REGNO. */ static bool @@ -10870,16 +10914,6 @@ while (nregs-- > 0) if ((i + nregs) == regno) return false; - - reg = crtl->return_bnd; - if (reg) - { - i = REGNO (reg); - nregs = REG_NREGS (reg); - while (nregs-- > 0) - if ((i + nregs) == regno) - return false; - } } return (df_regs_ever_live_p (regno) @@ -10994,7 +11028,7 @@ HOST_WIDE_INT ix86_initial_elimination_offset (int from, int to) { - struct ix86_frame frame = cfun->machine->frame; + struct ix86_frame &frame = cfun->machine->frame; if (from == ARG_POINTER_REGNUM && to == HARD_FRAME_POINTER_REGNUM) return frame.hard_frame_pointer_offset; @@ -11036,6 +11070,18 @@ } } +/* Return the probing interval for -fstack-clash-protection. */ + +static HOST_WIDE_INT +get_probe_interval (void) +{ + if (flag_stack_clash_protection) + return (HOST_WIDE_INT_1U + << PARAM_VALUE (PARAM_STACK_CLASH_PROTECTION_PROBE_INTERVAL)); + else + return (HOST_WIDE_INT_1U << STACK_CHECK_PROBE_INTERVAL_EXP); +} + /* When using -fsplit-stack, the allocation routines set a field in the TCB to the bottom of the stack plus this much space, measured in bytes. */ @@ -11185,7 +11231,11 @@ /* Calculate the size of the va-arg area (not including padding, if any). */ frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size; - if (stack_realign_fp) + /* Also adjust stack_realign_offset for the largest alignment of + stack slot actually used. */ + if (stack_realign_fp + || (cfun->machine->max_used_stack_alignment != 0 + && (offset % cfun->machine->max_used_stack_alignment) != 0)) { /* We may need a 16-byte aligned stack for the remainder of the register save area, but the stack frame for the local function @@ -11261,11 +11311,16 @@ offset += frame->va_arg_size; } - /* Align start of frame for local function. */ + /* Align start of frame for local function. When a function call + is removed, it may become a leaf function. But if argument may + be passed on stack, we need to align the stack when there is no + tail call. */ if (m->call_ms2sysv || frame->va_arg_size != 0 || size != 0 || !crtl->is_leaf + || (!crtl->tail_call_emit + && cfun->machine->outgoing_args_on_stack) || cfun->calls_alloca || ix86_current_function_calls_tls_descriptor) offset = ROUND_UP (offset, stack_alignment_needed); @@ -11303,7 +11358,14 @@ to_allocate = offset - frame->sse_reg_save_offset; if ((!to_allocate && frame->nregs <= 1) - || (TARGET_64BIT && to_allocate >= HOST_WIDE_INT_C (0x80000000))) + || (TARGET_64BIT && to_allocate >= HOST_WIDE_INT_C (0x80000000)) + /* If stack clash probing needs a loop, then it needs a + scratch register. But the returned register is only guaranteed + to be safe to use after register saves are complete. So if + stack clash protections are enabled and the allocated frame is + larger than the probe interval, then use pushes to save + callee saved registers. */ + || (flag_stack_clash_protection && to_allocate > get_probe_interval ())) frame->save_regs_using_mov = false; if (ix86_using_red_zone () @@ -11515,12 +11577,15 @@ an alignment value (in bits) that is preferred or zero and will recieve the alignment of the base register that was selected, irrespective of rather or not CFA_OFFSET is a multiple of that - alignment value. + alignment value. If it is possible for the base register offset to be + non-immediate then SCRATCH_REGNO should specify a scratch register to + use. The valid base registers are taken from CFUN->MACHINE->FS. */ static rtx -choose_baseaddr (HOST_WIDE_INT cfa_offset, unsigned int *align) +choose_baseaddr (HOST_WIDE_INT cfa_offset, unsigned int *align, + unsigned int scratch_regno = INVALID_REGNUM) { rtx base_reg = NULL; HOST_WIDE_INT base_offset = 0; @@ -11534,6 +11599,19 @@ choose_basereg (cfa_offset, base_reg, base_offset, 0, align); gcc_assert (base_reg != NULL); + + rtx base_offset_rtx = GEN_INT (base_offset); + + if (!x86_64_immediate_operand (base_offset_rtx, Pmode)) + { + gcc_assert (scratch_regno != INVALID_REGNUM); + + rtx scratch_reg = gen_rtx_REG (Pmode, scratch_regno); + emit_move_insn (scratch_reg, base_offset_rtx); + + return gen_rtx_PLUS (Pmode, base_reg, scratch_reg); + } + return plus_constant (Pmode, base_reg, base_offset); } @@ -12062,36 +12140,55 @@ } } -/* Release a scratch register obtained from the preceding function. */ - -static void -release_scratch_register_on_entry (struct scratch_reg *sr) +/* Release a scratch register obtained from the preceding function. + + If RELEASE_VIA_POP is true, we just pop the register off the stack + to release it. This is what non-Linux systems use with -fstack-check. + + Otherwise we use OFFSET to locate the saved register and the + allocated stack space becomes part of the local frame and is + deallocated by the epilogue. */ + +static void +release_scratch_register_on_entry (struct scratch_reg *sr, HOST_WIDE_INT offset, + bool release_via_pop) { if (sr->saved) { - struct machine_function *m = cfun->machine; - rtx x, insn = emit_insn (gen_pop (sr->reg)); - - /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */ - RTX_FRAME_RELATED_P (insn) = 1; - x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (UNITS_PER_WORD)); - x = gen_rtx_SET (stack_pointer_rtx, x); - add_reg_note (insn, REG_FRAME_RELATED_EXPR, x); - m->fs.sp_offset -= UNITS_PER_WORD; - } -} - -#define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP) + if (release_via_pop) + { + struct machine_function *m = cfun->machine; + rtx x, insn = emit_insn (gen_pop (sr->reg)); + + /* The RX FRAME_RELATED_P mechanism doesn't know about pop. */ + RTX_FRAME_RELATED_P (insn) = 1; + x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (UNITS_PER_WORD)); + x = gen_rtx_SET (stack_pointer_rtx, x); + add_reg_note (insn, REG_FRAME_RELATED_EXPR, x); + m->fs.sp_offset -= UNITS_PER_WORD; + } + else + { + rtx x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (offset)); + x = gen_rtx_SET (sr->reg, gen_rtx_MEM (word_mode, x)); + emit_insn (x); + } + } +} /* Emit code to adjust the stack pointer by SIZE bytes while probing it. This differs from the next routine in that it tries hard to prevent attacks that jump the stack guard. Thus it is never allowed to allocate more than PROBE_INTERVAL bytes of stack space without a suitable - probe. */ - -static void -ix86_adjust_stack_and_probe_stack_clash (const HOST_WIDE_INT size) + probe. + + INT_REGISTERS_SAVED is true if integer registers have already been + pushed on the stack. */ + +static void +ix86_adjust_stack_and_probe_stack_clash (HOST_WIDE_INT size, + const bool int_registers_saved) { struct machine_function *m = cfun->machine; @@ -12117,15 +12214,39 @@ pointer could be anywhere in the guard page. The safe thing to do is emit a probe now. + The probe can be avoided if we have already emitted any callee + register saves into the stack or have a frame pointer (which will + have been saved as well). Those saves will function as implicit + probes. + ?!? This should be revamped to work like aarch64 and s390 where we track the offset from the most recent probe. Normally that offset would be zero. For a noreturn function we would reset it to PROBE_INTERVAL - (STACK_BOUNDARY / BITS_PER_UNIT). Then we just probe when we cross PROBE_INTERVAL. */ - if (TREE_THIS_VOLATILE (cfun->decl)) - { - emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx, - -GET_MODE_SIZE (word_mode))); + if (TREE_THIS_VOLATILE (cfun->decl) + && !(m->frame.nregs || m->frame.nsseregs || frame_pointer_needed)) + { + /* We can safely use any register here since we're just going to push + its value and immediately pop it back. But we do try and avoid + argument passing registers so as not to introduce dependencies in + the pipeline. For 32 bit we use %esi and for 64 bit we use %rax. */ + rtx dummy_reg = gen_rtx_REG (word_mode, TARGET_64BIT ? AX_REG : SI_REG); + rtx_insn *insn_push = emit_insn (gen_push (dummy_reg)); + rtx_insn *insn_pop = emit_insn (gen_pop (dummy_reg)); + m->fs.sp_offset -= UNITS_PER_WORD; + if (m->fs.cfa_reg == stack_pointer_rtx) + { + m->fs.cfa_offset -= UNITS_PER_WORD; + rtx x = plus_constant (Pmode, stack_pointer_rtx, -UNITS_PER_WORD); + x = gen_rtx_SET (stack_pointer_rtx, x); + add_reg_note (insn_push, REG_CFA_ADJUST_CFA, x); + RTX_FRAME_RELATED_P (insn_push) = 1; + x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD); + x = gen_rtx_SET (stack_pointer_rtx, x); + add_reg_note (insn_pop, REG_CFA_ADJUST_CFA, x); + RTX_FRAME_RELATED_P (insn_pop) = 1; + } emit_insn (gen_blockage ()); } @@ -12144,8 +12265,7 @@ /* We're allocating a large enough stack frame that we need to emit probes. Either emit them inline or in a loop depending on the size. */ - HOST_WIDE_INT probe_interval - = 1 << PARAM_VALUE (PARAM_STACK_CLASH_PROTECTION_PROBE_INTERVAL); + HOST_WIDE_INT probe_interval = get_probe_interval (); if (size <= 4 * probe_interval) { HOST_WIDE_INT i; @@ -12154,7 +12274,7 @@ /* Allocate PROBE_INTERVAL bytes. */ rtx insn = pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, - GEN_INT (-PROBE_INTERVAL), -1, + GEN_INT (-probe_interval), -1, m->fs.cfa_reg == stack_pointer_rtx); add_reg_note (insn, REG_STACK_CHECK, const0_rtx); @@ -12174,9 +12294,21 @@ } else { + /* We expect the GP registers to be saved when probes are used + as the probing sequences might need a scratch register and + the routine to allocate one assumes the integer registers + have already been saved. */ + gcc_assert (int_registers_saved); + struct scratch_reg sr; get_scratch_register_on_entry (&sr); + /* If we needed to save a register, then account for any space + that was pushed (we are not going to pop the register when + we do the restore). */ + if (sr.saved) + size -= UNITS_PER_WORD; + /* Step 1: round SIZE down to a multiple of the interval. */ HOST_WIDE_INT rounded_size = size & -probe_interval; @@ -12225,17 +12357,23 @@ m->fs.cfa_reg == stack_pointer_rtx); dump_stack_clash_frame_info (PROBE_LOOP, size != rounded_size); - release_scratch_register_on_entry (&sr); + /* This does not deallocate the space reserved for the scratch + register. That will be deallocated in the epilogue. */ + release_scratch_register_on_entry (&sr, size, false); } /* Make sure nothing is scheduled before we are done. */ emit_insn (gen_blockage ()); } -/* Emit code to adjust the stack pointer by SIZE bytes while probing it. */ - -static void -ix86_adjust_stack_and_probe (const HOST_WIDE_INT size) +/* Emit code to adjust the stack pointer by SIZE bytes while probing it. + + INT_REGISTERS_SAVED is true if integer registers have already been + pushed on the stack. */ + +static void +ix86_adjust_stack_and_probe (HOST_WIDE_INT size, + const bool int_registers_saved) { /* We skip the probe for the first interval + a small dope of 4 words and probe that many bytes past the specified size to maintain a protection @@ -12247,7 +12385,7 @@ that's the easy case. The run-time loop is made up of 9 insns in the generic case while the compile-time loop is made up of 3+2*(n-1) insns for n # of intervals. */ - if (size <= 4 * PROBE_INTERVAL) + if (size <= 4 * get_probe_interval ()) { HOST_WIDE_INT i, adjust; bool first_probe = true; @@ -12256,15 +12394,15 @@ values of N from 1 until it exceeds SIZE. If only one probe is needed, this will not generate any code. Then adjust and probe to PROBE_INTERVAL + SIZE. */ - for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL) + for (i = get_probe_interval (); i < size; i += get_probe_interval ()) { if (first_probe) { - adjust = 2 * PROBE_INTERVAL + dope; + adjust = 2 * get_probe_interval () + dope; first_probe = false; } else - adjust = PROBE_INTERVAL; + adjust = get_probe_interval (); emit_insn (gen_rtx_SET (stack_pointer_rtx, plus_constant (Pmode, stack_pointer_rtx, @@ -12273,9 +12411,9 @@ } if (first_probe) - adjust = size + PROBE_INTERVAL + dope; - else - adjust = size + PROBE_INTERVAL - i; + adjust = size + get_probe_interval () + dope; + else + adjust = size + get_probe_interval () - i; emit_insn (gen_rtx_SET (stack_pointer_rtx, plus_constant (Pmode, stack_pointer_rtx, @@ -12285,7 +12423,8 @@ /* Adjust back to account for the additional first interval. */ last = emit_insn (gen_rtx_SET (stack_pointer_rtx, plus_constant (Pmode, stack_pointer_rtx, - PROBE_INTERVAL + dope))); + (get_probe_interval () + + dope)))); } /* Otherwise, do the same as above, but in a loop. Note that we must be @@ -12295,15 +12434,26 @@ equality test for the loop condition. */ else { + /* We expect the GP registers to be saved when probes are used + as the probing sequences might need a scratch register and + the routine to allocate one assumes the integer registers + have already been saved. */ + gcc_assert (int_registers_saved); + HOST_WIDE_INT rounded_size; struct scratch_reg sr; get_scratch_register_on_entry (&sr); + /* If we needed to save a register, then account for any space + that was pushed (we are not going to pop the register when + we do the restore). */ + if (sr.saved) + size -= UNITS_PER_WORD; /* Step 1: round SIZE to the previous multiple of the interval. */ - rounded_size = ROUND_DOWN (size, PROBE_INTERVAL); + rounded_size = ROUND_DOWN (size, get_probe_interval ()); /* Step 2: compute initial and final value of the loop counter. */ @@ -12311,7 +12461,7 @@ /* SP = SP_0 + PROBE_INTERVAL. */ emit_insn (gen_rtx_SET (stack_pointer_rtx, plus_constant (Pmode, stack_pointer_rtx, - - (PROBE_INTERVAL + dope)))); + - (get_probe_interval () + dope)))); /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */ if (rounded_size <= (HOST_WIDE_INT_1 << 31)) @@ -12356,9 +12506,12 @@ /* Adjust back to account for the additional first interval. */ last = emit_insn (gen_rtx_SET (stack_pointer_rtx, plus_constant (Pmode, stack_pointer_rtx, - PROBE_INTERVAL + dope))); - - release_scratch_register_on_entry (&sr); + (get_probe_interval () + + dope)))); + + /* This does not deallocate the space reserved for the scratch + register. That will be deallocated in the epilogue. */ + release_scratch_register_on_entry (&sr, size, false); } /* Even if the stack pointer isn't the CFA register, we need to correctly @@ -12373,7 +12526,7 @@ XVECEXP (expr, 0, 1) = gen_rtx_SET (stack_pointer_rtx, plus_constant (Pmode, stack_pointer_rtx, - PROBE_INTERVAL + dope + size)); + get_probe_interval () + dope + size)); add_reg_note (last, REG_FRAME_RELATED_EXPR, expr); RTX_FRAME_RELATED_P (last) = 1; @@ -12400,7 +12553,7 @@ /* SP = SP + PROBE_INTERVAL. */ xops[0] = stack_pointer_rtx; - xops[1] = GEN_INT (PROBE_INTERVAL); + xops[1] = GEN_INT (get_probe_interval ()); output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops); /* Probe at SP. */ @@ -12421,23 +12574,27 @@ } /* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE, - inclusive. These are offsets from the current stack pointer. */ - -static void -ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size) + inclusive. These are offsets from the current stack pointer. + + INT_REGISTERS_SAVED is true if integer registers have already been + pushed on the stack. */ + +static void +ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size, + const bool int_registers_saved) { /* See if we have a constant small number of probes to generate. If so, that's the easy case. The run-time loop is made up of 6 insns in the generic case while the compile-time loop is made up of n insns for n # of intervals. */ - if (size <= 6 * PROBE_INTERVAL) + if (size <= 6 * get_probe_interval ()) { HOST_WIDE_INT i; /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until it exceeds SIZE. If only one probe is needed, this will not generate any code. Then probe at FIRST + SIZE. */ - for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL) + for (i = get_probe_interval (); i < size; i += get_probe_interval ()) emit_stack_probe (plus_constant (Pmode, stack_pointer_rtx, -(first + i))); @@ -12452,6 +12609,12 @@ equality test for the loop condition. */ else { + /* We expect the GP registers to be saved when probes are used + as the probing sequences might need a scratch register and + the routine to allocate one assumes the integer registers + have already been saved. */ + gcc_assert (int_registers_saved); + HOST_WIDE_INT rounded_size, last; struct scratch_reg sr; @@ -12460,7 +12623,7 @@ /* Step 1: round SIZE to the previous multiple of the interval. */ - rounded_size = ROUND_DOWN (size, PROBE_INTERVAL); + rounded_size = ROUND_DOWN (size, get_probe_interval ()); /* Step 2: compute initial and final value of the loop counter. */ @@ -12497,7 +12660,7 @@ sr.reg), rounded_size - size)); - release_scratch_register_on_entry (&sr); + release_scratch_register_on_entry (&sr, size, true); } /* Make sure nothing is scheduled before we are done. */ @@ -12521,7 +12684,7 @@ /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */ xops[0] = reg; - xops[1] = GEN_INT (PROBE_INTERVAL); + xops[1] = GEN_INT (get_probe_interval ()); output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops); /* Probe at TEST_ADDR. */ @@ -12543,6 +12706,62 @@ return ""; } +/* Return true if stack frame is required. Update STACK_ALIGNMENT + to the largest alignment, in bits, of stack slot used if stack + frame is required and CHECK_STACK_SLOT is true. */ + +static bool +ix86_find_max_used_stack_alignment (unsigned int &stack_alignment, + bool check_stack_slot) +{ + HARD_REG_SET set_up_by_prologue, prologue_used; + basic_block bb; + + CLEAR_HARD_REG_SET (prologue_used); + CLEAR_HARD_REG_SET (set_up_by_prologue); + add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM); + add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM); + add_to_hard_reg_set (&set_up_by_prologue, Pmode, + HARD_FRAME_POINTER_REGNUM); + + /* The preferred stack alignment is the minimum stack alignment. */ + if (stack_alignment > crtl->preferred_stack_boundary) + stack_alignment = crtl->preferred_stack_boundary; + + bool require_stack_frame = false; + + FOR_EACH_BB_FN (bb, cfun) + { + rtx_insn *insn; + FOR_BB_INSNS (bb, insn) + if (NONDEBUG_INSN_P (insn) + && requires_stack_frame_p (insn, prologue_used, + set_up_by_prologue)) + { + require_stack_frame = true; + + if (check_stack_slot) + { + /* Find the maximum stack alignment. */ + subrtx_iterator::array_type array; + FOR_EACH_SUBRTX (iter, array, PATTERN (insn), ALL) + if (MEM_P (*iter) + && (reg_mentioned_p (stack_pointer_rtx, + *iter) + || reg_mentioned_p (frame_pointer_rtx, + *iter))) + { + unsigned int alignment = MEM_ALIGN (*iter); + if (alignment > stack_alignment) + stack_alignment = alignment; + } + } + } + } + + return require_stack_frame; +} + /* Finalize stack_realign_needed and frame_pointer_needed flags, which will guide prologue/epilogue to be generated in correct form. */ @@ -12575,7 +12794,7 @@ is used, but in the end nothing that needed the stack alignment had been spilled nor stack access, clear frame_pointer_needed and say we don't need stack realignment. */ - if ((stack_realign || !flag_omit_frame_pointer) + if ((stack_realign || (!flag_omit_frame_pointer && optimize)) && frame_pointer_needed && crtl->is_leaf && crtl->sp_is_unchanging @@ -12593,52 +12812,8 @@ && ix86_nsaved_sseregs () == 0 && ix86_varargs_gpr_size + ix86_varargs_fpr_size == 0) { - HARD_REG_SET set_up_by_prologue, prologue_used; - basic_block bb; - - CLEAR_HARD_REG_SET (prologue_used); - CLEAR_HARD_REG_SET (set_up_by_prologue); - add_to_hard_reg_set (&set_up_by_prologue, Pmode, STACK_POINTER_REGNUM); - add_to_hard_reg_set (&set_up_by_prologue, Pmode, ARG_POINTER_REGNUM); - add_to_hard_reg_set (&set_up_by_prologue, Pmode, - HARD_FRAME_POINTER_REGNUM); - - /* The preferred stack alignment is the minimum stack alignment. */ - if (stack_alignment > crtl->preferred_stack_boundary) - stack_alignment = crtl->preferred_stack_boundary; - - bool require_stack_frame = false; - - FOR_EACH_BB_FN (bb, cfun) - { - rtx_insn *insn; - FOR_BB_INSNS (bb, insn) - if (NONDEBUG_INSN_P (insn) - && requires_stack_frame_p (insn, prologue_used, - set_up_by_prologue)) - { - require_stack_frame = true; - - if (stack_realign) - { - /* Find the maximum stack alignment. */ - subrtx_iterator::array_type array; - FOR_EACH_SUBRTX (iter, array, PATTERN (insn), ALL) - if (MEM_P (*iter) - && (reg_mentioned_p (stack_pointer_rtx, - *iter) - || reg_mentioned_p (frame_pointer_rtx, - *iter))) - { - unsigned int alignment = MEM_ALIGN (*iter); - if (alignment > stack_alignment) - stack_alignment = alignment; - } - } - } - } - - if (require_stack_frame) + if (ix86_find_max_used_stack_alignment (stack_alignment, + stack_realign)) { /* Stack frame is required. If stack alignment needed is less than incoming stack boundary, don't realign stack. */ @@ -12726,6 +12901,18 @@ recompute_frame_layout_p = true; } } + else if (crtl->max_used_stack_slot_alignment >= 128) + { + /* We don't need to realign stack. max_used_stack_alignment is + used to decide how stack frame should be aligned. This is + independent of any psABIs nor 32-bit vs 64-bit. It is always + safe to compute max_used_stack_alignment. We compute it only + if 128-bit aligned load/store may be generated on misaligned + stack slot which will lead to segfault. */ + if (ix86_find_max_used_stack_alignment (stack_alignment, true)) + cfun->machine->max_used_stack_alignment + = stack_alignment / BITS_PER_UNIT; + } if (crtl->stack_realign_needed != stack_realign) recompute_frame_layout_p = true; @@ -12793,24 +12980,20 @@ rtx sym, addr; rtx rax = gen_rtx_REG (word_mode, AX_REG); const struct xlogue_layout &xlogue = xlogue_layout::get_instance (); - HOST_WIDE_INT allocate = frame.stack_pointer_offset - m->fs.sp_offset; /* AL should only be live with sysv_abi. */ gcc_assert (!ix86_eax_live_at_start_p ()); + gcc_assert (m->fs.sp_offset >= frame.sse_reg_save_offset); /* Setup RAX as the stub's base pointer. We use stack_realign_offset rather we've actually realigned the stack or not. */ align = GET_MODE_ALIGNMENT (V4SFmode); addr = choose_baseaddr (frame.stack_realign_offset - + xlogue.get_stub_ptr_offset (), &align); + + xlogue.get_stub_ptr_offset (), &align, AX_REG); gcc_assert (align >= GET_MODE_ALIGNMENT (V4SFmode)); + emit_insn (gen_rtx_SET (rax, addr)); - /* Allocate stack if not already done. */ - if (allocate > 0) - pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, - GEN_INT (-allocate), -1, false); - /* Get the stub symbol. */ sym = xlogue.get_stub_rtx (frame_pointer_needed ? XLOGUE_STUB_SAVE_HFP : XLOGUE_STUB_SAVE); @@ -12837,10 +13020,10 @@ { struct machine_function *m = cfun->machine; rtx insn, t; - struct ix86_frame frame; HOST_WIDE_INT allocate; bool int_registers_saved; bool sse_registers_saved; + bool save_stub_call_needed; rtx static_chain = NULL_RTX; if (ix86_function_naked (current_function_decl)) @@ -12864,7 +13047,7 @@ m->fs.sp_valid = true; m->fs.sp_realigned = false; - frame = m->frame; + const struct ix86_frame &frame = cfun->machine->frame; if (!TARGET_64BIT && ix86_function_ms_hook_prologue (current_function_decl)) { @@ -13016,11 +13199,13 @@ int_registers_saved = (frame.nregs == 0); sse_registers_saved = (frame.nsseregs == 0); + save_stub_call_needed = (m->call_ms2sysv); + gcc_assert (sse_registers_saved || !save_stub_call_needed); if (frame_pointer_needed && !m->fs.fp_valid) { /* Note: AT&T enter does NOT have reversed args. Enter is probably - slower on all targets. Also sdb doesn't like it. */ + slower on all targets. Also sdb didn't like it. */ insn = emit_insn (gen_push (hard_frame_pointer_rtx)); RTX_FRAME_RELATED_P (insn) = 1; @@ -13110,10 +13295,28 @@ target. */ if (TARGET_SEH) m->fs.sp_valid = false; - } - - if (m->call_ms2sysv) - ix86_emit_outlined_ms2sysv_save (frame); + + /* If SP offset is non-immediate after allocation of the stack frame, + then emit SSE saves or stub call prior to allocating the rest of the + stack frame. This is less efficient for the out-of-line stub because + we can't combine allocations across the call barrier, but it's better + than using a scratch register. */ + else if (!x86_64_immediate_operand (GEN_INT (frame.stack_pointer_offset + - m->fs.sp_realigned_offset), + Pmode)) + { + if (!sse_registers_saved) + { + ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset); + sse_registers_saved = true; + } + else if (save_stub_call_needed) + { + ix86_emit_outlined_ms2sysv_save (frame); + save_stub_call_needed = false; + } + } + } allocate = frame.stack_pointer_offset - m->fs.sp_offset; @@ -13167,20 +13370,18 @@ && (flag_stack_check == STATIC_BUILTIN_STACK_CHECK || flag_stack_clash_protection)) { - /* We expect the GP registers to be saved when probes are used. */ - gcc_assert (int_registers_saved); - if (flag_stack_clash_protection) { - ix86_adjust_stack_and_probe_stack_clash (allocate); + ix86_adjust_stack_and_probe_stack_clash (allocate, + int_registers_saved); allocate = 0; } else if (STACK_CHECK_MOVING_SP) { if (!(crtl->is_leaf && !cfun->calls_alloca - && allocate <= PROBE_INTERVAL)) - { - ix86_adjust_stack_and_probe (allocate); + && allocate <= get_probe_interval ())) + { + ix86_adjust_stack_and_probe (allocate, int_registers_saved); allocate = 0; } } @@ -13195,24 +13396,28 @@ { if (crtl->is_leaf && !cfun->calls_alloca) { - if (size > PROBE_INTERVAL) - ix86_emit_probe_stack_range (0, size); + if (size > get_probe_interval ()) + ix86_emit_probe_stack_range (0, size, int_registers_saved); } else ix86_emit_probe_stack_range (0, - size + get_stack_check_protect ()); + size + get_stack_check_protect (), + int_registers_saved); } else { if (crtl->is_leaf && !cfun->calls_alloca) { - if (size > PROBE_INTERVAL + if (size > get_probe_interval () && size > get_stack_check_protect ()) ix86_emit_probe_stack_range (get_stack_check_protect (), - size - get_stack_check_protect ()); + (size + - get_stack_check_protect ()), + int_registers_saved); } else - ix86_emit_probe_stack_range (get_stack_check_protect (), size); + ix86_emit_probe_stack_range (get_stack_check_protect (), size, + int_registers_saved); } } } @@ -13337,6 +13542,8 @@ ix86_emit_save_regs_using_mov (frame.reg_save_offset); if (!sse_registers_saved) ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset); + else if (save_stub_call_needed) + ix86_emit_outlined_ms2sysv_save (frame); /* For the mcount profiling on 32 bit PIC mode we need to emit SET_GOT in PROLOGUE. */ @@ -13577,8 +13784,9 @@ /* Setup RSI as the stub's base pointer. */ align = GET_MODE_ALIGNMENT (V4SFmode); - tmp = choose_baseaddr (rsi_offset, &align); + tmp = choose_baseaddr (rsi_offset, &align, SI_REG); gcc_assert (align >= GET_MODE_ALIGNMENT (V4SFmode)); + emit_insn (gen_rtx_SET (rsi, tmp)); /* Get a symbol for the stub. */ @@ -13715,7 +13923,6 @@ { struct machine_function *m = cfun->machine; struct machine_frame_state frame_state_save = m->fs; - struct ix86_frame frame; bool restore_regs_via_mov; bool using_drap; bool restore_stub_is_tail = false; @@ -13728,7 +13935,7 @@ } ix86_finalize_stack_frame_flags (); - frame = m->frame; + const struct ix86_frame &frame = cfun->machine->frame; m->fs.sp_realigned = stack_realign_fp; m->fs.sp_valid = stack_realign_fp @@ -13772,11 +13979,13 @@ + UNITS_PER_WORD); } + HOST_WIDE_INT reg_save_offset = frame.reg_save_offset; + /* Special care must be taken for the normal return case of a function using eh_return: the eax and edx registers are marked as saved, but not restored along this path. Adjust the save location to match. */ if (crtl->calls_eh_return && style != 2) - frame.reg_save_offset -= 2 * UNITS_PER_WORD; + reg_save_offset -= 2 * UNITS_PER_WORD; /* EH_RETURN requires the use of moves to function properly. */ if (crtl->calls_eh_return) @@ -13792,11 +14001,11 @@ else if (TARGET_EPILOGUE_USING_MOVE && cfun->machine->use_fast_prologue_epilogue && (frame.nregs > 1 - || m->fs.sp_offset != frame.reg_save_offset)) + || m->fs.sp_offset != reg_save_offset)) restore_regs_via_mov = true; else if (frame_pointer_needed && !frame.nregs - && m->fs.sp_offset != frame.reg_save_offset) + && m->fs.sp_offset != reg_save_offset) restore_regs_via_mov = true; else if (frame_pointer_needed && TARGET_USE_LEAVE @@ -13812,7 +14021,7 @@ the stack pointer, if we will restore SSE regs via sp. */ if (TARGET_64BIT && m->fs.sp_offset > 0x7fffffff - && sp_valid_at (frame.stack_realign_offset) + && sp_valid_at (frame.stack_realign_offset + 1) && (frame.nsseregs + frame.nregs) != 0) { pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, @@ -13864,7 +14073,7 @@ rtx t; if (frame.nregs) - ix86_emit_restore_regs_using_mov (frame.reg_save_offset, style == 2); + ix86_emit_restore_regs_using_mov (reg_save_offset, style == 2); /* eh_return epilogues need %ecx added to the stack pointer. */ if (style == 2) @@ -13895,7 +14104,9 @@ offset relative to SA, and after this insn we have no other reasonable register to use for the CFA. We don't bother resetting the CFA to the SP for the duration of - the return insn. */ + the return insn, unless the control flow instrumentation + is done. In this case the SP is used later and we have + to reset CFA to SP. */ add_reg_note (insn, REG_CFA_DEF_CFA, plus_constant (Pmode, sa, UNITS_PER_WORD)); ix86_add_queued_cfa_restore_notes (insn); @@ -13907,7 +14118,8 @@ m->fs.fp_valid = false; pro_epilogue_adjust_stack (stack_pointer_rtx, sa, - const0_rtx, style, false); + const0_rtx, style, + flag_cf_protection); } else { @@ -13940,9 +14152,9 @@ if (TARGET_SEH) { /* Prevent a catch region from being adjacent to the standard - epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor - several other flags that would be interesting to test are - not yet set up. */ + epilogue sequence. Unfortunately neither crtl->uses_eh_lsda + nor several other flags that would be interesting to test are + set up yet. */ if (flag_non_call_exceptions) emit_insn (gen_nops (const1_rtx)); else @@ -13956,19 +14168,19 @@ in epilogues. */ if (!m->fs.sp_valid || m->fs.sp_realigned || (TARGET_SEH - && (m->fs.sp_offset - frame.reg_save_offset + && (m->fs.sp_offset - reg_save_offset >= SEH_MAX_FRAME_SIZE))) { pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx, GEN_INT (m->fs.fp_offset - - frame.reg_save_offset), + - reg_save_offset), style, false); } - else if (m->fs.sp_offset != frame.reg_save_offset) + else if (m->fs.sp_offset != reg_save_offset) { pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, GEN_INT (m->fs.sp_offset - - frame.reg_save_offset), + - reg_save_offset), style, m->fs.cfa_reg == stack_pointer_rtx); } @@ -14091,7 +14303,32 @@ emit_jump_insn (gen_simple_return_pop_internal (popc)); } else if (!m->call_ms2sysv || !restore_stub_is_tail) - emit_jump_insn (gen_simple_return_internal ()); + { + /* In case of return from EH a simple return cannot be used + as a return address will be compared with a shadow stack + return address. Use indirect jump instead. */ + if (style == 2 && flag_cf_protection) + { + /* Register used in indirect jump must be in word_mode. But + Pmode may not be the same as word_mode for x32. */ + rtx ecx = gen_rtx_REG (word_mode, CX_REG); + rtx_insn *insn; + + insn = emit_insn (gen_pop (ecx)); + m->fs.cfa_offset -= UNITS_PER_WORD; + m->fs.sp_offset -= UNITS_PER_WORD; + + rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD); + x = gen_rtx_SET (stack_pointer_rtx, x); + add_reg_note (insn, REG_CFA_ADJUST_CFA, x); + add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx)); + RTX_FRAME_RELATED_P (insn) = 1; + + emit_jump_insn (gen_simple_return_indirect_internal (ecx)); + } + else + emit_jump_insn (gen_simple_return_internal ()); + } /* Restore the state back to the state from the prologue, so that it's correct for the next epilogue. */ @@ -14275,11 +14512,11 @@ void ix86_expand_split_stack_prologue (void) { - struct ix86_frame frame; HOST_WIDE_INT allocate; unsigned HOST_WIDE_INT args_size; rtx_code_label *label; - rtx limit, current, allocate_rtx, call_insn, call_fusage; + rtx limit, current, allocate_rtx, call_fusage; + rtx_insn *call_insn; rtx scratch_reg = NULL_RTX; rtx_code_label *varargs_label = NULL; rtx fn; @@ -14287,7 +14524,7 @@ gcc_assert (flag_split_stack && reload_completed); ix86_finalize_stack_frame_flags (); - frame = cfun->machine->frame; + struct ix86_frame &frame = cfun->machine->frame; allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET; /* This is the label we will branch to if we have enough stack @@ -14356,7 +14593,7 @@ anyhow. In 64-bit mode we pass the parameters in r10 and r11. */ allocate_rtx = GEN_INT (allocate); - args_size = crtl->args.size >= 0 ? crtl->args.size : 0; + args_size = crtl->args.size >= 0 ? (HOST_WIDE_INT) crtl->args.size : 0; call_fusage = NULL_RTX; rtx pop = NULL_RTX; if (TARGET_64BIT) @@ -14449,7 +14686,7 @@ if (!TARGET_64BIT) add_reg_note (call_insn, REG_ARGS_SIZE, GEN_INT (0)); /* Indicate that this function can't jump to non-local gotos. */ - make_reg_eh_region_note_nothrow_nononlocal (as_a <rtx_insn *> (call_insn)); + make_reg_eh_region_note_nothrow_nononlocal (call_insn); /* In order to make call/return prediction work right, we now need to execute a return instruction. See @@ -14459,7 +14696,16 @@ instruction--we need control flow to continue at the subsequent label. Therefore, we use an unspec. */ gcc_assert (crtl->args.pops_args < 65536); - emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args))); + rtx_insn *ret_insn + = emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args))); + + if ((flag_cf_protection & CF_BRANCH)) + { + /* Insert ENDBR since __morestack will jump back here via indirect + call. */ + rtx cet_eb = gen_nop_endbr (); + emit_insn_after (cet_eb, ret_insn); + } /* If we are in 64-bit mode and this function uses a static chain, we saved %r10 in %rax before calling _morestack. */ @@ -14878,10 +15124,6 @@ static bool ix86_legitimate_constant_p (machine_mode mode, rtx x) { - /* Pointer bounds constants are not valid. */ - if (POINTER_BOUNDS_MODE_P (GET_MODE (x))) - return false; - switch (GET_CODE (x)) { case CONST: @@ -15079,10 +15321,16 @@ break; op0 = XEXP (XEXP (disp, 0), 0); op1 = XEXP (XEXP (disp, 0), 1); - if (!CONST_INT_P (op1) - || INTVAL (op1) >= 16*1024*1024 + if (!CONST_INT_P (op1)) + break; + if (GET_CODE (op0) == UNSPEC + && (XINT (op0, 1) == UNSPEC_DTPOFF + || XINT (op0, 1) == UNSPEC_NTPOFF) + && trunc_int_for_mode (INTVAL (op1), SImode) == INTVAL (op1)) + return true; + if (INTVAL (op1) >= 16*1024*1024 || INTVAL (op1) < -16*1024*1024) - break; + break; if (GET_CODE (op0) == LABEL_REF) return true; if (GET_CODE (op0) == CONST @@ -16930,7 +17178,7 @@ { const char *suffix; - if (mode == CCFPmode || mode == CCFPUmode) + if (mode == CCFPmode) { code = ix86_fp_compare_code_to_integer (code); mode = CCmode; @@ -17085,6 +17333,7 @@ If CODE is 'h', pretend the reg is the 'high' byte register. If CODE is 'y', print "st(0)" instead of "st", if the reg is stack op. If CODE is 'd', duplicate the operand for AVX instruction. + If CODE is 'V', print naked full integer register name without %. */ void @@ -17095,7 +17344,7 @@ unsigned int regno; bool duplicated; - if (ASSEMBLER_DIALECT == ASM_ATT) + if (ASSEMBLER_DIALECT == ASM_ATT && code != 'V') putc ('%', file); if (x == pc_rtx) @@ -17134,8 +17383,7 @@ if (regno == ARG_POINTER_REGNUM || regno == FRAME_POINTER_REGNUM - || regno == FPSR_REG - || regno == FPCR_REG) + || regno == FPSR_REG) { output_operand_lossage ("invalid use of register '%s'", reg_names[regno]); @@ -17147,6 +17395,14 @@ return; } + if (code == 'V') + { + if (GENERAL_REGNO_P (regno)) + msize = GET_MODE_SIZE (word_mode); + else + error ("'V' modifier on non-integer register"); + } + duplicated = code == 'd' && TARGET_AVX; switch (msize) @@ -17238,7 +17494,7 @@ F,f -- likewise, but for floating-point. O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.", otherwise nothing - R -- print embeded rounding and sae. + R -- print embedded rounding and sae. r -- print only sae. z -- print the opcode suffix for the size of the current operand. Z -- likewise, with special suffixes for x87 instructions. @@ -17266,11 +17522,12 @@ & -- print some in-use local-dynamic symbol name. H -- print a memory address offset by 8; used for sse high-parts Y -- print condition for XOP pcom* instruction. + V -- print naked full integer register name without %. + -- print a branch hint as 'cs' or 'ds' prefix ; -- print a semicolon (after prefixes due to bug in older gas). ~ -- print "i" if TARGET_AVX2, "f" otherwise. ^ -- print addr32 prefix if TARGET_64BIT and Pmode != word_mode - ! -- print MPX prefix for jxx/call/ret instructions if required. + ! -- print NOTRACK prefix for jxx/call/ret instructions if required. */ void @@ -17489,6 +17746,7 @@ case 'X': case 'P': case 'p': + case 'V': break; case 's': @@ -17822,8 +18080,6 @@ return; case '!': - if (ix86_bnd_prefixed_insn_p (current_output_insn)) - fputs ("bnd ", file); if (ix86_notrack_prefixed_insn_p (current_output_insn)) fputs ("notrack ", file); return; @@ -17940,7 +18196,8 @@ since we can in fact encode that into an immediate. */ if (GET_CODE (x) == CONST_VECTOR) { - gcc_assert (x == CONST0_RTX (GET_MODE (x))); + if (x != CONST0_RTX (GET_MODE (x))) + output_operand_lossage ("invalid vector immediate"); x = const0_rtx; } @@ -18004,25 +18261,6 @@ ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts); code = 'q'; } - else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDMK_ADDR) - { - ok = ix86_decompose_address (XVECEXP (addr, 0, 1), &parts); - gcc_assert (parts.base == NULL_RTX || parts.index == NULL_RTX); - if (parts.base != NULL_RTX) - { - parts.index = parts.base; - parts.scale = 1; - } - parts.base = XVECEXP (addr, 0, 0); - addr = XVECEXP (addr, 0, 0); - } - else if (GET_CODE (addr) == UNSPEC && XINT (addr, 1) == UNSPEC_BNDLDX_ADDR) - { - ok = ix86_decompose_address (XVECEXP (addr, 0, 0), &parts); - gcc_assert (parts.index == NULL_RTX); - parts.index = XVECEXP (addr, 0, 1); - addr = XVECEXP (addr, 0, 0); - } else ok = ix86_decompose_address (addr, &parts); @@ -18578,16 +18816,17 @@ return X86_DIRFLAG_ANY; } -/* Check if a 256bit AVX register is referenced inside of EXP. */ - -static bool -ix86_check_avx256_register (const_rtx exp) +/* Check if a 256bit or 512 bit AVX register is referenced inside of EXP. */ + +static bool +ix86_check_avx_upper_register (const_rtx exp) { if (SUBREG_P (exp)) exp = SUBREG_REG (exp); return (REG_P (exp) - && VALID_AVX256_REG_OR_OI_MODE (GET_MODE (exp))); + && (VALID_AVX256_REG_OR_OI_MODE (GET_MODE (exp)) + || VALID_AVX512F_REG_OR_XI_MODE (GET_MODE (exp)))); } /* Return needed mode for entity in optimize_mode_switching pass. */ @@ -18600,7 +18839,7 @@ rtx link; /* Needed mode is set to AVX_U128_CLEAN if there are - no 256bit modes used in function arguments. */ + no 256bit or 512bit modes used in function arguments. */ for (link = CALL_INSN_FUNCTION_USAGE (insn); link; link = XEXP (link, 1)) @@ -18609,7 +18848,7 @@ { rtx arg = XEXP (XEXP (link, 0), 0); - if (ix86_check_avx256_register (arg)) + if (ix86_check_avx_upper_register (arg)) return AVX_U128_DIRTY; } } @@ -18617,13 +18856,13 @@ return AVX_U128_CLEAN; } - /* Require DIRTY mode if a 256bit AVX register is referenced. Hardware - changes state only when a 256bit register is written to, but we need - to prevent the compiler from moving optimal insertion point above - eventual read from 256bit register. */ + /* Require DIRTY mode if a 256bit or 512bit AVX register is referenced. + Hardware changes state only when a 256bit register is written to, + but we need to prevent the compiler from moving optimal insertion + point above eventual read from 256bit or 512 bit register. */ subrtx_iterator::array_type array; FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST) - if (ix86_check_avx256_register (*iter)) + if (ix86_check_avx_upper_register (*iter)) return AVX_U128_DIRTY; return AVX_U128_ANY; @@ -18670,11 +18909,6 @@ return mode; break; - case I387_MASK_PM: - if (mode == I387_CW_MASK_PM) - return mode; - break; - default: gcc_unreachable (); } @@ -18697,7 +18931,6 @@ case I387_TRUNC: case I387_FLOOR: case I387_CEIL: - case I387_MASK_PM: return ix86_i387_mode_needed (entity, insn); default: gcc_unreachable (); @@ -18705,12 +18938,12 @@ return 0; } -/* Check if a 256bit AVX register is referenced in stores. */ +/* Check if a 256bit or 512bit AVX register is referenced in stores. */ static void -ix86_check_avx256_stores (rtx dest, const_rtx, void *data) +ix86_check_avx_upper_stores (rtx dest, const_rtx, void *data) { - if (ix86_check_avx256_register (dest)) + if (ix86_check_avx_upper_register (dest)) { bool *used = (bool *) data; *used = true; @@ -18724,23 +18957,23 @@ { rtx pat = PATTERN (insn); - if (vzeroupper_operation (pat, VOIDmode) - || vzeroall_operation (pat, VOIDmode)) + if (vzeroupper_pattern (pat, VOIDmode) + || vzeroall_pattern (pat, VOIDmode)) return AVX_U128_CLEAN; /* We know that state is clean after CALL insn if there are no - 256bit registers used in the function return register. */ + 256bit or 512bit registers used in the function return register. */ if (CALL_P (insn)) { - bool avx_reg256_found = false; - note_stores (pat, ix86_check_avx256_stores, &avx_reg256_found); - - return avx_reg256_found ? AVX_U128_DIRTY : AVX_U128_CLEAN; + bool avx_upper_reg_found = false; + note_stores (pat, ix86_check_avx_upper_stores, &avx_upper_reg_found); + + return avx_upper_reg_found ? AVX_U128_DIRTY : AVX_U128_CLEAN; } /* Otherwise, return current mode. Remember that if insn - references AVX 256bit registers, the mode was already changed - to DIRTY from MODE_NEEDED. */ + references AVX 256bit or 512bit registers, the mode was already + changed to DIRTY from MODE_NEEDED. */ return mode; } @@ -18758,7 +18991,6 @@ case I387_TRUNC: case I387_FLOOR: case I387_CEIL: - case I387_MASK_PM: return mode; default: gcc_unreachable (); @@ -18783,13 +19015,13 @@ tree arg; /* Entry mode is set to AVX_U128_DIRTY if there are - 256bit modes used in function arguments. */ + 256bit or 512bit modes used in function arguments. */ for (arg = DECL_ARGUMENTS (current_function_decl); arg; arg = TREE_CHAIN (arg)) { rtx incoming = DECL_INCOMING_RTL (arg); - if (incoming && ix86_check_avx256_register (incoming)) + if (incoming && ix86_check_avx_upper_register (incoming)) return AVX_U128_DIRTY; } @@ -18811,7 +19043,6 @@ case I387_TRUNC: case I387_FLOOR: case I387_CEIL: - case I387_MASK_PM: return I387_CW_ANY; default: gcc_unreachable (); @@ -18823,9 +19054,9 @@ { rtx reg = crtl->return_rtx; - /* Exit mode is set to AVX_U128_DIRTY if there are - 256bit modes used in the function return register. */ - if (reg && ix86_check_avx256_register (reg)) + /* Exit mode is set to AVX_U128_DIRTY if there are 256bit + or 512 bit modes used in the function return register. */ + if (reg && ix86_check_avx_upper_register (reg)) return AVX_U128_DIRTY; return AVX_U128_CLEAN; @@ -18846,7 +19077,6 @@ case I387_TRUNC: case I387_FLOOR: case I387_CEIL: - case I387_MASK_PM: return I387_CW_ANY; default: gcc_unreachable (); @@ -18876,72 +19106,30 @@ emit_insn (gen_x86_fnstcw_1 (stored_mode)); emit_move_insn (reg, copy_rtx (stored_mode)); - if (TARGET_64BIT || TARGET_PARTIAL_REG_STALL - || optimize_insn_for_size_p ()) - { - switch (mode) - { - case I387_CW_TRUNC: - /* round toward zero (truncate) */ - emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00))); - slot = SLOT_CW_TRUNC; - break; - - case I387_CW_FLOOR: - /* round down toward -oo */ - emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00))); - emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400))); - slot = SLOT_CW_FLOOR; - break; - - case I387_CW_CEIL: - /* round up toward +oo */ - emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00))); - emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800))); - slot = SLOT_CW_CEIL; - break; - - case I387_CW_MASK_PM: - /* mask precision exception for nearbyint() */ - emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020))); - slot = SLOT_CW_MASK_PM; - break; - - default: - gcc_unreachable (); - } - } - else - { - switch (mode) - { - case I387_CW_TRUNC: - /* round toward zero (truncate) */ - emit_insn (gen_insvsi_1 (reg, GEN_INT (0xc))); - slot = SLOT_CW_TRUNC; - break; - - case I387_CW_FLOOR: - /* round down toward -oo */ - emit_insn (gen_insvsi_1 (reg, GEN_INT (0x4))); - slot = SLOT_CW_FLOOR; - break; - - case I387_CW_CEIL: - /* round up toward +oo */ - emit_insn (gen_insvsi_1 (reg, GEN_INT (0x8))); - slot = SLOT_CW_CEIL; - break; - - case I387_CW_MASK_PM: - /* mask precision exception for nearbyint() */ - emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0020))); - slot = SLOT_CW_MASK_PM; - break; - - default: - gcc_unreachable (); - } + switch (mode) + { + case I387_CW_TRUNC: + /* round toward zero (truncate) */ + emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0c00))); + slot = SLOT_CW_TRUNC; + break; + + case I387_CW_FLOOR: + /* round down toward -oo */ + emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00))); + emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0400))); + slot = SLOT_CW_FLOOR; + break; + + case I387_CW_CEIL: + /* round up toward +oo */ + emit_insn (gen_andhi3 (reg, reg, GEN_INT (~0x0c00))); + emit_insn (gen_iorhi3 (reg, reg, GEN_INT (0x0800))); + slot = SLOT_CW_CEIL; + break; + + default: + gcc_unreachable (); } gcc_assert (slot < MAX_386_STACK_LOCALS); @@ -18995,7 +19183,6 @@ case I387_TRUNC: case I387_FLOOR: case I387_CEIL: - case I387_MASK_PM: if (mode != I387_CW_ANY && mode != I387_CW_UNINITIALIZED) emit_i387_cw_initialization (mode); @@ -19171,8 +19358,6 @@ if (TARGET_64BIT || TARGET_VXWORKS_RTP) fprintf (file, "%s%s%d-%s%d\n", directive, LPREFIX, value, LPREFIX, rel); - else if (HAVE_AS_GOTOFF_IN_DATA) - fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value); #if TARGET_MACHO else if (TARGET_MACHO) { @@ -19181,6 +19366,8 @@ putc ('\n', file); } #endif + else if (HAVE_AS_GOTOFF_IN_DATA) + fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value); else asm_fprintf (file, ASM_LONG "%U%s+[.-%s%d]\n", GOT_SYMBOL_NAME, LPREFIX, value); @@ -19211,20 +19398,6 @@ emit_insn (tmp); } -/* X is an unchanging MEM. If it is a constant pool reference, return - the constant pool rtx, else NULL. */ - -rtx -maybe_get_pool_constant (rtx x) -{ - x = ix86_delegitimize_address (XEXP (x, 0)); - - if (GET_CODE (x) == SYMBOL_REF && CONSTANT_POOL_ADDRESS_P (x)) - return get_pool_constant (x); - - return NULL_RTX; -} - void ix86_expand_move (machine_mode mode, rtx operands[]) { @@ -19730,7 +19903,8 @@ rtx src2 = operands[2]; /* If the operation is not commutative, we can't do anything. */ - if (GET_RTX_CLASS (code) != RTX_COMM_ARITH) + if (GET_RTX_CLASS (code) != RTX_COMM_ARITH + && GET_RTX_CLASS (code) != RTX_COMM_COMPARE) return false; /* Highest priority is that src1 should match dst. */ @@ -19961,7 +20135,7 @@ /* If the destination is memory, we must have a matching source operand. */ if (MEM_P (dst) && !rtx_equal_p (dst, src1)) - return false; + return false; /* Source 1 cannot be a constant. */ if (CONSTANT_P (src1)) @@ -20486,7 +20660,8 @@ /* For Silvermont if using a 2-source or 3-source LEA for non-destructive destination purposes, or due to wanting ability to use SCALE, the use of LEA is justified. */ - if (TARGET_SILVERMONT || TARGET_INTEL) + if (TARGET_SILVERMONT || TARGET_GOLDMONT || TARGET_GOLDMONT_PLUS + || TARGET_TREMONT || TARGET_INTEL) { if (has_scale) return true; @@ -21709,14 +21884,13 @@ return gen_rtx_fmt_ee (code, VOIDmode, flags, const0_rtx); } -/* Figure out whether to use ordered or unordered fp comparisons. - Return the appropriate mode to use. */ - -machine_mode -ix86_fp_compare_mode (enum rtx_code code) +/* Figure out whether to use unordered fp comparisons. */ + +static bool +ix86_unordered_fp_compare (enum rtx_code code) { if (!TARGET_IEEE_FP) - return CCFPmode; + return false; switch (code) { @@ -21724,7 +21898,7 @@ case GE: case LT: case LE: - return CCFPmode; + return false; case EQ: case NE: @@ -21737,7 +21911,7 @@ case UNGT: case UNGE: case UNEQ: - return CCFPUmode; + return true; default: gcc_unreachable (); @@ -21752,7 +21926,7 @@ if (SCALAR_FLOAT_MODE_P (mode)) { gcc_assert (!DECIMAL_FLOAT_MODE_P (mode)); - return ix86_fp_compare_mode (code); + return CCFPmode; } switch (code) @@ -21808,7 +21982,7 @@ ix86_fixed_condition_code_regs (unsigned int *p1, unsigned int *p2) { *p1 = FLAGS_REG; - *p2 = FPSR_REG; + *p2 = INVALID_REGNUM; return true; } @@ -21874,7 +22048,6 @@ } case E_CCFPmode: - case E_CCFPUmode: /* These are only compatible with themselves, which we already checked above. */ return VOIDmode; @@ -21978,7 +22151,7 @@ static enum rtx_code ix86_prepare_fp_compare_args (enum rtx_code code, rtx *pop0, rtx *pop1) { - machine_mode fpcmp_mode = ix86_fp_compare_mode (code); + bool unordered_compare = ix86_unordered_fp_compare (code); rtx op0 = *pop0, op1 = *pop1; machine_mode op_mode = GET_MODE (op0); bool is_sse = TARGET_SSE_MATH && SSE_FLOAT_MODE_P (op_mode); @@ -21990,7 +22163,7 @@ floating point. */ if (!is_sse - && (fpcmp_mode == CCFPUmode + && (unordered_compare || (op_mode == XFmode && ! (standard_80387_constant_p (op0) == 1 || standard_80387_constant_p (op1) == 1) @@ -22085,49 +22258,45 @@ /* Generate insn patterns to do a floating point compare of OPERANDS. */ static rtx -ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1, rtx scratch) -{ - machine_mode fpcmp_mode, intcmp_mode; - rtx tmp, tmp2; - - fpcmp_mode = ix86_fp_compare_mode (code); +ix86_expand_fp_compare (enum rtx_code code, rtx op0, rtx op1) +{ + bool unordered_compare = ix86_unordered_fp_compare (code); + machine_mode cmp_mode; + rtx tmp, scratch; + code = ix86_prepare_fp_compare_args (code, &op0, &op1); + tmp = gen_rtx_COMPARE (CCFPmode, op0, op1); + if (unordered_compare) + tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP); + /* Do fcomi/sahf based test when profitable. */ switch (ix86_fp_comparison_strategy (code)) { case IX86_FPCMP_COMI: - intcmp_mode = fpcmp_mode; - tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1); - tmp = gen_rtx_SET (gen_rtx_REG (fpcmp_mode, FLAGS_REG), tmp); - emit_insn (tmp); + cmp_mode = CCFPmode; + emit_insn (gen_rtx_SET (gen_rtx_REG (CCFPmode, FLAGS_REG), tmp)); break; case IX86_FPCMP_SAHF: - intcmp_mode = fpcmp_mode; - tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1); - tmp = gen_rtx_SET (gen_rtx_REG (fpcmp_mode, FLAGS_REG), tmp); - - if (!scratch) - scratch = gen_reg_rtx (HImode); - tmp2 = gen_rtx_CLOBBER (VOIDmode, scratch); - emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, tmp, tmp2))); + cmp_mode = CCFPmode; + tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW); + scratch = gen_reg_rtx (HImode); + emit_insn (gen_rtx_SET (scratch, tmp)); + emit_insn (gen_x86_sahf_1 (scratch)); break; case IX86_FPCMP_ARITH: - /* Sadness wrt reg-stack pops killing fpsr -- gotta get fnstsw first. */ - tmp = gen_rtx_COMPARE (fpcmp_mode, op0, op1); - tmp2 = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW); - if (!scratch) - scratch = gen_reg_rtx (HImode); - emit_insn (gen_rtx_SET (scratch, tmp2)); + cmp_mode = CCNOmode; + tmp = gen_rtx_UNSPEC (HImode, gen_rtvec (1, tmp), UNSPEC_FNSTSW); + scratch = gen_reg_rtx (HImode); + emit_insn (gen_rtx_SET (scratch, tmp)); /* In the unordered case, we have to check C2 for NaN's, which doesn't happen to work out to anything nice combination-wise. So do some bit twiddling on the value we've got in AH to come up with an appropriate set of condition codes. */ - intcmp_mode = CCNOmode; switch (code) { case GT: @@ -22142,7 +22311,7 @@ emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45))); emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx)); emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x44))); - intcmp_mode = CCmode; + cmp_mode = CCmode; code = GEU; } break; @@ -22152,7 +22321,7 @@ { emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45))); emit_insn (gen_cmpqi_ext_3 (scratch, const1_rtx)); - intcmp_mode = CCmode; + cmp_mode = CCmode; code = EQ; } else @@ -22182,7 +22351,7 @@ emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45))); emit_insn (gen_addqi_ext_1 (scratch, scratch, constm1_rtx)); emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40))); - intcmp_mode = CCmode; + cmp_mode = CCmode; code = LTU; } else @@ -22197,7 +22366,7 @@ { emit_insn (gen_andqi_ext_1 (scratch, scratch, GEN_INT (0x45))); emit_insn (gen_cmpqi_ext_3 (scratch, GEN_INT (0x40))); - intcmp_mode = CCmode; + cmp_mode = CCmode; code = EQ; } else @@ -22243,7 +22412,7 @@ /* Return the test that should be put into the flags user, i.e. the bcc, scc, or cmov instruction. */ return gen_rtx_fmt_ee (code, VOIDmode, - gen_rtx_REG (intcmp_mode, FLAGS_REG), + gen_rtx_REG (cmp_mode, FLAGS_REG), const0_rtx); } @@ -22258,7 +22427,7 @@ else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0))) { gcc_assert (!DECIMAL_FLOAT_MODE_P (GET_MODE (op0))); - ret = ix86_expand_fp_compare (code, op0, op1, NULL_RTX); + ret = ix86_expand_fp_compare (code, op0, op1); } else ret = ix86_expand_int_compare (code, op0, op1); @@ -22558,12 +22727,11 @@ we decide to expand comparison using arithmetic that is not too common scenario. */ start_sequence (); - compare_op = ix86_expand_fp_compare (code, op0, op1, NULL_RTX); + compare_op = ix86_expand_fp_compare (code, op0, op1); compare_seq = get_insns (); end_sequence (); - if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode - || GET_MODE (XEXP (compare_op, 0)) == CCFPUmode) + if (GET_MODE (XEXP (compare_op, 0)) == CCFPmode) code = ix86_fp_compare_code_to_integer (GET_CODE (compare_op)); else code = GET_CODE (compare_op); @@ -22703,8 +22871,7 @@ flags = XEXP (compare_op, 0); - if (GET_MODE (flags) == CCFPmode - || GET_MODE (flags) == CCFPUmode) + if (GET_MODE (flags) == CCFPmode) { fpcmp = true; compare_code @@ -24342,9 +24509,7 @@ t2 = gen_reg_rtx (V32QImode); t3 = gen_reg_rtx (V32QImode); vt2 = GEN_INT (-128); - for (i = 0; i < 32; i++) - vec[i] = vt2; - vt = gen_rtx_CONST_VECTOR (V32QImode, gen_rtvec_v (32, vec)); + vt = gen_const_vec_duplicate (V32QImode, vt2); vt = force_reg (V32QImode, vt); for (i = 0; i < 32; i++) vec[i] = i < 16 ? vt2 : const0_rtx; @@ -24452,9 +24617,7 @@ vt = GEN_INT (w - 1); } - for (i = 0; i < w; i++) - vec[i] = vt; - vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec)); + vt = gen_const_vec_duplicate (maskmode, vt); mask = expand_simple_binop (maskmode, AND, mask, vt, NULL_RTX, 0, OPTAB_DIRECT); @@ -24544,9 +24707,7 @@ e = w = 4; } - for (i = 0; i < w; i++) - vec[i] = vt; - vt = gen_rtx_CONST_VECTOR (maskmode, gen_rtvec_v (w, vec)); + vt = gen_const_vec_duplicate (maskmode, vt); vt = force_reg (maskmode, vt); mask = expand_simple_binop (maskmode, AND, mask, vt, NULL_RTX, 0, OPTAB_DIRECT); @@ -24744,8 +24905,7 @@ flags = XEXP (compare_op, 0); - if (GET_MODE (flags) == CCFPmode - || GET_MODE (flags) == CCFPUmode) + if (GET_MODE (flags) == CCFPmode) { fpcmp = true; code = ix86_fp_compare_code_to_integer (code); @@ -24832,11 +24992,7 @@ /* Optimize constant pool reference to immediates. This is used by fp moves, that force all constants to memory to allow combining. */ if (MEM_P (operand) && MEM_READONLY_P (operand)) - { - rtx tmp = maybe_get_pool_constant (operand); - if (tmp) - operand = tmp; - } + operand = avoid_constant_pool_reference (operand); if (MEM_P (operand) && !offsettable_memref_p (operand)) { @@ -27048,6 +27204,11 @@ issetmem, issetmem && val_exp == const0_rtx, have_as, &dynamic_check, &noalign, false); + + if (dump_file) + fprintf (dump_file, "Selected stringop expansion strategy: %s\n", + stringop_alg_names[alg]); + if (alg == libcall) return false; gcc_assert (alg != no_stringop); @@ -27712,7 +27873,7 @@ return tmp; } -rtx +rtx_insn * ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1, rtx callarg2, rtx pop, bool sibcall) @@ -27847,30 +28008,7 @@ call = gen_rtx_CALL (VOIDmode, fnaddr, callarg1); if (retval) - { - /* We should add bounds as destination register in case - pointer with bounds may be returned. */ - if (TARGET_MPX && SCALAR_INT_MODE_P (GET_MODE (retval))) - { - rtx b0 = gen_rtx_REG (BND64mode, FIRST_BND_REG); - rtx b1 = gen_rtx_REG (BND64mode, FIRST_BND_REG + 1); - if (GET_CODE (retval) == PARALLEL) - { - b0 = gen_rtx_EXPR_LIST (VOIDmode, b0, const0_rtx); - b1 = gen_rtx_EXPR_LIST (VOIDmode, b1, const0_rtx); - rtx par = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, b0, b1)); - retval = chkp_join_splitted_slot (retval, par); - } - else - { - retval = gen_rtx_PARALLEL (VOIDmode, - gen_rtvec (3, retval, b0, b1)); - chkp_put_regs_to_expr_list (retval); - } - } - - call = gen_rtx_SET (retval, call); - } + call = gen_rtx_SET (retval, call); vec[vec_len++] = call; if (pop) @@ -27939,11 +28077,11 @@ if (vec_len > 1) call = gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (vec_len, vec)); - call = emit_call_insn (call); + rtx_insn *call_insn = emit_call_insn (call); if (use) - CALL_INSN_FUNCTION_USAGE (call) = use; - - return call; + CALL_INSN_FUNCTION_USAGE (call_insn) = use; + + return call_insn; } /* Return true if the function being called was marked with attribute @@ -27972,12 +28110,354 @@ return false; } +/* Output indirect branch via a call and return thunk. CALL_OP is a + register which contains the branch target. XASM is the assembly + template for CALL_OP. Branch is a tail call if SIBCALL_P is true. + A normal call is converted to: + + call __x86_indirect_thunk_reg + + and a tail call is converted to: + + jmp __x86_indirect_thunk_reg + */ + +static void +ix86_output_indirect_branch_via_reg (rtx call_op, bool sibcall_p) +{ + char thunk_name_buf[32]; + char *thunk_name; + enum indirect_thunk_prefix need_prefix + = indirect_thunk_need_prefix (current_output_insn); + int regno = REGNO (call_op); + + if (cfun->machine->indirect_branch_type + != indirect_branch_thunk_inline) + { + if (cfun->machine->indirect_branch_type == indirect_branch_thunk) + { + int i = regno; + if (i >= FIRST_REX_INT_REG) + i -= (FIRST_REX_INT_REG - LAST_INT_REG - 1); + indirect_thunks_used |= 1 << i; + } + indirect_thunk_name (thunk_name_buf, regno, need_prefix, false); + thunk_name = thunk_name_buf; + } + else + thunk_name = NULL; + + if (sibcall_p) + { + if (thunk_name != NULL) + fprintf (asm_out_file, "\tjmp\t%s\n", thunk_name); + else + output_indirect_thunk (regno); + } + else + { + if (thunk_name != NULL) + { + fprintf (asm_out_file, "\tcall\t%s\n", thunk_name); + return; + } + + char indirectlabel1[32]; + char indirectlabel2[32]; + + ASM_GENERATE_INTERNAL_LABEL (indirectlabel1, + INDIRECT_LABEL, + indirectlabelno++); + ASM_GENERATE_INTERNAL_LABEL (indirectlabel2, + INDIRECT_LABEL, + indirectlabelno++); + + /* Jump. */ + fputs ("\tjmp\t", asm_out_file); + assemble_name_raw (asm_out_file, indirectlabel2); + fputc ('\n', asm_out_file); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1); + + if (thunk_name != NULL) + fprintf (asm_out_file, "\tjmp\t%s\n", thunk_name); + else + output_indirect_thunk (regno); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2); + + /* Call. */ + fputs ("\tcall\t", asm_out_file); + assemble_name_raw (asm_out_file, indirectlabel1); + fputc ('\n', asm_out_file); + } +} + +/* Output indirect branch via a call and return thunk. CALL_OP is + the branch target. XASM is the assembly template for CALL_OP. + Branch is a tail call if SIBCALL_P is true. A normal call is + converted to: + + jmp L2 + L1: + push CALL_OP + jmp __x86_indirect_thunk + L2: + call L1 + + and a tail call is converted to: + + push CALL_OP + jmp __x86_indirect_thunk + */ + +static void +ix86_output_indirect_branch_via_push (rtx call_op, const char *xasm, + bool sibcall_p) +{ + char thunk_name_buf[32]; + char *thunk_name; + char push_buf[64]; + enum indirect_thunk_prefix need_prefix + = indirect_thunk_need_prefix (current_output_insn); + int regno = -1; + + if (cfun->machine->indirect_branch_type + != indirect_branch_thunk_inline) + { + if (cfun->machine->indirect_branch_type == indirect_branch_thunk) + indirect_thunk_needed = true; + indirect_thunk_name (thunk_name_buf, regno, need_prefix, false); + thunk_name = thunk_name_buf; + } + else + thunk_name = NULL; + + snprintf (push_buf, sizeof (push_buf), "push{%c}\t%s", + TARGET_64BIT ? 'q' : 'l', xasm); + + if (sibcall_p) + { + output_asm_insn (push_buf, &call_op); + if (thunk_name != NULL) + fprintf (asm_out_file, "\tjmp\t%s\n", thunk_name); + else + output_indirect_thunk (regno); + } + else + { + char indirectlabel1[32]; + char indirectlabel2[32]; + + ASM_GENERATE_INTERNAL_LABEL (indirectlabel1, + INDIRECT_LABEL, + indirectlabelno++); + ASM_GENERATE_INTERNAL_LABEL (indirectlabel2, + INDIRECT_LABEL, + indirectlabelno++); + + /* Jump. */ + fputs ("\tjmp\t", asm_out_file); + assemble_name_raw (asm_out_file, indirectlabel2); + fputc ('\n', asm_out_file); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel1); + + /* An external function may be called via GOT, instead of PLT. */ + if (MEM_P (call_op)) + { + struct ix86_address parts; + rtx addr = XEXP (call_op, 0); + if (ix86_decompose_address (addr, &parts) + && parts.base == stack_pointer_rtx) + { + /* Since call will adjust stack by -UNITS_PER_WORD, + we must convert "disp(stack, index, scale)" to + "disp+UNITS_PER_WORD(stack, index, scale)". */ + if (parts.index) + { + addr = gen_rtx_MULT (Pmode, parts.index, + GEN_INT (parts.scale)); + addr = gen_rtx_PLUS (Pmode, stack_pointer_rtx, + addr); + } + else + addr = stack_pointer_rtx; + + rtx disp; + if (parts.disp != NULL_RTX) + disp = plus_constant (Pmode, parts.disp, + UNITS_PER_WORD); + else + disp = GEN_INT (UNITS_PER_WORD); + + addr = gen_rtx_PLUS (Pmode, addr, disp); + call_op = gen_rtx_MEM (GET_MODE (call_op), addr); + } + } + + output_asm_insn (push_buf, &call_op); + + if (thunk_name != NULL) + fprintf (asm_out_file, "\tjmp\t%s\n", thunk_name); + else + output_indirect_thunk (regno); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, indirectlabel2); + + /* Call. */ + fputs ("\tcall\t", asm_out_file); + assemble_name_raw (asm_out_file, indirectlabel1); + fputc ('\n', asm_out_file); + } +} + +/* Output indirect branch via a call and return thunk. CALL_OP is + the branch target. XASM is the assembly template for CALL_OP. + Branch is a tail call if SIBCALL_P is true. */ + +static void +ix86_output_indirect_branch (rtx call_op, const char *xasm, + bool sibcall_p) +{ + if (REG_P (call_op)) + ix86_output_indirect_branch_via_reg (call_op, sibcall_p); + else + ix86_output_indirect_branch_via_push (call_op, xasm, sibcall_p); +} + +/* Output indirect jump. CALL_OP is the jump target. */ + +const char * +ix86_output_indirect_jmp (rtx call_op) +{ + if (cfun->machine->indirect_branch_type != indirect_branch_keep) + { + /* We can't have red-zone since "call" in the indirect thunk + pushes the return address onto stack, destroying red-zone. */ + if (ix86_red_zone_size != 0) + gcc_unreachable (); + + ix86_output_indirect_branch (call_op, "%0", true); + return ""; + } + else + return "%!jmp\t%A0"; +} + +/* Output function return. CALL_OP is the jump target. Add a REP + prefix to RET if LONG_P is true and function return is kept. */ + +const char * +ix86_output_function_return (bool long_p) +{ + if (cfun->machine->function_return_type != indirect_branch_keep) + { + char thunk_name[32]; + enum indirect_thunk_prefix need_prefix + = indirect_thunk_need_prefix (current_output_insn); + + if (cfun->machine->function_return_type + != indirect_branch_thunk_inline) + { + bool need_thunk = (cfun->machine->function_return_type + == indirect_branch_thunk); + indirect_thunk_name (thunk_name, INVALID_REGNUM, need_prefix, + true); + indirect_return_needed |= need_thunk; + fprintf (asm_out_file, "\tjmp\t%s\n", thunk_name); + } + else + output_indirect_thunk (INVALID_REGNUM); + + return ""; + } + + if (!long_p) + return "%!ret"; + + return "rep%; ret"; +} + +/* Output indirect function return. RET_OP is the function return + target. */ + +const char * +ix86_output_indirect_function_return (rtx ret_op) +{ + if (cfun->machine->function_return_type != indirect_branch_keep) + { + char thunk_name[32]; + enum indirect_thunk_prefix need_prefix + = indirect_thunk_need_prefix (current_output_insn); + unsigned int regno = REGNO (ret_op); + gcc_assert (regno == CX_REG); + + if (cfun->machine->function_return_type + != indirect_branch_thunk_inline) + { + bool need_thunk = (cfun->machine->function_return_type + == indirect_branch_thunk); + indirect_thunk_name (thunk_name, regno, need_prefix, true); + + if (need_thunk) + { + indirect_return_via_cx = true; + indirect_thunks_used |= 1 << CX_REG; + } + fprintf (asm_out_file, "\tjmp\t%s\n", thunk_name); + } + else + output_indirect_thunk (regno); + + return ""; + } + else + return "%!jmp\t%A0"; +} + +/* Split simple return with popping POPC bytes from stack to indirect + branch with stack adjustment . */ + +void +ix86_split_simple_return_pop_internal (rtx popc) +{ + struct machine_function *m = cfun->machine; + rtx ecx = gen_rtx_REG (SImode, CX_REG); + rtx_insn *insn; + + /* There is no "pascal" calling convention in any 64bit ABI. */ + gcc_assert (!TARGET_64BIT); + + insn = emit_insn (gen_pop (ecx)); + m->fs.cfa_offset -= UNITS_PER_WORD; + m->fs.sp_offset -= UNITS_PER_WORD; + + rtx x = plus_constant (Pmode, stack_pointer_rtx, UNITS_PER_WORD); + x = gen_rtx_SET (stack_pointer_rtx, x); + add_reg_note (insn, REG_CFA_ADJUST_CFA, x); + add_reg_note (insn, REG_CFA_REGISTER, gen_rtx_SET (ecx, pc_rtx)); + RTX_FRAME_RELATED_P (insn) = 1; + + x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, popc); + x = gen_rtx_SET (stack_pointer_rtx, x); + insn = emit_insn (x); + add_reg_note (insn, REG_CFA_ADJUST_CFA, x); + RTX_FRAME_RELATED_P (insn) = 1; + + /* Now return address is in ECX. */ + emit_jump_insn (gen_simple_return_indirect_internal (ecx)); +} + /* Output the assembly for a call instruction. */ const char * ix86_output_call_insn (rtx_insn *insn, rtx call_op) { bool direct_p = constant_call_address_operand (call_op, VOIDmode); + bool output_indirect_p + = (!TARGET_SEH + && cfun->machine->indirect_branch_type != indirect_branch_keep); bool seh_nop_p = false; const char *xasm; @@ -27987,10 +28467,21 @@ { if (ix86_nopic_noplt_attribute_p (call_op)) { + direct_p = false; if (TARGET_64BIT) - xasm = "%!jmp\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}"; + { + if (output_indirect_p) + xasm = "{%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}"; + else + xasm = "%!jmp\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}"; + } else - xasm = "%!jmp\t{*%p0@GOT|[DWORD PTR %p0@GOT]}"; + { + if (output_indirect_p) + xasm = "{%p0@GOT|[DWORD PTR %p0@GOT]}"; + else + xasm = "%!jmp\t{*%p0@GOT|[DWORD PTR %p0@GOT]}"; + } } else xasm = "%!jmp\t%P0"; @@ -28000,9 +28491,17 @@ else if (TARGET_SEH) xasm = "%!rex.W jmp\t%A0"; else - xasm = "%!jmp\t%A0"; - - output_asm_insn (xasm, &call_op); + { + if (output_indirect_p) + xasm = "%0"; + else + xasm = "%!jmp\t%A0"; + } + + if (output_indirect_p && !direct_p) + ix86_output_indirect_branch (call_op, xasm, true); + else + output_asm_insn (xasm, &call_op); return ""; } @@ -28014,6 +28513,14 @@ for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i)) { + /* Prevent a catch region from being adjacent to a jump that would + be interpreted as an epilogue sequence by the unwinder. */ + if (JUMP_P(i) && CROSSING_JUMP_P (i)) + { + seh_nop_p = true; + break; + } + /* If we get to another real insn, we don't need the nop. */ if (INSN_P (i)) break; @@ -28040,18 +28547,37 @@ { if (ix86_nopic_noplt_attribute_p (call_op)) { + direct_p = false; if (TARGET_64BIT) - xasm = "%!call\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}"; - else - xasm = "%!call\t{*%p0@GOT|[DWORD PTR %p0@GOT]}"; + { + if (output_indirect_p) + xasm = "{%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}"; + else + xasm = "%!call\t{*%p0@GOTPCREL(%%rip)|[QWORD PTR %p0@GOTPCREL[rip]]}"; + } + else + { + if (output_indirect_p) + xasm = "{%p0@GOT|[DWORD PTR %p0@GOT]}"; + else + xasm = "%!call\t{*%p0@GOT|[DWORD PTR %p0@GOT]}"; + } } else xasm = "%!call\t%P0"; } else - xasm = "%!call\t%A0"; - - output_asm_insn (xasm, &call_op); + { + if (output_indirect_p) + xasm = "%0"; + else + xasm = "%!call\t%A0"; + } + + if (output_indirect_p && !direct_p) + ix86_output_indirect_branch (call_op, xasm, false); + else + output_asm_insn (xasm, &call_op); if (seh_nop_p) return "nop"; @@ -28111,100 +28637,6 @@ instantiate_decl_rtl (s->rtl); } -/* Return the number used for encoding REG, in the range 0..7. */ - -static int -reg_encoded_number (rtx reg) -{ - unsigned regno = REGNO (reg); - switch (regno) - { - case AX_REG: - return 0; - case CX_REG: - return 1; - case DX_REG: - return 2; - case BX_REG: - return 3; - case SP_REG: - return 4; - case BP_REG: - return 5; - case SI_REG: - return 6; - case DI_REG: - return 7; - default: - break; - } - if (IN_RANGE (regno, FIRST_STACK_REG, LAST_STACK_REG)) - return regno - FIRST_STACK_REG; - if (IN_RANGE (regno, FIRST_SSE_REG, LAST_SSE_REG)) - return regno - FIRST_SSE_REG; - if (IN_RANGE (regno, FIRST_MMX_REG, LAST_MMX_REG)) - return regno - FIRST_MMX_REG; - if (IN_RANGE (regno, FIRST_REX_SSE_REG, LAST_REX_SSE_REG)) - return regno - FIRST_REX_SSE_REG; - if (IN_RANGE (regno, FIRST_REX_INT_REG, LAST_REX_INT_REG)) - return regno - FIRST_REX_INT_REG; - if (IN_RANGE (regno, FIRST_MASK_REG, LAST_MASK_REG)) - return regno - FIRST_MASK_REG; - if (IN_RANGE (regno, FIRST_BND_REG, LAST_BND_REG)) - return regno - FIRST_BND_REG; - return -1; -} - -/* Given an insn INSN with NOPERANDS OPERANDS, return the modr/m byte used - in its encoding if it could be relevant for ROP mitigation, otherwise - return -1. If POPNO0 and POPNO1 are nonnull, store the operand numbers - used for calculating it into them. */ - -static int -ix86_get_modrm_for_rop (rtx_insn *insn, rtx *operands, int noperands, - int *popno0 = 0, int *popno1 = 0) -{ - if (asm_noperands (PATTERN (insn)) >= 0) - return -1; - int has_modrm = get_attr_modrm (insn); - if (!has_modrm) - return -1; - enum attr_modrm_class cls = get_attr_modrm_class (insn); - rtx op0, op1; - switch (cls) - { - case MODRM_CLASS_OP02: - gcc_assert (noperands >= 3); - if (popno0) - { - *popno0 = 0; - *popno1 = 2; - } - op0 = operands[0]; - op1 = operands[2]; - break; - case MODRM_CLASS_OP01: - gcc_assert (noperands >= 2); - if (popno0) - { - *popno0 = 0; - *popno1 = 1; - } - op0 = operands[0]; - op1 = operands[1]; - break; - default: - return -1; - } - if (REG_P (op0) && REG_P (op1)) - { - int enc0 = reg_encoded_number (op0); - int enc1 = reg_encoded_number (op1); - return 0xc0 + (enc1 << 3) + enc0; - } - return -1; -} - /* Check whether x86 address PARTS is a pc-relative address. */ bool @@ -28591,16 +29023,6 @@ rtx set; rtx tmp; - /* Add anti dependencies for bounds stores. */ - if (INSN_P (insn) - && GET_CODE (PATTERN (insn)) == PARALLEL - && GET_CODE (XVECEXP (PATTERN (insn), 0, 0)) == UNSPEC - && XINT (XVECEXP (PATTERN (insn), 0, 0), 1) == UNSPEC_BNDSTX) - { - add_dependence (first_arg, insn, REG_DEP_ANTI); - return; - } - set = single_set (insn); if (!set) return; @@ -28732,6 +29154,7 @@ case PROCESSOR_NEHALEM: case PROCESSOR_SANDYBRIDGE: case PROCESSOR_HASWELL: + case PROCESSOR_GENERIC: /* Do not perform multipass scheduling for pre-reload schedule to save compile time. */ if (reload_completed) @@ -28784,6 +29207,46 @@ return align; } +/* Implement TARGET_EMPTY_RECORD_P. */ + +static bool +ix86_is_empty_record (const_tree type) +{ + if (!TARGET_64BIT) + return false; + return default_is_empty_record (type); +} + +/* Implement TARGET_WARN_PARAMETER_PASSING_ABI. */ + +static void +ix86_warn_parameter_passing_abi (cumulative_args_t cum_v, tree type) +{ + CUMULATIVE_ARGS *cum = get_cumulative_args (cum_v); + + if (!cum->warn_empty) + return; + + if (!TYPE_EMPTY_P (type)) + return; + + const_tree ctx = get_ultimate_context (cum->decl); + if (ctx != NULL_TREE + && !TRANSLATION_UNIT_WARN_EMPTY_P (ctx)) + return; + + /* If the actual size of the type is zero, then there is no change + in how objects of this size are passed. */ + if (int_size_in_bytes (type) == 0) + return; + + warning (OPT_Wabi, "empty class %qT parameter passing ABI " + "changes in -fabi-version=12 (GCC 8)", type); + + /* Only warn once. */ + cum->warn_empty = false; +} + /* Compute the alignment for a variable for Intel MCU psABI. TYPE is the data type, and ALIGN is the alignment that the object would ordinarily have. */ @@ -29075,12 +29538,6 @@ { unsigned regno; - /* While this function won't be called by the middle-end when a static - chain isn't needed, it's also used throughout the backend so it's - easiest to keep this check centralized. */ - if (DECL_P (fndecl_or_type) && !DECL_STATIC_CHAIN (fndecl_or_type)) - return NULL; - if (TARGET_64BIT) { /* We always use R10 in 64-bit mode. */ @@ -29161,6 +29618,7 @@ rtx mem, fnaddr; int opcode; int offset = 0; + bool need_endbr = (flag_cf_protection & CF_BRANCH); fnaddr = XEXP (DECL_RTL (fndecl), 0); @@ -29168,6 +29626,14 @@ { int size; + if (need_endbr) + { + /* Insert ENDBR64. */ + mem = adjust_address (m_tramp, SImode, offset); + emit_move_insn (mem, gen_int_mode (0xfa1e0ff3, SImode)); + offset += 4; + } + /* Load the function address to r11. Try to load address using the shorter movl instead of movabs. We may want to support movq for kernel mode, but kernel does not use trampolines at @@ -29245,6 +29711,14 @@ else opcode = 0x68; + if (need_endbr) + { + /* Insert ENDBR32. */ + mem = adjust_address (m_tramp, SImode, offset); + emit_move_insn (mem, gen_int_mode (0xfb1e0ff3, SImode)); + offset += 4; + } + mem = adjust_address (m_tramp, QImode, offset); emit_move_insn (mem, gen_int_mode (opcode, QImode)); @@ -29410,7 +29884,11 @@ IX86_BUILTIN_CLFLUSH, IX86_BUILTIN_MONITOR, IX86_BUILTIN_MWAIT, + IX86_BUILTIN_UMONITOR, + IX86_BUILTIN_UMWAIT, + IX86_BUILTIN_TPAUSE, IX86_BUILTIN_CLZERO, + IX86_BUILTIN_CLDEMOTE, IX86_BUILTIN_VEC_INIT_V2SI, IX86_BUILTIN_VEC_INIT_V4HI, IX86_BUILTIN_VEC_INIT_V8QI, @@ -29634,20 +30112,20 @@ { ix86_builtins_isa[(int) code].isa = mask; - /* OPTION_MASK_ISA_AVX512VL has special meaning. Despite of generic case, - where any bit set means that built-in is enable, this bit must be *and-ed* - with another one. E.g.: OPTION_MASK_ISA_AVX512DQ | OPTION_MASK_ISA_AVX512VL - means that *both* cpuid bits must be set for the built-in to be available. - Handle this here. */ - if (mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512VL) + mask &= ~OPTION_MASK_ISA_64BIT; + + /* Filter out the masks most often ored together with others. */ + if ((mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512VL) + && mask != OPTION_MASK_ISA_AVX512VL) mask &= ~OPTION_MASK_ISA_AVX512VL; - - mask &= ~OPTION_MASK_ISA_64BIT; + if ((mask & ix86_isa_flags & OPTION_MASK_ISA_AVX512BW) + && mask != OPTION_MASK_ISA_AVX512BW) + mask &= ~OPTION_MASK_ISA_AVX512BW; + if (mask == 0 || (mask & ix86_isa_flags) != 0 || (lang_hooks.builtin_function == lang_hooks.builtin_function_ext_scope)) - { tree type = ix86_get_builtin_func_type (tcode); decl = add_builtin_function (name, type, code, BUILT_IN_MD, @@ -29760,21 +30238,6 @@ return decl; } -/* Like def_builtin, but also marks the function decl "pure". */ - -static inline tree -def_builtin_pure2 (HOST_WIDE_INT mask, const char *name, - enum ix86_builtin_func_type tcode, enum ix86_builtins code) -{ - tree decl = def_builtin2 (mask, name, tcode, code); - if (decl) - DECL_PURE_P (decl) = 1; - else - ix86_builtins_isa[(int) code].pure_p = true; - - return decl; -} - /* Add any new builtin functions for a given ISA that may not have been declared. This saves a bit of space compared to adding all of the declarations to the tree, even if we didn't use them. */ @@ -29782,6 +30245,8 @@ static void ix86_add_new_builtins (HOST_WIDE_INT isa, HOST_WIDE_INT isa2) { + isa &= ~OPTION_MASK_ISA_64BIT; + if ((isa & deferred_isa_values) == 0 && (isa2 & deferred_isa_values2) == 0) return; @@ -30039,12 +30504,10 @@ IX86_BUILTIN__BDESC_ARGS_LAST, 1); BDESC_VERIFYS (IX86_BUILTIN__BDESC_ARGS2_FIRST, IX86_BUILTIN__BDESC_ROUND_ARGS_LAST, 1); -BDESC_VERIFYS (IX86_BUILTIN__BDESC_MPX_FIRST, +BDESC_VERIFYS (IX86_BUILTIN__BDESC_SPECIAL_ARGS2_FIRST, IX86_BUILTIN__BDESC_ARGS2_LAST, 1); -BDESC_VERIFYS (IX86_BUILTIN__BDESC_MPX_CONST_FIRST, - IX86_BUILTIN__BDESC_MPX_LAST, 1); BDESC_VERIFYS (IX86_BUILTIN__BDESC_MULTI_ARG_FIRST, - IX86_BUILTIN__BDESC_MPX_CONST_LAST, 1); + IX86_BUILTIN__BDESC_SPECIAL_ARGS2_LAST, 1); BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_FIRST, IX86_BUILTIN__BDESC_MULTI_ARG_LAST, 1); BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_NORMAL_FIRST, @@ -30079,6 +30542,22 @@ IX86_BUILTIN__BDESC_SPECIAL_ARGS_FIRST, ARRAY_SIZE (bdesc_special_args) - 1); + /* Add all special builtins with variable number of operands. */ + for (i = 0, d = bdesc_special_args2; + i < ARRAY_SIZE (bdesc_special_args2); + i++, d++) + { + BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_SPECIAL_ARGS2_FIRST, i); + if (d->name == 0) + continue; + + ftype = (enum ix86_builtin_func_type) d->flag; + def_builtin2 (d->mask, d->name, ftype, d->code); + } + BDESC_VERIFYS (IX86_BUILTIN__BDESC_SPECIAL_ARGS2_LAST, + IX86_BUILTIN__BDESC_SPECIAL_ARGS2_FIRST, + ARRAY_SIZE (bdesc_special_args2) - 1); + /* Add all builtins with variable number of operands. */ for (i = 0, d = bdesc_args; i < ARRAY_SIZE (bdesc_args); @@ -30100,13 +30579,17 @@ i < ARRAY_SIZE (bdesc_args2); i++, d++) { + BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_ARGS2_FIRST, i); if (d->name == 0) continue; ftype = (enum ix86_builtin_func_type) d->flag; def_builtin_const2 (d->mask, d->name, ftype, d->code); } - + BDESC_VERIFYS (IX86_BUILTIN__BDESC_ARGS2_LAST, + IX86_BUILTIN__BDESC_ARGS2_FIRST, + ARRAY_SIZE (bdesc_args2) - 1); + /* Add all builtins with rounding. */ for (i = 0, d = bdesc_round_args; i < ARRAY_SIZE (bdesc_round_args); @@ -30198,21 +30681,28 @@ VOID_FTYPE_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAIT); /* AES */ - def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenc128", + def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, + "__builtin_ia32_aesenc128", V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESENC128); - def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesenclast128", + def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, + "__builtin_ia32_aesenclast128", V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESENCLAST128); - def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdec128", + def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, + "__builtin_ia32_aesdec128", V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESDEC128); - def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesdeclast128", + def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, + "__builtin_ia32_aesdeclast128", V2DI_FTYPE_V2DI_V2DI, IX86_BUILTIN_AESDECLAST128); - def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aesimc128", + def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, + "__builtin_ia32_aesimc128", V2DI_FTYPE_V2DI, IX86_BUILTIN_AESIMC128); - def_builtin_const (OPTION_MASK_ISA_AES, "__builtin_ia32_aeskeygenassist128", + def_builtin_const (OPTION_MASK_ISA_AES | OPTION_MASK_ISA_SSE2, + "__builtin_ia32_aeskeygenassist128", V2DI_FTYPE_V2DI_INT, IX86_BUILTIN_AESKEYGENASSIST128); /* PCLMUL */ - def_builtin_const (OPTION_MASK_ISA_PCLMUL, "__builtin_ia32_pclmulqdq128", + def_builtin_const (OPTION_MASK_ISA_PCLMUL | OPTION_MASK_ISA_SSE2, + "__builtin_ia32_pclmulqdq128", V2DI_FTYPE_V2DI_V2DI_INT, IX86_BUILTIN_PCLMULQDQ128); /* RDRND */ @@ -30677,14 +31167,20 @@ IX86_BUILTIN_SBB64); /* Read/write FLAGS. */ - def_builtin (0, "__builtin_ia32_readeflags_u32", - UNSIGNED_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS); - def_builtin (OPTION_MASK_ISA_64BIT, "__builtin_ia32_readeflags_u64", - UINT64_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS); - def_builtin (0, "__builtin_ia32_writeeflags_u32", - VOID_FTYPE_UNSIGNED, IX86_BUILTIN_WRITE_FLAGS); - def_builtin (OPTION_MASK_ISA_64BIT, "__builtin_ia32_writeeflags_u64", - VOID_FTYPE_UINT64, IX86_BUILTIN_WRITE_FLAGS); + if (TARGET_64BIT) + { + def_builtin (OPTION_MASK_ISA_64BIT, "__builtin_ia32_readeflags_u64", + UINT64_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS); + def_builtin (OPTION_MASK_ISA_64BIT, "__builtin_ia32_writeeflags_u64", + VOID_FTYPE_UINT64, IX86_BUILTIN_WRITE_FLAGS); + } + else + { + def_builtin (0, "__builtin_ia32_readeflags_u32", + UNSIGNED_FTYPE_VOID, IX86_BUILTIN_READ_FLAGS); + def_builtin (0, "__builtin_ia32_writeeflags_u32", + VOID_FTYPE_UNSIGNED, IX86_BUILTIN_WRITE_FLAGS); + } /* CLFLUSHOPT. */ def_builtin (OPTION_MASK_ISA_CLFLUSHOPT, "__builtin_ia32_clflushopt", @@ -30695,14 +31191,26 @@ VOID_FTYPE_PCVOID, IX86_BUILTIN_CLWB); /* MONITORX and MWAITX. */ - def_builtin (OPTION_MASK_ISA_MWAITX, "__builtin_ia32_monitorx", - VOID_FTYPE_PCVOID_UNSIGNED_UNSIGNED, IX86_BUILTIN_MONITORX); - def_builtin (OPTION_MASK_ISA_MWAITX, "__builtin_ia32_mwaitx", - VOID_FTYPE_UNSIGNED_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAITX); + def_builtin2 (OPTION_MASK_ISA_MWAITX, "__builtin_ia32_monitorx", + VOID_FTYPE_PCVOID_UNSIGNED_UNSIGNED, IX86_BUILTIN_MONITORX); + def_builtin2 (OPTION_MASK_ISA_MWAITX, "__builtin_ia32_mwaitx", + VOID_FTYPE_UNSIGNED_UNSIGNED_UNSIGNED, IX86_BUILTIN_MWAITX); /* CLZERO. */ - def_builtin (OPTION_MASK_ISA_CLZERO, "__builtin_ia32_clzero", - VOID_FTYPE_PCVOID, IX86_BUILTIN_CLZERO); + def_builtin2 (OPTION_MASK_ISA_CLZERO, "__builtin_ia32_clzero", + VOID_FTYPE_PCVOID, IX86_BUILTIN_CLZERO); + + /* WAITPKG. */ + def_builtin2 (OPTION_MASK_ISA_WAITPKG, "__builtin_ia32_umonitor", + VOID_FTYPE_PVOID, IX86_BUILTIN_UMONITOR); + def_builtin2 (OPTION_MASK_ISA_WAITPKG, "__builtin_ia32_umwait", + UINT8_FTYPE_UNSIGNED_UINT64, IX86_BUILTIN_UMWAIT); + def_builtin2 (OPTION_MASK_ISA_WAITPKG, "__builtin_ia32_tpause", + UINT8_FTYPE_UNSIGNED_UINT64, IX86_BUILTIN_TPAUSE); + + /* CLDEMOTE. */ + def_builtin2 (OPTION_MASK_ISA_CLDEMOTE, "__builtin_ia32_cldemote", + VOID_FTYPE_PCVOID, IX86_BUILTIN_CLDEMOTE); /* Add FMA4 multi-arg argument instructions */ for (i = 0, d = bdesc_multi_arg; i < ARRAY_SIZE (bdesc_multi_arg); i++, d++) @@ -30726,7 +31234,7 @@ continue; ftype = (enum ix86_builtin_func_type) d->flag; - def_builtin2 (d->mask, d->name, ftype, d->code); + def_builtin (d->mask, d->name, ftype, d->code); } BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_LAST, IX86_BUILTIN__BDESC_CET_FIRST, @@ -30741,81 +31249,13 @@ continue; ftype = (enum ix86_builtin_func_type) d->flag; - def_builtin2 (d->mask, d->name, ftype, d->code); + def_builtin (d->mask, d->name, ftype, d->code); } BDESC_VERIFYS (IX86_BUILTIN__BDESC_CET_NORMAL_LAST, IX86_BUILTIN__BDESC_CET_NORMAL_FIRST, ARRAY_SIZE (bdesc_cet_rdssp) - 1); } -static void -ix86_init_mpx_builtins () -{ - const struct builtin_description * d; - enum ix86_builtin_func_type ftype; - tree decl; - size_t i; - - for (i = 0, d = bdesc_mpx; - i < ARRAY_SIZE (bdesc_mpx); - i++, d++) - { - BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_MPX_FIRST, i); - if (d->name == 0) - continue; - - ftype = (enum ix86_builtin_func_type) d->flag; - decl = def_builtin (d->mask, d->name, ftype, d->code); - - /* With no leaf and nothrow flags for MPX builtins - abnormal edges may follow its call when setjmp - presents in the function. Since we may have a lot - of MPX builtins calls it causes lots of useless - edges and enormous PHI nodes. To avoid this we mark - MPX builtins as leaf and nothrow. */ - if (decl) - { - DECL_ATTRIBUTES (decl) = build_tree_list (get_identifier ("leaf"), - NULL_TREE); - TREE_NOTHROW (decl) = 1; - } - else - { - ix86_builtins_isa[(int)d->code].leaf_p = true; - ix86_builtins_isa[(int)d->code].nothrow_p = true; - } - } - BDESC_VERIFYS (IX86_BUILTIN__BDESC_MPX_LAST, - IX86_BUILTIN__BDESC_MPX_FIRST, - ARRAY_SIZE (bdesc_mpx) - 1); - - for (i = 0, d = bdesc_mpx_const; - i < ARRAY_SIZE (bdesc_mpx_const); - i++, d++) - { - BDESC_VERIFY (d->code, IX86_BUILTIN__BDESC_MPX_CONST_FIRST, i); - if (d->name == 0) - continue; - - ftype = (enum ix86_builtin_func_type) d->flag; - decl = def_builtin_const (d->mask, d->name, ftype, d->code); - - if (decl) - { - DECL_ATTRIBUTES (decl) = build_tree_list (get_identifier ("leaf"), - NULL_TREE); - TREE_NOTHROW (decl) = 1; - } - else - { - ix86_builtins_isa[(int)d->code].leaf_p = true; - ix86_builtins_isa[(int)d->code].nothrow_p = true; - } - } - BDESC_VERIFYS (IX86_BUILTIN__BDESC_MPX_CONST_LAST, - IX86_BUILTIN__BDESC_MPX_CONST_FIRST, - ARRAY_SIZE (bdesc_mpx_const) - 1); -} #undef BDESC_VERIFY #undef BDESC_VERIFYS @@ -31051,6 +31491,8 @@ &global_options_set); gcc_assert (target_node); + if (target_node == error_mark_node) + return 0; new_target = TREE_TARGET_OPTION (target_node); gcc_assert (new_target); @@ -31085,16 +31527,32 @@ priority = P_PROC_AVX; break; case PROCESSOR_HASWELL: - if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_AVX512VL) - arg_str = "skylake-avx512"; - else if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_XSAVES) - arg_str = "skylake"; - else if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_ADX) + if (new_target->x_ix86_isa_flags & OPTION_MASK_ISA_ADX) arg_str = "broadwell"; else arg_str = "haswell"; priority = P_PROC_AVX2; break; + case PROCESSOR_SKYLAKE: + arg_str = "skylake"; + priority = P_PROC_AVX2; + break; + case PROCESSOR_SKYLAKE_AVX512: + arg_str = "skylake-avx512"; + priority = P_PROC_AVX512F; + break; + case PROCESSOR_CANNONLAKE: + arg_str = "cannonlake"; + priority = P_PROC_AVX512F; + break; + case PROCESSOR_ICELAKE_CLIENT: + arg_str = "icelake-client"; + priority = P_PROC_AVX512F; + break; + case PROCESSOR_ICELAKE_SERVER: + arg_str = "icelake-server"; + priority = P_PROC_AVX512F; + break; case PROCESSOR_BONNELL: arg_str = "bonnell"; priority = P_PROC_SSSE3; @@ -31111,6 +31569,18 @@ arg_str = "silvermont"; priority = P_PROC_SSE4_2; break; + case PROCESSOR_GOLDMONT: + arg_str = "goldmont"; + priority = P_PROC_SSE4_2; + break; + case PROCESSOR_GOLDMONT_PLUS: + arg_str = "goldmont-plus"; + priority = P_PROC_SSE4_2; + break; + case PROCESSOR_TREMONT: + arg_str = "tremont"; + priority = P_PROC_SSE4_2; + break; case PROCESSOR_AMDFAM10: arg_str = "amdfam10h"; priority = P_PROC_SSE4_A; @@ -31769,6 +32239,11 @@ F_AVX5124VNNIW, F_AVX5124FMAPS, F_AVX512VPOPCNTDQ, + F_AVX512VBMI2, + F_GFNI, + F_VPCLMULQDQ, + F_AVX512VNNI, + F_AVX512BITALG, F_MAX }; @@ -31791,6 +32266,9 @@ M_AMD_BTVER2, M_AMDFAM17H, M_INTEL_KNM, + M_INTEL_GOLDMONT, + M_INTEL_GOLDMONT_PLUS, + M_INTEL_TREMONT, M_CPU_SUBTYPE_START, M_INTEL_COREI7_NEHALEM, M_INTEL_COREI7_WESTMERE, @@ -31807,7 +32285,10 @@ M_INTEL_COREI7_HASWELL, M_INTEL_COREI7_BROADWELL, M_INTEL_COREI7_SKYLAKE, - M_INTEL_COREI7_SKYLAKE_AVX512 + M_INTEL_COREI7_SKYLAKE_AVX512, + M_INTEL_COREI7_CANNONLAKE, + M_INTEL_COREI7_ICELAKE_CLIENT, + M_INTEL_COREI7_ICELAKE_SERVER }; static struct _arch_names_table @@ -31831,8 +32312,14 @@ {"broadwell", M_INTEL_COREI7_BROADWELL}, {"skylake", M_INTEL_COREI7_SKYLAKE}, {"skylake-avx512", M_INTEL_COREI7_SKYLAKE_AVX512}, + {"cannonlake", M_INTEL_COREI7_CANNONLAKE}, + {"icelake-client", M_INTEL_COREI7_ICELAKE_CLIENT}, + {"icelake-server", M_INTEL_COREI7_ICELAKE_SERVER}, {"bonnell", M_INTEL_BONNELL}, {"silvermont", M_INTEL_SILVERMONT}, + {"goldmont", M_INTEL_GOLDMONT}, + {"goldmont-plus", M_INTEL_GOLDMONT_PLUS}, + {"tremont", M_INTEL_TREMONT}, {"knl", M_INTEL_KNL}, {"knm", M_INTEL_KNM}, {"amdfam10h", M_AMDFAM10H}, @@ -31887,7 +32374,12 @@ {"avx512ifma",F_AVX512IFMA}, {"avx5124vnniw",F_AVX5124VNNIW}, {"avx5124fmaps",F_AVX5124FMAPS}, - {"avx512vpopcntdq",F_AVX512VPOPCNTDQ} + {"avx512vpopcntdq",F_AVX512VPOPCNTDQ}, + {"avx512vbmi2", F_AVX512VBMI2}, + {"gfni", F_GFNI}, + {"vpclmulqdq", F_VPCLMULQDQ}, + {"avx512vnni", F_AVX512VNNI}, + {"avx512bitalg", F_AVX512BITALG} }; tree __processor_model_type = build_processor_model_struct (); @@ -31956,8 +32448,8 @@ } /* Get the appropriate field in __cpu_model. */ - ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var, - field, NULL_TREE); + ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var, + field, NULL_TREE); /* Check the value. */ final = build2 (EQ_EXPR, unsigned_type_node, ref, @@ -31987,20 +32479,34 @@ return integer_zero_node; } + if (isa_names_table[i].feature >= 32) + { + tree __cpu_features2_var = make_var_decl (unsigned_type_node, + "__cpu_features2"); + + varpool_node::add (__cpu_features2_var); + field_val = (1U << (isa_names_table[i].feature - 32)); + /* Return __cpu_features2 & field_val */ + final = build2 (BIT_AND_EXPR, unsigned_type_node, + __cpu_features2_var, + build_int_cstu (unsigned_type_node, field_val)); + return build1 (CONVERT_EXPR, integer_type_node, final); + } + field = TYPE_FIELDS (__processor_model_type); /* Get the last field, which is __cpu_features. */ while (DECL_CHAIN (field)) field = DECL_CHAIN (field); /* Get the appropriate field: __cpu_model.__cpu_features */ - ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var, - field, NULL_TREE); + ref = build3 (COMPONENT_REF, TREE_TYPE (field), __cpu_model_var, + field, NULL_TREE); /* Access the 0th element of __cpu_features array. */ array_elt = build4 (ARRAY_REF, unsigned_type_node, ref, integer_zero_node, NULL_TREE, NULL_TREE); - field_val = (1 << isa_names_table[i].feature); + field_val = (1U << isa_names_table[i].feature); /* Return __cpu_model.__cpu_features[0] & field_val */ final = build2 (BIT_AND_EXPR, unsigned_type_node, array_elt, build_int_cstu (unsigned_type_node, field_val)); @@ -32009,6 +32515,28 @@ gcc_unreachable (); } +/* Return the shift count of a vector by scalar shift builtin second argument + ARG1. */ +static tree +ix86_vector_shift_count (tree arg1) +{ + if (tree_fits_uhwi_p (arg1)) + return arg1; + else if (TREE_CODE (arg1) == VECTOR_CST && CHAR_BIT == 8) + { + /* The count argument is weird, passed in as various 128-bit + (or 64-bit) vectors, the low 64 bits from it are the count. */ + unsigned char buf[16]; + int len = native_encode_expr (arg1, buf, 16); + if (len == 0) + return NULL_TREE; + tree t = native_interpret_expr (uint64_type_node, buf, len); + if (t && tree_fits_uhwi_p (t)) + return t; + } + return NULL_TREE; +} + static tree ix86_fold_builtin (tree fndecl, int n_args, tree *args, bool ignore ATTRIBUTE_UNUSED) @@ -32017,6 +32545,10 @@ { enum ix86_builtins fn_code = (enum ix86_builtins) DECL_FUNCTION_CODE (fndecl); + enum rtx_code rcode; + bool is_vshift; + unsigned HOST_WIDE_INT mask; + switch (fn_code) { case IX86_BUILTIN_CPU_IS: @@ -32166,6 +32698,293 @@ } break; + case IX86_BUILTIN_MOVMSKPS: + case IX86_BUILTIN_PMOVMSKB: + case IX86_BUILTIN_MOVMSKPD: + case IX86_BUILTIN_PMOVMSKB128: + case IX86_BUILTIN_MOVMSKPD256: + case IX86_BUILTIN_MOVMSKPS256: + case IX86_BUILTIN_PMOVMSKB256: + gcc_assert (n_args == 1); + if (TREE_CODE (args[0]) == VECTOR_CST) + { + HOST_WIDE_INT res = 0; + for (unsigned i = 0; i < VECTOR_CST_NELTS (args[0]); ++i) + { + tree e = VECTOR_CST_ELT (args[0], i); + if (TREE_CODE (e) == INTEGER_CST && !TREE_OVERFLOW (e)) + { + if (wi::neg_p (wi::to_wide (e))) + res |= HOST_WIDE_INT_1 << i; + } + else if (TREE_CODE (e) == REAL_CST && !TREE_OVERFLOW (e)) + { + if (TREE_REAL_CST (e).sign) + res |= HOST_WIDE_INT_1 << i; + } + else + return NULL_TREE; + } + return build_int_cst (TREE_TYPE (TREE_TYPE (fndecl)), res); + } + break; + + case IX86_BUILTIN_PSLLD: + case IX86_BUILTIN_PSLLD128: + case IX86_BUILTIN_PSLLD128_MASK: + case IX86_BUILTIN_PSLLD256: + case IX86_BUILTIN_PSLLD256_MASK: + case IX86_BUILTIN_PSLLD512: + case IX86_BUILTIN_PSLLDI: + case IX86_BUILTIN_PSLLDI128: + case IX86_BUILTIN_PSLLDI128_MASK: + case IX86_BUILTIN_PSLLDI256: + case IX86_BUILTIN_PSLLDI256_MASK: + case IX86_BUILTIN_PSLLDI512: + case IX86_BUILTIN_PSLLQ: + case IX86_BUILTIN_PSLLQ128: + case IX86_BUILTIN_PSLLQ128_MASK: + case IX86_BUILTIN_PSLLQ256: + case IX86_BUILTIN_PSLLQ256_MASK: + case IX86_BUILTIN_PSLLQ512: + case IX86_BUILTIN_PSLLQI: + case IX86_BUILTIN_PSLLQI128: + case IX86_BUILTIN_PSLLQI128_MASK: + case IX86_BUILTIN_PSLLQI256: + case IX86_BUILTIN_PSLLQI256_MASK: + case IX86_BUILTIN_PSLLQI512: + case IX86_BUILTIN_PSLLW: + case IX86_BUILTIN_PSLLW128: + case IX86_BUILTIN_PSLLW128_MASK: + case IX86_BUILTIN_PSLLW256: + case IX86_BUILTIN_PSLLW256_MASK: + case IX86_BUILTIN_PSLLW512_MASK: + case IX86_BUILTIN_PSLLWI: + case IX86_BUILTIN_PSLLWI128: + case IX86_BUILTIN_PSLLWI128_MASK: + case IX86_BUILTIN_PSLLWI256: + case IX86_BUILTIN_PSLLWI256_MASK: + case IX86_BUILTIN_PSLLWI512_MASK: + rcode = ASHIFT; + is_vshift = false; + goto do_shift; + case IX86_BUILTIN_PSRAD: + case IX86_BUILTIN_PSRAD128: + case IX86_BUILTIN_PSRAD128_MASK: + case IX86_BUILTIN_PSRAD256: + case IX86_BUILTIN_PSRAD256_MASK: + case IX86_BUILTIN_PSRAD512: + case IX86_BUILTIN_PSRADI: + case IX86_BUILTIN_PSRADI128: + case IX86_BUILTIN_PSRADI128_MASK: + case IX86_BUILTIN_PSRADI256: + case IX86_BUILTIN_PSRADI256_MASK: + case IX86_BUILTIN_PSRADI512: + case IX86_BUILTIN_PSRAQ128_MASK: + case IX86_BUILTIN_PSRAQ256_MASK: + case IX86_BUILTIN_PSRAQ512: + case IX86_BUILTIN_PSRAQI128_MASK: + case IX86_BUILTIN_PSRAQI256_MASK: + case IX86_BUILTIN_PSRAQI512: + case IX86_BUILTIN_PSRAW: + case IX86_BUILTIN_PSRAW128: + case IX86_BUILTIN_PSRAW128_MASK: + case IX86_BUILTIN_PSRAW256: + case IX86_BUILTIN_PSRAW256_MASK: + case IX86_BUILTIN_PSRAW512: + case IX86_BUILTIN_PSRAWI: + case IX86_BUILTIN_PSRAWI128: + case IX86_BUILTIN_PSRAWI128_MASK: + case IX86_BUILTIN_PSRAWI256: + case IX86_BUILTIN_PSRAWI256_MASK: + case IX86_BUILTIN_PSRAWI512: + rcode = ASHIFTRT; + is_vshift = false; + goto do_shift; + case IX86_BUILTIN_PSRLD: + case IX86_BUILTIN_PSRLD128: + case IX86_BUILTIN_PSRLD128_MASK: + case IX86_BUILTIN_PSRLD256: + case IX86_BUILTIN_PSRLD256_MASK: + case IX86_BUILTIN_PSRLD512: + case IX86_BUILTIN_PSRLDI: + case IX86_BUILTIN_PSRLDI128: + case IX86_BUILTIN_PSRLDI128_MASK: + case IX86_BUILTIN_PSRLDI256: + case IX86_BUILTIN_PSRLDI256_MASK: + case IX86_BUILTIN_PSRLDI512: + case IX86_BUILTIN_PSRLQ: + case IX86_BUILTIN_PSRLQ128: + case IX86_BUILTIN_PSRLQ128_MASK: + case IX86_BUILTIN_PSRLQ256: + case IX86_BUILTIN_PSRLQ256_MASK: + case IX86_BUILTIN_PSRLQ512: + case IX86_BUILTIN_PSRLQI: + case IX86_BUILTIN_PSRLQI128: + case IX86_BUILTIN_PSRLQI128_MASK: + case IX86_BUILTIN_PSRLQI256: + case IX86_BUILTIN_PSRLQI256_MASK: + case IX86_BUILTIN_PSRLQI512: + case IX86_BUILTIN_PSRLW: + case IX86_BUILTIN_PSRLW128: + case IX86_BUILTIN_PSRLW128_MASK: + case IX86_BUILTIN_PSRLW256: + case IX86_BUILTIN_PSRLW256_MASK: + case IX86_BUILTIN_PSRLW512: + case IX86_BUILTIN_PSRLWI: + case IX86_BUILTIN_PSRLWI128: + case IX86_BUILTIN_PSRLWI128_MASK: + case IX86_BUILTIN_PSRLWI256: + case IX86_BUILTIN_PSRLWI256_MASK: + case IX86_BUILTIN_PSRLWI512: + rcode = LSHIFTRT; + is_vshift = false; + goto do_shift; + case IX86_BUILTIN_PSLLVV16HI: + case IX86_BUILTIN_PSLLVV16SI: + case IX86_BUILTIN_PSLLVV2DI: + case IX86_BUILTIN_PSLLVV2DI_MASK: + case IX86_BUILTIN_PSLLVV32HI: + case IX86_BUILTIN_PSLLVV4DI: + case IX86_BUILTIN_PSLLVV4DI_MASK: + case IX86_BUILTIN_PSLLVV4SI: + case IX86_BUILTIN_PSLLVV4SI_MASK: + case IX86_BUILTIN_PSLLVV8DI: + case IX86_BUILTIN_PSLLVV8HI: + case IX86_BUILTIN_PSLLVV8SI: + case IX86_BUILTIN_PSLLVV8SI_MASK: + rcode = ASHIFT; + is_vshift = true; + goto do_shift; + case IX86_BUILTIN_PSRAVQ128: + case IX86_BUILTIN_PSRAVQ256: + case IX86_BUILTIN_PSRAVV16HI: + case IX86_BUILTIN_PSRAVV16SI: + case IX86_BUILTIN_PSRAVV32HI: + case IX86_BUILTIN_PSRAVV4SI: + case IX86_BUILTIN_PSRAVV4SI_MASK: + case IX86_BUILTIN_PSRAVV8DI: + case IX86_BUILTIN_PSRAVV8HI: + case IX86_BUILTIN_PSRAVV8SI: + case IX86_BUILTIN_PSRAVV8SI_MASK: + rcode = ASHIFTRT; + is_vshift = true; + goto do_shift; + case IX86_BUILTIN_PSRLVV16HI: + case IX86_BUILTIN_PSRLVV16SI: + case IX86_BUILTIN_PSRLVV2DI: + case IX86_BUILTIN_PSRLVV2DI_MASK: + case IX86_BUILTIN_PSRLVV32HI: + case IX86_BUILTIN_PSRLVV4DI: + case IX86_BUILTIN_PSRLVV4DI_MASK: + case IX86_BUILTIN_PSRLVV4SI: + case IX86_BUILTIN_PSRLVV4SI_MASK: + case IX86_BUILTIN_PSRLVV8DI: + case IX86_BUILTIN_PSRLVV8HI: + case IX86_BUILTIN_PSRLVV8SI: + case IX86_BUILTIN_PSRLVV8SI_MASK: + rcode = LSHIFTRT; + is_vshift = true; + goto do_shift; + + do_shift: + gcc_assert (n_args >= 2); + if (TREE_CODE (args[0]) != VECTOR_CST) + break; + mask = HOST_WIDE_INT_M1U; + if (n_args > 2) + { + /* This is masked shift. */ + if (!tree_fits_uhwi_p (args[n_args - 1]) + || TREE_SIDE_EFFECTS (args[n_args - 2])) + break; + mask = tree_to_uhwi (args[n_args - 1]); + unsigned elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (args[0])); + mask |= HOST_WIDE_INT_M1U << elems; + if (mask != HOST_WIDE_INT_M1U + && TREE_CODE (args[n_args - 2]) != VECTOR_CST) + break; + if (mask == (HOST_WIDE_INT_M1U << elems)) + return args[n_args - 2]; + } + if (is_vshift && TREE_CODE (args[1]) != VECTOR_CST) + break; + if (tree tem = (is_vshift ? integer_one_node + : ix86_vector_shift_count (args[1]))) + { + unsigned HOST_WIDE_INT count = tree_to_uhwi (tem); + unsigned HOST_WIDE_INT prec + = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (args[0]))); + if (count == 0 && mask == HOST_WIDE_INT_M1U) + return args[0]; + if (count >= prec) + { + if (rcode == ASHIFTRT) + count = prec - 1; + else if (mask == HOST_WIDE_INT_M1U) + return build_zero_cst (TREE_TYPE (args[0])); + } + tree countt = NULL_TREE; + if (!is_vshift) + { + if (count >= prec) + countt = integer_zero_node; + else + countt = build_int_cst (integer_type_node, count); + } + tree_vector_builder builder; + builder.new_unary_operation (TREE_TYPE (args[0]), args[0], + false); + unsigned int cnt = builder.encoded_nelts (); + for (unsigned int i = 0; i < cnt; ++i) + { + tree elt = VECTOR_CST_ELT (args[0], i); + if (TREE_CODE (elt) != INTEGER_CST || TREE_OVERFLOW (elt)) + return NULL_TREE; + tree type = TREE_TYPE (elt); + if (rcode == LSHIFTRT) + elt = fold_convert (unsigned_type_for (type), elt); + if (is_vshift) + { + countt = VECTOR_CST_ELT (args[1], i); + if (TREE_CODE (countt) != INTEGER_CST + || TREE_OVERFLOW (countt)) + return NULL_TREE; + if (wi::neg_p (wi::to_wide (countt)) + || wi::to_widest (countt) >= prec) + { + if (rcode == ASHIFTRT) + countt = build_int_cst (TREE_TYPE (countt), + prec - 1); + else + { + elt = build_zero_cst (TREE_TYPE (elt)); + countt = build_zero_cst (TREE_TYPE (countt)); + } + } + } + else if (count >= prec) + elt = build_zero_cst (TREE_TYPE (elt)); + elt = const_binop (rcode == ASHIFT + ? LSHIFT_EXPR : RSHIFT_EXPR, + TREE_TYPE (elt), elt, countt); + if (!elt || TREE_CODE (elt) != INTEGER_CST) + return NULL_TREE; + if (rcode == LSHIFTRT) + elt = fold_convert (type, elt); + if ((mask & (HOST_WIDE_INT_1U << i)) == 0) + { + elt = VECTOR_CST_ELT (args[n_args - 2], i); + if (TREE_CODE (elt) != INTEGER_CST + || TREE_OVERFLOW (elt)) + return NULL_TREE; + } + builder.quick_push (elt); + } + return builder.build (); + } + break; + default: break; } @@ -32186,11 +33005,14 @@ { gimple *stmt = gsi_stmt (*gsi); tree fndecl = gimple_call_fndecl (stmt); - gcc_checking_assert (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD); + gcc_checking_assert (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_MD)); int n_args = gimple_call_num_args (stmt); enum ix86_builtins fn_code = (enum ix86_builtins) DECL_FUNCTION_CODE (fndecl); tree decl = NULL_TREE; tree arg0, arg1; + enum rtx_code rcode; + unsigned HOST_WIDE_INT count; + bool is_vshift; switch (fn_code) { @@ -32270,6 +33092,227 @@ } break; + case IX86_BUILTIN_PSLLD: + case IX86_BUILTIN_PSLLD128: + case IX86_BUILTIN_PSLLD128_MASK: + case IX86_BUILTIN_PSLLD256: + case IX86_BUILTIN_PSLLD256_MASK: + case IX86_BUILTIN_PSLLD512: + case IX86_BUILTIN_PSLLDI: + case IX86_BUILTIN_PSLLDI128: + case IX86_BUILTIN_PSLLDI128_MASK: + case IX86_BUILTIN_PSLLDI256: + case IX86_BUILTIN_PSLLDI256_MASK: + case IX86_BUILTIN_PSLLDI512: + case IX86_BUILTIN_PSLLQ: + case IX86_BUILTIN_PSLLQ128: + case IX86_BUILTIN_PSLLQ128_MASK: + case IX86_BUILTIN_PSLLQ256: + case IX86_BUILTIN_PSLLQ256_MASK: + case IX86_BUILTIN_PSLLQ512: + case IX86_BUILTIN_PSLLQI: + case IX86_BUILTIN_PSLLQI128: + case IX86_BUILTIN_PSLLQI128_MASK: + case IX86_BUILTIN_PSLLQI256: + case IX86_BUILTIN_PSLLQI256_MASK: + case IX86_BUILTIN_PSLLQI512: + case IX86_BUILTIN_PSLLW: + case IX86_BUILTIN_PSLLW128: + case IX86_BUILTIN_PSLLW128_MASK: + case IX86_BUILTIN_PSLLW256: + case IX86_BUILTIN_PSLLW256_MASK: + case IX86_BUILTIN_PSLLW512_MASK: + case IX86_BUILTIN_PSLLWI: + case IX86_BUILTIN_PSLLWI128: + case IX86_BUILTIN_PSLLWI128_MASK: + case IX86_BUILTIN_PSLLWI256: + case IX86_BUILTIN_PSLLWI256_MASK: + case IX86_BUILTIN_PSLLWI512_MASK: + rcode = ASHIFT; + is_vshift = false; + goto do_shift; + case IX86_BUILTIN_PSRAD: + case IX86_BUILTIN_PSRAD128: + case IX86_BUILTIN_PSRAD128_MASK: + case IX86_BUILTIN_PSRAD256: + case IX86_BUILTIN_PSRAD256_MASK: + case IX86_BUILTIN_PSRAD512: + case IX86_BUILTIN_PSRADI: + case IX86_BUILTIN_PSRADI128: + case IX86_BUILTIN_PSRADI128_MASK: + case IX86_BUILTIN_PSRADI256: + case IX86_BUILTIN_PSRADI256_MASK: + case IX86_BUILTIN_PSRADI512: + case IX86_BUILTIN_PSRAQ128_MASK: + case IX86_BUILTIN_PSRAQ256_MASK: + case IX86_BUILTIN_PSRAQ512: + case IX86_BUILTIN_PSRAQI128_MASK: + case IX86_BUILTIN_PSRAQI256_MASK: + case IX86_BUILTIN_PSRAQI512: + case IX86_BUILTIN_PSRAW: + case IX86_BUILTIN_PSRAW128: + case IX86_BUILTIN_PSRAW128_MASK: + case IX86_BUILTIN_PSRAW256: + case IX86_BUILTIN_PSRAW256_MASK: + case IX86_BUILTIN_PSRAW512: + case IX86_BUILTIN_PSRAWI: + case IX86_BUILTIN_PSRAWI128: + case IX86_BUILTIN_PSRAWI128_MASK: + case IX86_BUILTIN_PSRAWI256: + case IX86_BUILTIN_PSRAWI256_MASK: + case IX86_BUILTIN_PSRAWI512: + rcode = ASHIFTRT; + is_vshift = false; + goto do_shift; + case IX86_BUILTIN_PSRLD: + case IX86_BUILTIN_PSRLD128: + case IX86_BUILTIN_PSRLD128_MASK: + case IX86_BUILTIN_PSRLD256: + case IX86_BUILTIN_PSRLD256_MASK: + case IX86_BUILTIN_PSRLD512: + case IX86_BUILTIN_PSRLDI: + case IX86_BUILTIN_PSRLDI128: + case IX86_BUILTIN_PSRLDI128_MASK: + case IX86_BUILTIN_PSRLDI256: + case IX86_BUILTIN_PSRLDI256_MASK: + case IX86_BUILTIN_PSRLDI512: + case IX86_BUILTIN_PSRLQ: + case IX86_BUILTIN_PSRLQ128: + case IX86_BUILTIN_PSRLQ128_MASK: + case IX86_BUILTIN_PSRLQ256: + case IX86_BUILTIN_PSRLQ256_MASK: + case IX86_BUILTIN_PSRLQ512: + case IX86_BUILTIN_PSRLQI: + case IX86_BUILTIN_PSRLQI128: + case IX86_BUILTIN_PSRLQI128_MASK: + case IX86_BUILTIN_PSRLQI256: + case IX86_BUILTIN_PSRLQI256_MASK: + case IX86_BUILTIN_PSRLQI512: + case IX86_BUILTIN_PSRLW: + case IX86_BUILTIN_PSRLW128: + case IX86_BUILTIN_PSRLW128_MASK: + case IX86_BUILTIN_PSRLW256: + case IX86_BUILTIN_PSRLW256_MASK: + case IX86_BUILTIN_PSRLW512: + case IX86_BUILTIN_PSRLWI: + case IX86_BUILTIN_PSRLWI128: + case IX86_BUILTIN_PSRLWI128_MASK: + case IX86_BUILTIN_PSRLWI256: + case IX86_BUILTIN_PSRLWI256_MASK: + case IX86_BUILTIN_PSRLWI512: + rcode = LSHIFTRT; + is_vshift = false; + goto do_shift; + case IX86_BUILTIN_PSLLVV16HI: + case IX86_BUILTIN_PSLLVV16SI: + case IX86_BUILTIN_PSLLVV2DI: + case IX86_BUILTIN_PSLLVV2DI_MASK: + case IX86_BUILTIN_PSLLVV32HI: + case IX86_BUILTIN_PSLLVV4DI: + case IX86_BUILTIN_PSLLVV4DI_MASK: + case IX86_BUILTIN_PSLLVV4SI: + case IX86_BUILTIN_PSLLVV4SI_MASK: + case IX86_BUILTIN_PSLLVV8DI: + case IX86_BUILTIN_PSLLVV8HI: + case IX86_BUILTIN_PSLLVV8SI: + case IX86_BUILTIN_PSLLVV8SI_MASK: + rcode = ASHIFT; + is_vshift = true; + goto do_shift; + case IX86_BUILTIN_PSRAVQ128: + case IX86_BUILTIN_PSRAVQ256: + case IX86_BUILTIN_PSRAVV16HI: + case IX86_BUILTIN_PSRAVV16SI: + case IX86_BUILTIN_PSRAVV32HI: + case IX86_BUILTIN_PSRAVV4SI: + case IX86_BUILTIN_PSRAVV4SI_MASK: + case IX86_BUILTIN_PSRAVV8DI: + case IX86_BUILTIN_PSRAVV8HI: + case IX86_BUILTIN_PSRAVV8SI: + case IX86_BUILTIN_PSRAVV8SI_MASK: + rcode = ASHIFTRT; + is_vshift = true; + goto do_shift; + case IX86_BUILTIN_PSRLVV16HI: + case IX86_BUILTIN_PSRLVV16SI: + case IX86_BUILTIN_PSRLVV2DI: + case IX86_BUILTIN_PSRLVV2DI_MASK: + case IX86_BUILTIN_PSRLVV32HI: + case IX86_BUILTIN_PSRLVV4DI: + case IX86_BUILTIN_PSRLVV4DI_MASK: + case IX86_BUILTIN_PSRLVV4SI: + case IX86_BUILTIN_PSRLVV4SI_MASK: + case IX86_BUILTIN_PSRLVV8DI: + case IX86_BUILTIN_PSRLVV8HI: + case IX86_BUILTIN_PSRLVV8SI: + case IX86_BUILTIN_PSRLVV8SI_MASK: + rcode = LSHIFTRT; + is_vshift = true; + goto do_shift; + + do_shift: + gcc_assert (n_args >= 2); + arg0 = gimple_call_arg (stmt, 0); + arg1 = gimple_call_arg (stmt, 1); + if (n_args > 2) + { + /* This is masked shift. Only optimize if the mask is all ones. */ + tree argl = gimple_call_arg (stmt, n_args - 1); + if (!tree_fits_uhwi_p (argl)) + break; + unsigned HOST_WIDE_INT mask = tree_to_uhwi (argl); + unsigned elems = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0)); + if ((mask | (HOST_WIDE_INT_M1U << elems)) != HOST_WIDE_INT_M1U) + break; + } + if (is_vshift) + { + if (TREE_CODE (arg1) != VECTOR_CST) + break; + count = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (arg0))); + if (integer_zerop (arg1)) + count = 0; + else if (rcode == ASHIFTRT) + break; + else + for (unsigned int i = 0; i < VECTOR_CST_NELTS (arg1); ++i) + { + tree elt = VECTOR_CST_ELT (arg1, i); + if (!wi::neg_p (wi::to_wide (elt)) + && wi::to_widest (elt) < count) + return false; + } + } + else + { + arg1 = ix86_vector_shift_count (arg1); + if (!arg1) + break; + count = tree_to_uhwi (arg1); + } + if (count == 0) + { + /* Just return the first argument for shift by 0. */ + location_t loc = gimple_location (stmt); + gimple *g = gimple_build_assign (gimple_call_lhs (stmt), arg0); + gimple_set_location (g, loc); + gsi_replace (gsi, g, false); + return true; + } + if (rcode != ASHIFTRT + && count >= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (arg0)))) + { + /* For shift counts equal or greater than precision, except for + arithmetic right shift the result is zero. */ + location_t loc = gimple_location (stmt); + gimple *g = gimple_build_assign (gimple_call_lhs (stmt), + build_zero_cst (TREE_TYPE (arg0))); + gimple_set_location (g, loc); + gsi_replace (gsi, g, false); + return true; + } + break; + default: break; } @@ -32444,7 +33487,6 @@ ix86_init_tm_builtins (); ix86_init_mmx_sse_builtins (); - ix86_init_mpx_builtins (); if (TARGET_LP64) ix86_init_builtins_va_builtins_abi (); @@ -33368,12 +34410,15 @@ case V16SI_FTYPE_V4SI: case V16SI_FTYPE_V16SF: case V16SI_FTYPE_V16SI: + case V64QI_FTYPE_V64QI: + case V32HI_FTYPE_V32HI: case V16SF_FTYPE_V16SF: case V8DI_FTYPE_UQI: case V8DI_FTYPE_V8DI: case V8DF_FTYPE_V4DF: case V8DF_FTYPE_V2DF: case V8DF_FTYPE_V8DF: + case V4DI_FTYPE_V4DI: nargs = 1; break; case V4SF_FTYPE_V4SF_VEC_MERGE: @@ -33425,6 +34470,7 @@ case V1DI_FTYPE_V2SI_V2SI: case V32QI_FTYPE_V16HI_V16HI: case V16HI_FTYPE_V8SI_V8SI: + case V64QI_FTYPE_V64QI_V64QI: case V32QI_FTYPE_V32QI_V32QI: case V16HI_FTYPE_V32QI_V32QI: case V16HI_FTYPE_V16HI_V16HI: @@ -33672,6 +34718,13 @@ case V8HI_FTYPE_V8DI_V8HI_UQI: case V8SI_FTYPE_V8DI_V8SI_UQI: case V4SI_FTYPE_V4SI_V4SI_V4SI: + case V16SI_FTYPE_V16SI_V16SI_V16SI: + case V8DI_FTYPE_V8DI_V8DI_V8DI: + case V32HI_FTYPE_V32HI_V32HI_V32HI: + case V2DI_FTYPE_V2DI_V2DI_V2DI: + case V16HI_FTYPE_V16HI_V16HI_V16HI: + case V8SI_FTYPE_V8SI_V8SI_V8SI: + case V8HI_FTYPE_V8HI_V8HI_V8HI: nargs = 3; break; case V32QI_FTYPE_V32QI_V32QI_INT: @@ -33699,6 +34752,10 @@ case UQI_FTYPE_V4SF_V4SF_INT: case UHI_FTYPE_V16SI_V16SI_INT: case UHI_FTYPE_V16SF_V16SF_INT: + case V64QI_FTYPE_V64QI_V64QI_INT: + case V32HI_FTYPE_V32HI_V32HI_INT: + case V16SI_FTYPE_V16SI_V16SI_INT: + case V8DI_FTYPE_V8DI_V8DI_INT: nargs = 3; nargs_constant = 1; break; @@ -33738,6 +34795,8 @@ case HI_FTYPE_V16SF_INT_UHI: case QI_FTYPE_V8SF_INT_UQI: case QI_FTYPE_V4SF_INT_UQI: + case V4SI_FTYPE_V4SI_V4SI_UHI: + case V8SI_FTYPE_V8SI_V8SI_UHI: nargs = 3; mask_pos = 1; nargs_constant = 1; @@ -33829,6 +34888,15 @@ case USI_FTYPE_V32HI_V32HI_INT_USI: case UHI_FTYPE_V16HI_V16HI_INT_UHI: case UQI_FTYPE_V8HI_V8HI_INT_UQI: + case V32HI_FTYPE_V32HI_V32HI_V32HI_INT: + case V16HI_FTYPE_V16HI_V16HI_V16HI_INT: + case V8HI_FTYPE_V8HI_V8HI_V8HI_INT: + case V8SI_FTYPE_V8SI_V8SI_V8SI_INT: + case V4DI_FTYPE_V4DI_V4DI_V4DI_INT: + case V8DI_FTYPE_V8DI_V8DI_V8DI_INT: + case V16SI_FTYPE_V16SI_V16SI_V16SI_INT: + case V2DI_FTYPE_V2DI_V2DI_V2DI_INT: + case V4SI_FTYPE_V4SI_V4SI_V4SI_INT: nargs = 4; mask_pos = 1; nargs_constant = 1; @@ -33926,6 +34994,22 @@ mask_pos = 1; nargs_constant = 1; break; + case V64QI_FTYPE_V64QI_V64QI_INT_V64QI_UDI: + case V32QI_FTYPE_V32QI_V32QI_INT_V32QI_USI: + case V16QI_FTYPE_V16QI_V16QI_INT_V16QI_UHI: + case V32HI_FTYPE_V32HI_V32HI_INT_V32HI_INT: + case V16SI_FTYPE_V16SI_V16SI_INT_V16SI_INT: + case V8DI_FTYPE_V8DI_V8DI_INT_V8DI_INT: + case V16HI_FTYPE_V16HI_V16HI_INT_V16HI_INT: + case V8SI_FTYPE_V8SI_V8SI_INT_V8SI_INT: + case V4DI_FTYPE_V4DI_V4DI_INT_V4DI_INT: + case V8HI_FTYPE_V8HI_V8HI_INT_V8HI_INT: + case V4SI_FTYPE_V4SI_V4SI_INT_V4SI_INT: + case V2DI_FTYPE_V2DI_V2DI_INT_V2DI_INT: + nargs = 5; + mask_pos = 1; + nargs_constant = 2; + break; default: gcc_unreachable (); @@ -34565,6 +35649,7 @@ case INT_FTYPE_VOID: case USHORT_FTYPE_VOID: case UINT64_FTYPE_VOID: + case UINT_FTYPE_VOID: case UNSIGNED_FTYPE_VOID: nargs = 0; klass = load; @@ -34611,6 +35696,7 @@ case VOID_FTYPE_PDOUBLE_V2DF: case VOID_FTYPE_PLONGLONG_LONGLONG: case VOID_FTYPE_PULONGLONG_ULONGLONG: + case VOID_FTYPE_PUNSIGNED_UNSIGNED: case VOID_FTYPE_PINT_INT: nargs = 1; klass = store; @@ -34640,6 +35726,12 @@ break; } break; + case VOID_FTYPE_PVOID_PCVOID: + nargs = 1; + klass = store; + memory = 0; + + break; case V4SF_FTYPE_V4SF_PCV2SF: case V2DF_FTYPE_V2DF_PCDOUBLE: nargs = 2; @@ -34671,6 +35763,12 @@ case VOID_FTYPE_PV16SI_V16SI_UHI: case VOID_FTYPE_PV8SI_V8SI_UQI: case VOID_FTYPE_PV4SI_V4SI_UQI: + case VOID_FTYPE_PV64QI_V64QI_UDI: + case VOID_FTYPE_PV32HI_V32HI_USI: + case VOID_FTYPE_PV32QI_V32QI_USI: + case VOID_FTYPE_PV16QI_V16QI_UHI: + case VOID_FTYPE_PV16HI_V16HI_UHI: + case VOID_FTYPE_PV8HI_V8HI_UQI: switch (icode) { /* These builtins and instructions require the memory @@ -34754,6 +35852,12 @@ case V2DI_FTYPE_PCV2DI_V2DI_UQI: case V4DI_FTYPE_PCV4DI_V4DI_UQI: case V8DI_FTYPE_PCV8DI_V8DI_UQI: + case V64QI_FTYPE_PCV64QI_V64QI_UDI: + case V32HI_FTYPE_PCV32HI_V32HI_USI: + case V32QI_FTYPE_PCV32QI_V32QI_USI: + case V16QI_FTYPE_PCV16QI_V16QI_UHI: + case V16HI_FTYPE_PCV16HI_V16HI_UHI: + case V8HI_FTYPE_PCV8HI_V8HI_UQI: switch (icode) { /* These builtins and instructions require the memory @@ -34781,6 +35885,7 @@ default: break; } + /* FALLTHRU */ case V64QI_FTYPE_PCCHAR_V64QI_UDI: case V32QI_FTYPE_PCCHAR_V32QI_USI: case V16QI_FTYPE_PCCHAR_V16QI_UHI: @@ -35063,36 +36168,6 @@ return target; } -/* Emit conditional move of SRC to DST with condition - OP1 CODE OP2. */ -static void -ix86_emit_cmove (rtx dst, rtx src, enum rtx_code code, rtx op1, rtx op2) -{ - rtx t; - - if (TARGET_CMOVE) - { - t = ix86_expand_compare (code, op1, op2); - emit_insn (gen_rtx_SET (dst, gen_rtx_IF_THEN_ELSE (GET_MODE (dst), t, - src, dst))); - } - else - { - rtx_code_label *nomove = gen_label_rtx (); - emit_cmp_and_jump_insns (op1, op2, reverse_condition (code), - const0_rtx, GET_MODE (op1), 1, nomove); - emit_move_insn (dst, src); - emit_label (nomove); - } -} - -/* Choose max of DST and SRC and put it to DST. */ -static void -ix86_emit_move_max (rtx dst, rtx src) -{ - ix86_emit_cmove (dst, src, LTU, dst, src); -} - /* Expand an expression EXP that calls a built-in function, with result going to TARGET if that's convenient (and in mode MODE if that's convenient). @@ -35133,34 +36208,34 @@ } } - /* Determine whether the builtin function is available under the current ISA. - Originally the builtin was not created if it wasn't applicable to the - current ISA based on the command line switches. With function specific - options, we need to check in the context of the function making the call - whether it is supported. Treat AVX512VL and MMX specially. For other flags, - if isa includes more than one ISA bit, treat those are requiring any - of them. For AVX512VL, require both AVX512VL and the non-AVX512VL - ISAs. Likewise for MMX, require both MMX and the non-MMX ISAs. - Similarly for 64BIT, but we shouldn't be building such builtins - at all, -m64 is a whole TU option. */ - if (((ix86_builtins_isa[fcode].isa - & ~(OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_MMX - | OPTION_MASK_ISA_64BIT)) - && !(ix86_builtins_isa[fcode].isa - & ~(OPTION_MASK_ISA_AVX512VL | OPTION_MASK_ISA_MMX - | OPTION_MASK_ISA_64BIT) - & ix86_isa_flags)) - || ((ix86_builtins_isa[fcode].isa & OPTION_MASK_ISA_AVX512VL) - && !(ix86_isa_flags & OPTION_MASK_ISA_AVX512VL)) - || ((ix86_builtins_isa[fcode].isa & OPTION_MASK_ISA_MMX) - && !(ix86_isa_flags & OPTION_MASK_ISA_MMX)) - || (ix86_builtins_isa[fcode].isa2 - && !(ix86_builtins_isa[fcode].isa2 & ix86_isa_flags2))) - { - char *opts = ix86_target_string (ix86_builtins_isa[fcode].isa, - ix86_builtins_isa[fcode].isa2, 0, 0, - NULL, NULL, (enum fpmath_unit) 0, - false); + HOST_WIDE_INT isa = ix86_isa_flags; + HOST_WIDE_INT isa2 = ix86_isa_flags2; + HOST_WIDE_INT bisa = ix86_builtins_isa[fcode].isa; + HOST_WIDE_INT bisa2 = ix86_builtins_isa[fcode].isa2; + /* The general case is we require all the ISAs specified in bisa{,2} + to be enabled. + The exceptions are: + OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A + OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32 + OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4 + where for each this pair it is sufficient if either of the ISAs is + enabled, plus if it is ored with other options also those others. */ + if (((bisa & (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A)) + == (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A)) + && (isa & (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A)) != 0) + isa |= (OPTION_MASK_ISA_SSE | OPTION_MASK_ISA_3DNOW_A); + if (((bisa & (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32)) + == (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32)) + && (isa & (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32)) != 0) + isa |= (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_CRC32); + if (((bisa & (OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4)) + == (OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4)) + && (isa & (OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4)) != 0) + isa |= (OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4); + if ((bisa & isa) != bisa || (bisa2 & isa2) != bisa2) + { + char *opts = ix86_target_string (bisa, bisa2, 0, 0, NULL, NULL, + (enum fpmath_unit) 0, false); if (!opts) error ("%qE needs unknown isa option", fndecl); else @@ -35174,342 +36249,6 @@ switch (fcode) { - case IX86_BUILTIN_BNDMK: - if (!target - || GET_MODE (target) != BNDmode - || !register_operand (target, BNDmode)) - target = gen_reg_rtx (BNDmode); - - arg0 = CALL_EXPR_ARG (exp, 0); - arg1 = CALL_EXPR_ARG (exp, 1); - - op0 = expand_normal (arg0); - op1 = expand_normal (arg1); - - if (!register_operand (op0, Pmode)) - op0 = ix86_zero_extend_to_Pmode (op0); - if (!register_operand (op1, Pmode)) - op1 = ix86_zero_extend_to_Pmode (op1); - - /* Builtin arg1 is size of block but instruction op1 should - be (size - 1). */ - op1 = expand_simple_binop (Pmode, PLUS, op1, constm1_rtx, - NULL_RTX, 1, OPTAB_DIRECT); - - emit_insn (BNDmode == BND64mode - ? gen_bnd64_mk (target, op0, op1) - : gen_bnd32_mk (target, op0, op1)); - return target; - - case IX86_BUILTIN_BNDSTX: - arg0 = CALL_EXPR_ARG (exp, 0); - arg1 = CALL_EXPR_ARG (exp, 1); - arg2 = CALL_EXPR_ARG (exp, 2); - - op0 = expand_normal (arg0); - op1 = expand_normal (arg1); - op2 = expand_normal (arg2); - - if (!register_operand (op0, Pmode)) - op0 = ix86_zero_extend_to_Pmode (op0); - if (!register_operand (op1, BNDmode)) - op1 = copy_to_mode_reg (BNDmode, op1); - if (!register_operand (op2, Pmode)) - op2 = ix86_zero_extend_to_Pmode (op2); - - emit_insn (BNDmode == BND64mode - ? gen_bnd64_stx (op2, op0, op1) - : gen_bnd32_stx (op2, op0, op1)); - return 0; - - case IX86_BUILTIN_BNDLDX: - if (!target - || GET_MODE (target) != BNDmode - || !register_operand (target, BNDmode)) - target = gen_reg_rtx (BNDmode); - - arg0 = CALL_EXPR_ARG (exp, 0); - arg1 = CALL_EXPR_ARG (exp, 1); - - op0 = expand_normal (arg0); - op1 = expand_normal (arg1); - - if (!register_operand (op0, Pmode)) - op0 = ix86_zero_extend_to_Pmode (op0); - if (!register_operand (op1, Pmode)) - op1 = ix86_zero_extend_to_Pmode (op1); - - emit_insn (BNDmode == BND64mode - ? gen_bnd64_ldx (target, op0, op1) - : gen_bnd32_ldx (target, op0, op1)); - return target; - - case IX86_BUILTIN_BNDCL: - arg0 = CALL_EXPR_ARG (exp, 0); - arg1 = CALL_EXPR_ARG (exp, 1); - - op0 = expand_normal (arg0); - op1 = expand_normal (arg1); - - if (!register_operand (op0, Pmode)) - op0 = ix86_zero_extend_to_Pmode (op0); - if (!register_operand (op1, BNDmode)) - op1 = copy_to_mode_reg (BNDmode, op1); - - emit_insn (BNDmode == BND64mode - ? gen_bnd64_cl (op1, op0) - : gen_bnd32_cl (op1, op0)); - return 0; - - case IX86_BUILTIN_BNDCU: - arg0 = CALL_EXPR_ARG (exp, 0); - arg1 = CALL_EXPR_ARG (exp, 1); - - op0 = expand_normal (arg0); - op1 = expand_normal (arg1); - - if (!register_operand (op0, Pmode)) - op0 = ix86_zero_extend_to_Pmode (op0); - if (!register_operand (op1, BNDmode)) - op1 = copy_to_mode_reg (BNDmode, op1); - - emit_insn (BNDmode == BND64mode - ? gen_bnd64_cu (op1, op0) - : gen_bnd32_cu (op1, op0)); - return 0; - - case IX86_BUILTIN_BNDRET: - arg0 = CALL_EXPR_ARG (exp, 0); - target = chkp_get_rtl_bounds (arg0); - - /* If no bounds were specified for returned value, - then use INIT bounds. It usually happens when - some built-in function is expanded. */ - if (!target) - { - rtx t1 = gen_reg_rtx (Pmode); - rtx t2 = gen_reg_rtx (Pmode); - target = gen_reg_rtx (BNDmode); - emit_move_insn (t1, const0_rtx); - emit_move_insn (t2, constm1_rtx); - emit_insn (BNDmode == BND64mode - ? gen_bnd64_mk (target, t1, t2) - : gen_bnd32_mk (target, t1, t2)); - } - - gcc_assert (target && REG_P (target)); - return target; - - case IX86_BUILTIN_BNDNARROW: - { - rtx m1, m1h1, m1h2, lb, ub, t1; - - /* Return value and lb. */ - arg0 = CALL_EXPR_ARG (exp, 0); - /* Bounds. */ - arg1 = CALL_EXPR_ARG (exp, 1); - /* Size. */ - arg2 = CALL_EXPR_ARG (exp, 2); - - lb = expand_normal (arg0); - op1 = expand_normal (arg1); - op2 = expand_normal (arg2); - - /* Size was passed but we need to use (size - 1) as for bndmk. */ - op2 = expand_simple_binop (Pmode, PLUS, op2, constm1_rtx, - NULL_RTX, 1, OPTAB_DIRECT); - - /* Add LB to size and inverse to get UB. */ - op2 = expand_simple_binop (Pmode, PLUS, op2, lb, - op2, 1, OPTAB_DIRECT); - ub = expand_simple_unop (Pmode, NOT, op2, op2, 1); - - if (!register_operand (lb, Pmode)) - lb = ix86_zero_extend_to_Pmode (lb); - if (!register_operand (ub, Pmode)) - ub = ix86_zero_extend_to_Pmode (ub); - - /* We need to move bounds to memory before any computations. */ - if (MEM_P (op1)) - m1 = op1; - else - { - m1 = assign_386_stack_local (BNDmode, SLOT_TEMP); - emit_move_insn (m1, op1); - } - - /* Generate mem expression to be used for access to LB and UB. */ - m1h1 = adjust_address (m1, Pmode, 0); - m1h2 = adjust_address (m1, Pmode, GET_MODE_SIZE (Pmode)); - - t1 = gen_reg_rtx (Pmode); - - /* Compute LB. */ - emit_move_insn (t1, m1h1); - ix86_emit_move_max (t1, lb); - emit_move_insn (m1h1, t1); - - /* Compute UB. UB is stored in 1's complement form. Therefore - we also use max here. */ - emit_move_insn (t1, m1h2); - ix86_emit_move_max (t1, ub); - emit_move_insn (m1h2, t1); - - op2 = gen_reg_rtx (BNDmode); - emit_move_insn (op2, m1); - - return chkp_join_splitted_slot (lb, op2); - } - - case IX86_BUILTIN_BNDINT: - { - rtx res, rh1, rh2, lb1, lb2, ub1, ub2; - - if (!target - || GET_MODE (target) != BNDmode - || !register_operand (target, BNDmode)) - target = gen_reg_rtx (BNDmode); - - arg0 = CALL_EXPR_ARG (exp, 0); - arg1 = CALL_EXPR_ARG (exp, 1); - - op0 = expand_normal (arg0); - op1 = expand_normal (arg1); - - res = assign_386_stack_local (BNDmode, SLOT_TEMP); - rh1 = adjust_address (res, Pmode, 0); - rh2 = adjust_address (res, Pmode, GET_MODE_SIZE (Pmode)); - - /* Put first bounds to temporaries. */ - lb1 = gen_reg_rtx (Pmode); - ub1 = gen_reg_rtx (Pmode); - if (MEM_P (op0)) - { - emit_move_insn (lb1, adjust_address (op0, Pmode, 0)); - emit_move_insn (ub1, adjust_address (op0, Pmode, - GET_MODE_SIZE (Pmode))); - } - else - { - emit_move_insn (res, op0); - emit_move_insn (lb1, rh1); - emit_move_insn (ub1, rh2); - } - - /* Put second bounds to temporaries. */ - lb2 = gen_reg_rtx (Pmode); - ub2 = gen_reg_rtx (Pmode); - if (MEM_P (op1)) - { - emit_move_insn (lb2, adjust_address (op1, Pmode, 0)); - emit_move_insn (ub2, adjust_address (op1, Pmode, - GET_MODE_SIZE (Pmode))); - } - else - { - emit_move_insn (res, op1); - emit_move_insn (lb2, rh1); - emit_move_insn (ub2, rh2); - } - - /* Compute LB. */ - ix86_emit_move_max (lb1, lb2); - emit_move_insn (rh1, lb1); - - /* Compute UB. UB is stored in 1's complement form. Therefore - we also use max here. */ - ix86_emit_move_max (ub1, ub2); - emit_move_insn (rh2, ub1); - - emit_move_insn (target, res); - - return target; - } - - case IX86_BUILTIN_SIZEOF: - { - tree name; - rtx symbol; - - if (!target - || GET_MODE (target) != Pmode - || !register_operand (target, Pmode)) - target = gen_reg_rtx (Pmode); - - arg0 = CALL_EXPR_ARG (exp, 0); - gcc_assert (VAR_P (arg0)); - - name = DECL_ASSEMBLER_NAME (arg0); - symbol = gen_rtx_SYMBOL_REF (Pmode, IDENTIFIER_POINTER (name)); - - emit_insn (Pmode == SImode - ? gen_move_size_reloc_si (target, symbol) - : gen_move_size_reloc_di (target, symbol)); - - return target; - } - - case IX86_BUILTIN_BNDLOWER: - { - rtx mem, hmem; - - if (!target - || GET_MODE (target) != Pmode - || !register_operand (target, Pmode)) - target = gen_reg_rtx (Pmode); - - arg0 = CALL_EXPR_ARG (exp, 0); - op0 = expand_normal (arg0); - - /* We need to move bounds to memory first. */ - if (MEM_P (op0)) - mem = op0; - else - { - mem = assign_386_stack_local (BNDmode, SLOT_TEMP); - emit_move_insn (mem, op0); - } - - /* Generate mem expression to access LB and load it. */ - hmem = adjust_address (mem, Pmode, 0); - emit_move_insn (target, hmem); - - return target; - } - - case IX86_BUILTIN_BNDUPPER: - { - rtx mem, hmem, res; - - if (!target - || GET_MODE (target) != Pmode - || !register_operand (target, Pmode)) - target = gen_reg_rtx (Pmode); - - arg0 = CALL_EXPR_ARG (exp, 0); - op0 = expand_normal (arg0); - - /* We need to move bounds to memory first. */ - if (MEM_P (op0)) - mem = op0; - else - { - mem = assign_386_stack_local (BNDmode, SLOT_TEMP); - emit_move_insn (mem, op0); - } - - /* Generate mem expression to access UB. */ - hmem = adjust_address (mem, Pmode, GET_MODE_SIZE (Pmode)); - - /* We need to inverse all bits of UB. */ - res = expand_simple_unop (Pmode, NOT, hmem, target, 1); - - if (res != target) - emit_move_insn (target, res); - - return target; - } - case IX86_BUILTIN_MASKMOVQ: case IX86_BUILTIN_MASKMOVDQU: icode = (fcode == IX86_BUILTIN_MASKMOVQ @@ -35631,6 +36370,82 @@ emit_insn (gen_mwaitx (op0, op1, op2)); return 0; + case IX86_BUILTIN_UMONITOR: + arg0 = CALL_EXPR_ARG (exp, 0); + op0 = expand_normal (arg0); + + op0 = ix86_zero_extend_to_Pmode (op0); + + insn = (TARGET_64BIT + ? gen_umonitor_di (op0) + : gen_umonitor_si (op0)); + + emit_insn (insn); + return 0; + + case IX86_BUILTIN_UMWAIT: + case IX86_BUILTIN_TPAUSE: + arg0 = CALL_EXPR_ARG (exp, 0); + arg1 = CALL_EXPR_ARG (exp, 1); + op0 = expand_normal (arg0); + op1 = expand_normal (arg1); + + if (!REG_P (op0)) + op0 = copy_to_mode_reg (SImode, op0); + + op1 = force_reg (DImode, op1); + + if (TARGET_64BIT) + { + op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32), + NULL, 1, OPTAB_DIRECT); + switch (fcode) + { + case IX86_BUILTIN_UMWAIT: + icode = CODE_FOR_umwait_rex64; + break; + case IX86_BUILTIN_TPAUSE: + icode = CODE_FOR_tpause_rex64; + break; + default: + gcc_unreachable (); + } + + op2 = gen_lowpart (SImode, op2); + op1 = gen_lowpart (SImode, op1); + pat = GEN_FCN (icode) (op0, op1, op2); + } + else + { + switch (fcode) + { + case IX86_BUILTIN_UMWAIT: + icode = CODE_FOR_umwait; + break; + case IX86_BUILTIN_TPAUSE: + icode = CODE_FOR_tpause; + break; + default: + gcc_unreachable (); + } + pat = GEN_FCN (icode) (op0, op1); + } + + if (!pat) + return 0; + + emit_insn (pat); + + if (target == 0 + || !register_operand (target, QImode)) + target = gen_reg_rtx (QImode); + + pat = gen_rtx_EQ (QImode, gen_rtx_REG (CCCmode, FLAGS_REG), + const0_rtx); + emit_insn (gen_rtx_SET (target, pat)); + + return target; + case IX86_BUILTIN_CLZERO: arg0 = CALL_EXPR_ARG (exp, 0); op0 = expand_normal (arg0); @@ -35639,6 +36454,16 @@ emit_insn (ix86_gen_clzero (op0)); return 0; + case IX86_BUILTIN_CLDEMOTE: + arg0 = CALL_EXPR_ARG (exp, 0); + op0 = expand_normal (arg0); + icode = CODE_FOR_cldemote; + if (!insn_data[icode].operand[0].predicate (op0, Pmode)) + op0 = ix86_zero_extend_to_Pmode (op0); + + emit_insn (gen_cldemote (op0)); + return 0; + case IX86_BUILTIN_VEC_INIT_V2SI: case IX86_BUILTIN_VEC_INIT_V4HI: case IX86_BUILTIN_VEC_INIT_V8QI: @@ -35666,6 +36491,27 @@ case IX86_BUILTIN_NANSQ: return expand_call (exp, target, ignore); + case IX86_BUILTIN_RDPID: + + op0 = gen_reg_rtx (word_mode); + + if (TARGET_64BIT) + { + insn = gen_rdpid_rex64 (op0); + op0 = convert_to_mode (SImode, op0, 1); + } + else + insn = gen_rdpid (op0); + + emit_insn (insn); + + if (target == 0 + || !register_operand (target, SImode)) + target = gen_reg_rtx (SImode); + + emit_move_insn (target, op0); + return target; + case IX86_BUILTIN_RDPMC: case IX86_BUILTIN_RDTSC: case IX86_BUILTIN_RDTSCP: @@ -35724,14 +36570,9 @@ emit_move_insn (gen_rtx_MEM (SImode, op4), op2); } - if (target == 0) - { - /* mode is VOIDmode if __builtin_rd* has been called - without lhs. */ - if (mode == VOIDmode) - return target; - target = gen_reg_rtx (mode); - } + if (target == 0 + || !register_operand (target, DImode)) + target = gen_reg_rtx (DImode); if (TARGET_64BIT) { @@ -35744,6 +36585,27 @@ emit_move_insn (target, op0); return target; + case IX86_BUILTIN_MOVDIR64B: + + arg0 = CALL_EXPR_ARG (exp, 0); + arg1 = CALL_EXPR_ARG (exp, 1); + op0 = expand_normal (arg0); + op1 = expand_normal (arg1); + + op0 = ix86_zero_extend_to_Pmode (op0); + if (!address_operand (op1, VOIDmode)) + { + op1 = convert_memory_address (Pmode, op1); + op1 = copy_addr_to_reg (op1); + } + op1 = gen_rtx_MEM (XImode, op1); + + insn = (TARGET_64BIT + ? gen_movdir64b_di (op0, op1) + : gen_movdir64b_si (op0, op1)); + emit_insn (insn); + return 0; + case IX86_BUILTIN_FXSAVE: case IX86_BUILTIN_FXRSTOR: case IX86_BUILTIN_FXSAVE64: @@ -35799,25 +36661,23 @@ if (!REG_P (op0)) op0 = copy_to_mode_reg (SImode, op0); + op1 = force_reg (DImode, op1); + if (TARGET_64BIT) { op2 = expand_simple_binop (DImode, LSHIFTRT, op1, GEN_INT (32), NULL, 1, OPTAB_DIRECT); + icode = CODE_FOR_xsetbv_rex64; + op2 = gen_lowpart (SImode, op2); op1 = gen_lowpart (SImode, op1); - if (!REG_P (op1)) - op1 = copy_to_mode_reg (SImode, op1); - if (!REG_P (op2)) - op2 = copy_to_mode_reg (SImode, op2); - icode = CODE_FOR_xsetbv_rex64; pat = GEN_FCN (icode) (op0, op1, op2); } else { - if (!REG_P (op1)) - op1 = copy_to_mode_reg (DImode, op1); icode = CODE_FOR_xsetbv; + pat = GEN_FCN (icode) (op0, op1); } if (pat) @@ -36735,7 +37595,7 @@ { tree fndecl = gimple_call_fndecl (def_stmt); if (fndecl - && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD) + && fndecl_built_in_p (fndecl, BUILT_IN_MD)) switch ((unsigned int) DECL_FUNCTION_CODE (fndecl)) { case IX86_BUILTIN_CMPPD: @@ -37016,6 +37876,14 @@ target); } + if (fcode >= IX86_BUILTIN__BDESC_SPECIAL_ARGS2_FIRST + && fcode <= IX86_BUILTIN__BDESC_SPECIAL_ARGS2_LAST) + { + i = fcode - IX86_BUILTIN__BDESC_SPECIAL_ARGS2_FIRST; + return ix86_expand_special_args_builtin (bdesc_special_args2 + i, exp, + target); + } + if (fcode >= IX86_BUILTIN__BDESC_ARGS_FIRST && fcode <= IX86_BUILTIN__BDESC_ARGS_LAST) { @@ -37260,8 +38128,8 @@ } return target; } - case IX86_BUILTIN_RDPID: - return ix86_expand_special_args_builtin (bdesc_args2 + i, exp, + case IX86_BUILTIN_RDPID: + return ix86_expand_special_args_builtin (bdesc_args2 + i, exp, target); default: return ix86_expand_args_builtin (bdesc_args2 + i, exp, target); @@ -37318,7 +38186,7 @@ && fcode <= IX86_BUILTIN__BDESC_CET_NORMAL_LAST) { i = fcode - IX86_BUILTIN__BDESC_CET_NORMAL_FIRST; - return ix86_expand_args_builtin (bdesc_cet_rdssp + i, exp, + return ix86_expand_special_args_builtin (bdesc_cet_rdssp + i, exp, target); } @@ -37351,199 +38219,6 @@ return NULL_TREE; } -/* Return function decl for target specific builtin - for given MPX builtin passed i FCODE. */ -static tree -ix86_builtin_mpx_function (unsigned fcode) -{ - switch (fcode) - { - case BUILT_IN_CHKP_BNDMK: - return ix86_builtins[IX86_BUILTIN_BNDMK]; - - case BUILT_IN_CHKP_BNDSTX: - return ix86_builtins[IX86_BUILTIN_BNDSTX]; - - case BUILT_IN_CHKP_BNDLDX: - return ix86_builtins[IX86_BUILTIN_BNDLDX]; - - case BUILT_IN_CHKP_BNDCL: - return ix86_builtins[IX86_BUILTIN_BNDCL]; - - case BUILT_IN_CHKP_BNDCU: - return ix86_builtins[IX86_BUILTIN_BNDCU]; - - case BUILT_IN_CHKP_BNDRET: - return ix86_builtins[IX86_BUILTIN_BNDRET]; - - case BUILT_IN_CHKP_INTERSECT: - return ix86_builtins[IX86_BUILTIN_BNDINT]; - - case BUILT_IN_CHKP_NARROW: - return ix86_builtins[IX86_BUILTIN_BNDNARROW]; - - case BUILT_IN_CHKP_SIZEOF: - return ix86_builtins[IX86_BUILTIN_SIZEOF]; - - case BUILT_IN_CHKP_EXTRACT_LOWER: - return ix86_builtins[IX86_BUILTIN_BNDLOWER]; - - case BUILT_IN_CHKP_EXTRACT_UPPER: - return ix86_builtins[IX86_BUILTIN_BNDUPPER]; - - default: - return NULL_TREE; - } - - gcc_unreachable (); -} - -/* Helper function for ix86_load_bounds and ix86_store_bounds. - - Return an address to be used to load/store bounds for pointer - passed in SLOT. - - SLOT_NO is an integer constant holding number of a target - dependent special slot to be used in case SLOT is not a memory. - - SPECIAL_BASE is a pointer to be used as a base of fake address - to access special slots in Bounds Table. SPECIAL_BASE[-1], - SPECIAL_BASE[-2] etc. will be used as fake pointer locations. */ - -static rtx -ix86_get_arg_address_for_bt (rtx slot, rtx slot_no, rtx special_base) -{ - rtx addr = NULL; - - /* NULL slot means we pass bounds for pointer not passed to the - function at all. Register slot means we pass pointer in a - register. In both these cases bounds are passed via Bounds - Table. Since we do not have actual pointer stored in memory, - we have to use fake addresses to access Bounds Table. We - start with (special_base - sizeof (void*)) and decrease this - address by pointer size to get addresses for other slots. */ - if (!slot || REG_P (slot)) - { - gcc_assert (CONST_INT_P (slot_no)); - addr = plus_constant (Pmode, special_base, - -(INTVAL (slot_no) + 1) * GET_MODE_SIZE (Pmode)); - } - /* If pointer is passed in a memory then its address is used to - access Bounds Table. */ - else if (MEM_P (slot)) - { - addr = XEXP (slot, 0); - if (!register_operand (addr, Pmode)) - addr = copy_addr_to_reg (addr); - } - else - gcc_unreachable (); - - return addr; -} - -/* Expand pass uses this hook to load bounds for function parameter - PTR passed in SLOT in case its bounds are not passed in a register. - - If SLOT is a memory, then bounds are loaded as for regular pointer - loaded from memory. PTR may be NULL in case SLOT is a memory. - In such case value of PTR (if required) may be loaded from SLOT. - - If SLOT is NULL or a register then SLOT_NO is an integer constant - holding number of the target dependent special slot which should be - used to obtain bounds. - - Return loaded bounds. */ - -static rtx -ix86_load_bounds (rtx slot, rtx ptr, rtx slot_no) -{ - rtx reg = gen_reg_rtx (BNDmode); - rtx addr; - - /* Get address to be used to access Bounds Table. Special slots start - at the location of return address of the current function. */ - addr = ix86_get_arg_address_for_bt (slot, slot_no, arg_pointer_rtx); - - /* Load pointer value from a memory if we don't have it. */ - if (!ptr) - { - gcc_assert (MEM_P (slot)); - ptr = copy_addr_to_reg (slot); - } - - if (!register_operand (ptr, Pmode)) - ptr = ix86_zero_extend_to_Pmode (ptr); - - emit_insn (BNDmode == BND64mode - ? gen_bnd64_ldx (reg, addr, ptr) - : gen_bnd32_ldx (reg, addr, ptr)); - - return reg; -} - -/* Expand pass uses this hook to store BOUNDS for call argument PTR - passed in SLOT in case BOUNDS are not passed in a register. - - If SLOT is a memory, then BOUNDS are stored as for regular pointer - stored in memory. PTR may be NULL in case SLOT is a memory. - In such case value of PTR (if required) may be loaded from SLOT. - - If SLOT is NULL or a register then SLOT_NO is an integer constant - holding number of the target dependent special slot which should be - used to store BOUNDS. */ - -static void -ix86_store_bounds (rtx ptr, rtx slot, rtx bounds, rtx slot_no) -{ - rtx addr; - - /* Get address to be used to access Bounds Table. Special slots start - at the location of return address of a called function. */ - addr = ix86_get_arg_address_for_bt (slot, slot_no, stack_pointer_rtx); - - /* Load pointer value from a memory if we don't have it. */ - if (!ptr) - { - gcc_assert (MEM_P (slot)); - ptr = copy_addr_to_reg (slot); - } - - if (!register_operand (ptr, Pmode)) - ptr = ix86_zero_extend_to_Pmode (ptr); - - gcc_assert (POINTER_BOUNDS_MODE_P (GET_MODE (bounds))); - if (!register_operand (bounds, BNDmode)) - bounds = copy_to_mode_reg (BNDmode, bounds); - - emit_insn (BNDmode == BND64mode - ? gen_bnd64_stx (addr, ptr, bounds) - : gen_bnd32_stx (addr, ptr, bounds)); -} - -/* Load and return bounds returned by function in SLOT. */ - -static rtx -ix86_load_returned_bounds (rtx slot) -{ - rtx res; - - gcc_assert (REG_P (slot)); - res = gen_reg_rtx (BNDmode); - emit_move_insn (res, slot); - - return res; -} - -/* Store BOUNDS returned by function into SLOT. */ - -static void -ix86_store_returned_bounds (rtx slot, rtx bounds) -{ - gcc_assert (REG_P (slot)); - emit_move_insn (slot, bounds); -} - /* Returns a function decl for a vectorized version of the combined function with combined_fn code FN and the result vector type TYPE, or NULL_TREE if it is not available. */ @@ -37983,7 +38658,7 @@ bool si; enum ix86_builtins code; - if (! TARGET_AVX2) + if (! TARGET_AVX2 || !TARGET_USE_GATHER) return NULL_TREE; if ((TREE_CODE (index_type) != INTEGER_TYPE @@ -38341,12 +39016,14 @@ return 1; /* New x86-64 int registers result in bigger code size. Discourage them. */ - if (FIRST_REX_INT_REG <= hard_regno && hard_regno <= LAST_REX_INT_REG) + if (IN_RANGE (hard_regno, FIRST_REX_INT_REG, LAST_REX_INT_REG)) return 2; /* New x86-64 SSE registers result in bigger code size. Discourage them. */ - if (FIRST_REX_SSE_REG <= hard_regno && hard_regno <= LAST_REX_SSE_REG) + if (IN_RANGE (hard_regno, FIRST_REX_SSE_REG, LAST_REX_SSE_REG)) return 2; + if (IN_RANGE (hard_regno, FIRST_EXT_REX_SSE_REG, LAST_EXT_REX_SSE_REG)) + return 1; /* Usage of AX register results in smaller code. Prefer it. */ if (hard_regno == AX_REG) return 4; @@ -38399,10 +39076,6 @@ /* Limit class to FP regs. */ if (FLOAT_CLASS_P (regclass)) return FLOAT_REGS; - else if (regclass == FP_TOP_SSE_REGS) - return FP_TOP_REG; - else if (regclass == FP_SECOND_SSE_REGS) - return FP_SECOND_REG; } return NO_REGS; @@ -38448,14 +39121,7 @@ return MAYBE_SSE_CLASS_P (regclass) ? ALL_SSE_REGS : NO_REGS; if (IS_STACK_MODE (mode)) - { - if (regclass == FP_TOP_SSE_REGS) - return FP_TOP_REG; - else if (regclass == FP_SECOND_SSE_REGS) - return FP_SECOND_REG; - else - return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS; - } + return FLOAT_CLASS_P (regclass) ? regclass : NO_REGS; return regclass; } @@ -38558,7 +39224,6 @@ case SSE_FIRST_REG: case FP_TOP_REG: case FP_SECOND_REG: - case BND_REGS: return true; default: @@ -38968,8 +39633,6 @@ return (VALID_MASK_REG_MODE (mode) || (TARGET_AVX512BW && VALID_MASK_AVX512BW_MODE (mode))); - if (BND_REGNO_P (regno)) - return VALID_BND_REG_MODE (mode); if (SSE_REGNO_P (regno)) { /* We implement the move patterns for all vector modes into and @@ -38986,16 +39649,11 @@ || VALID_AVX512F_SCALAR_MODE (mode))) return true; - /* For AVX-5124FMAPS allow V64SFmode for special regnos. */ + /* For AVX-5124FMAPS or AVX-5124VNNIW + allow V64SF and V64SI modes for special regnos. */ if ((TARGET_AVX5124FMAPS || TARGET_AVX5124VNNIW) - && MOD4_SSE_REGNO_P (regno) - && mode == V64SFmode) - return true; - - /* For AVX-5124VNNIW allow V64SImode for special regnos. */ - if ((TARGET_AVX5124FMAPS || TARGET_AVX5124VNNIW) - && MOD4_SSE_REGNO_P (regno) - && mode == V64SImode) + && (mode == V64SFmode || mode == V64SImode) + && MOD4_SSE_REGNO_P (regno)) return true; /* TODO check for QI/HI scalars. */ @@ -39123,6 +39781,10 @@ /* If MODE2 is only appropriate for an SSE register, then tie with any other mode acceptable to SSE registers. */ + if (GET_MODE_SIZE (mode2) == 64 + && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2)) + return (GET_MODE_SIZE (mode1) == 64 + && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode1)); if (GET_MODE_SIZE (mode2) == 32 && ix86_hard_regno_mode_ok (FIRST_SSE_REG, mode2)) return (GET_MODE_SIZE (mode1) == 32 @@ -39190,17 +39852,14 @@ } /* Return cost of vector operation in MODE given that scalar version has - COST. If PARALLEL is true assume that CPU has more than one unit - performing the operation. */ - -static int -ix86_vec_cost (machine_mode mode, int cost, bool parallel) + COST. */ + +static int +ix86_vec_cost (machine_mode mode, int cost) { if (!VECTOR_MODE_P (mode)) return cost; - - if (!parallel) - return cost * GET_MODE_NUNITS (mode); + if (GET_MODE_BITSIZE (mode) == 128 && TARGET_SSE_SPLIT_REGS) return cost * 2; @@ -39210,6 +39869,163 @@ return cost; } +/* Return cost of multiplication in MODE. */ + +static int +ix86_multiplication_cost (const struct processor_costs *cost, + enum machine_mode mode) +{ + machine_mode inner_mode = mode; + if (VECTOR_MODE_P (mode)) + inner_mode = GET_MODE_INNER (mode); + + if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH) + return inner_mode == DFmode ? cost->mulsd : cost->mulss; + else if (X87_FLOAT_MODE_P (mode)) + return cost->fmul; + else if (FLOAT_MODE_P (mode)) + return ix86_vec_cost (mode, + inner_mode == DFmode ? cost->mulsd : cost->mulss); + else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT) + { + /* vpmullq is used in this case. No emulation is needed. */ + if (TARGET_AVX512DQ) + return ix86_vec_cost (mode, cost->mulss); + + /* V*QImode is emulated with 7-13 insns. */ + if (mode == V16QImode || mode == V32QImode) + { + int extra = 11; + if (TARGET_XOP && mode == V16QImode) + extra = 5; + else if (TARGET_SSSE3) + extra = 6; + return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * extra); + } + /* V*DImode is emulated with 5-8 insns. */ + else if (mode == V2DImode || mode == V4DImode) + { + if (TARGET_XOP && mode == V2DImode) + return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 3); + else + return ix86_vec_cost (mode, cost->mulss * 3 + cost->sse_op * 5); + } + /* Without sse4.1, we don't have PMULLD; it's emulated with 7 + insns, including two PMULUDQ. */ + else if (mode == V4SImode && !(TARGET_SSE4_1 || TARGET_AVX)) + return ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 5); + else + return ix86_vec_cost (mode, cost->mulss); + } + else + return (cost->mult_init[MODE_INDEX (mode)] + cost->mult_bit * 7); +} + +/* Return cost of multiplication in MODE. */ + +static int +ix86_division_cost (const struct processor_costs *cost, + enum machine_mode mode) +{ + machine_mode inner_mode = mode; + if (VECTOR_MODE_P (mode)) + inner_mode = GET_MODE_INNER (mode); + + if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH) + return inner_mode == DFmode ? cost->divsd : cost->divss; + else if (X87_FLOAT_MODE_P (mode)) + return cost->fdiv; + else if (FLOAT_MODE_P (mode)) + return ix86_vec_cost (mode, + inner_mode == DFmode ? cost->divsd : cost->divss); + else + return cost->divide[MODE_INDEX (mode)]; +} + +/* Return cost of shift in MODE. + If CONSTANT_OP1 is true, the op1 value is known and set in OP1_VAL. + AND_IN_OP1 specify in op1 is result of and and SHIFT_AND_TRUNCATE + if op1 is a result of subreg. + + SKIP_OP0/1 is set to true if cost of OP0/1 should be ignored. */ + +static int +ix86_shift_rotate_cost (const struct processor_costs *cost, + enum machine_mode mode, bool constant_op1, + HOST_WIDE_INT op1_val, + bool speed, + bool and_in_op1, + bool shift_and_truncate, + bool *skip_op0, bool *skip_op1) +{ + if (skip_op0) + *skip_op0 = *skip_op1 = false; + if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT) + { + /* V*QImode is emulated with 1-11 insns. */ + if (mode == V16QImode || mode == V32QImode) + { + int count = 11; + if (TARGET_XOP && mode == V16QImode) + { + /* For XOP we use vpshab, which requires a broadcast of the + value to the variable shift insn. For constants this + means a V16Q const in mem; even when we can perform the + shift with one insn set the cost to prefer paddb. */ + if (constant_op1) + { + if (skip_op1) + *skip_op1 = true; + return ix86_vec_cost (mode, + cost->sse_op + + (speed + ? 2 + : COSTS_N_BYTES + (GET_MODE_UNIT_SIZE (mode)))); + } + count = 3; + } + else if (TARGET_SSSE3) + count = 7; + return ix86_vec_cost (mode, cost->sse_op * count); + } + else + return ix86_vec_cost (mode, cost->sse_op); + } + if (GET_MODE_SIZE (mode) > UNITS_PER_WORD) + { + if (constant_op1) + { + if (op1_val > 32) + return cost->shift_const + COSTS_N_INSNS (2); + else + return cost->shift_const * 2; + } + else + { + if (and_in_op1) + return cost->shift_var * 2; + else + return cost->shift_var * 6 + COSTS_N_INSNS (2); + } + } + else + { + if (constant_op1) + return cost->shift_const; + else if (shift_and_truncate) + { + if (skip_op0) + *skip_op0 = *skip_op1 = true; + /* Return the cost after shift-and truncation. */ + return cost->shift_var; + } + else + return cost->shift_var; + } + return cost->shift_const; +} + /* Compute a (partial) cost for rtx X. Return true if the complete cost has been computed, and false if subexpressions should be scanned. In either case, *TOTAL contains the cost result. */ @@ -39223,15 +40039,12 @@ enum rtx_code outer_code = (enum rtx_code) outer_code_i; const struct processor_costs *cost = speed ? ix86_cost : &ix86_size_cost; int src_cost; - machine_mode inner_mode = mode; - if (VECTOR_MODE_P (mode)) - inner_mode = GET_MODE_INNER (mode); switch (code) { case SET: if (register_operand (SET_DEST (x), VOIDmode) - && reg_or_0_operand (SET_SRC (x), VOIDmode)) + && register_operand (SET_SRC (x), VOIDmode)) { *total = ix86_set_reg_reg_cost (GET_MODE (SET_DEST (x))); return true; @@ -39258,20 +40071,10 @@ case CONST: case LABEL_REF: case SYMBOL_REF: - if (TARGET_64BIT && !x86_64_immediate_operand (x, VOIDmode)) - *total = 3; - else if (TARGET_64BIT && !x86_64_zext_immediate_operand (x, VOIDmode)) - *total = 2; - else if (flag_pic && SYMBOLIC_CONST (x) - && !(TARGET_64BIT - && (GET_CODE (x) == LABEL_REF - || (GET_CODE (x) == SYMBOL_REF - && SYMBOL_REF_LOCAL_P (x)))) - /* Use 0 cost for CONST to improve its propagation. */ - && (TARGET_64BIT || GET_CODE (x) != CONST)) + if (x86_64_immediate_operand (x, VOIDmode)) + *total = 0; + else *total = 1; - else - *total = 0; return true; case CONST_DOUBLE: @@ -39353,68 +40156,22 @@ case ASHIFTRT: case LSHIFTRT: case ROTATERT: - if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT) - { - /* ??? Should be SSE vector operation cost. */ - /* At least for published AMD latencies, this really is the same - as the latency for a simple fpu operation like fabs. */ - /* V*QImode is emulated with 1-11 insns. */ - if (mode == V16QImode || mode == V32QImode) - { - int count = 11; - if (TARGET_XOP && mode == V16QImode) - { - /* For XOP we use vpshab, which requires a broadcast of the - value to the variable shift insn. For constants this - means a V16Q const in mem; even when we can perform the - shift with one insn set the cost to prefer paddb. */ - if (CONSTANT_P (XEXP (x, 1))) - { - *total = ix86_vec_cost (mode, - cost->sse_op - + rtx_cost (XEXP (x, 0), mode, code, 0, speed) - + (speed ? 2 : COSTS_N_BYTES (16)), true); - return true; - } - count = 3; - } - else if (TARGET_SSSE3) - count = 7; - *total = ix86_vec_cost (mode, cost->sse_op * count, true); - } - else - *total = ix86_vec_cost (mode, cost->sse_op, true); - } - else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD) - { - if (CONST_INT_P (XEXP (x, 1))) - { - if (INTVAL (XEXP (x, 1)) > 32) - *total = cost->shift_const + COSTS_N_INSNS (2); - else - *total = cost->shift_const * 2; - } - else - { - if (GET_CODE (XEXP (x, 1)) == AND) - *total = cost->shift_var * 2; - else - *total = cost->shift_var * 6 + COSTS_N_INSNS (2); - } - } - else - { - if (CONST_INT_P (XEXP (x, 1))) - *total = cost->shift_const; - else if (SUBREG_P (XEXP (x, 1)) - && GET_CODE (XEXP (XEXP (x, 1), 0)) == AND) - { - /* Return the cost after shift-and truncation. */ - *total = cost->shift_var; - return true; - } - else - *total = cost->shift_var; + bool skip_op0, skip_op1; + *total = ix86_shift_rotate_cost (cost, mode, CONSTANT_P (XEXP (x, 1)), + CONST_INT_P (XEXP (x, 1)) + ? INTVAL (XEXP (x, 1)) : -1, + speed, + GET_CODE (XEXP (x, 1)) == AND, + SUBREG_P (XEXP (x, 1)) + && GET_CODE (XEXP (XEXP (x, 1), 0)) == AND, + &skip_op0, &skip_op1); + if (skip_op0 || skip_op1) + { + if (!skip_op0) + *total += rtx_cost (XEXP (x, 0), mode, code, 0, speed); + if (!skip_op1) + *total += rtx_cost (XEXP (x, 1), mode, code, 0, speed); + return true; } return false; @@ -39426,8 +40183,8 @@ gcc_assert (TARGET_FMA || TARGET_FMA4 || TARGET_AVX512F); *total = ix86_vec_cost (mode, - mode == SFmode ? cost->fmass : cost->fmasd, - true); + GET_MODE_INNER (mode) == SFmode + ? cost->fmass : cost->fmasd); *total += rtx_cost (XEXP (x, 1), mode, FMA, 1, speed); /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */ @@ -39444,59 +40201,7 @@ } case MULT: - if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH) - { - *total = inner_mode == DFmode ? cost->mulsd : cost->mulss; - return false; - } - else if (X87_FLOAT_MODE_P (mode)) - { - *total = cost->fmul; - return false; - } - else if (FLOAT_MODE_P (mode)) - { - *total = ix86_vec_cost (mode, - inner_mode == DFmode - ? cost->mulsd : cost->mulss, true); - return false; - } - else if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT) - { - /* V*QImode is emulated with 7-13 insns. */ - if (mode == V16QImode || mode == V32QImode) - { - int extra = 11; - if (TARGET_XOP && mode == V16QImode) - extra = 5; - else if (TARGET_SSSE3) - extra = 6; - *total = ix86_vec_cost (mode, - cost->mulss * 2 + cost->sse_op * extra, - true); - } - /* V*DImode is emulated with 5-8 insns. */ - else if (mode == V2DImode || mode == V4DImode) - { - if (TARGET_XOP && mode == V2DImode) - *total = ix86_vec_cost (mode, - cost->mulss * 2 + cost->sse_op * 3, - true); - else - *total = ix86_vec_cost (mode, - cost->mulss * 3 + cost->sse_op * 5, - true); - } - /* Without sse4.1, we don't have PMULLD; it's emulated with 7 - insns, including two PMULUDQ. */ - else if (mode == V4SImode && !(TARGET_SSE4_1 || TARGET_AVX)) - *total = ix86_vec_cost (mode, cost->mulss * 2 + cost->sse_op * 5, - true); - else - *total = ix86_vec_cost (mode, cost->mulss, true); - return false; - } - else + if (!FLOAT_MODE_P (mode) && !VECTOR_MODE_P (mode)) { rtx op0 = XEXP (x, 0); rtx op1 = XEXP (x, 1); @@ -39541,21 +40246,14 @@ return true; } + *total = ix86_multiplication_cost (cost, mode); + return false; case DIV: case UDIV: case MOD: case UMOD: - if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH) - *total = inner_mode == DFmode ? cost->divsd : cost->divss; - else if (X87_FLOAT_MODE_P (mode)) - *total = cost->fdiv; - else if (FLOAT_MODE_P (mode)) - *total = ix86_vec_cost (mode, - inner_mode == DFmode ? cost->divsd : cost->divss, - true); - else - *total = cost->divide[MODE_INDEX (mode)]; + *total = ix86_division_cost (cost, mode); return false; case PLUS: @@ -39642,7 +40340,7 @@ } else if (FLOAT_MODE_P (mode)) { - *total = ix86_vec_cost (mode, cost->addss, true); + *total = ix86_vec_cost (mode, cost->addss); return false; } /* FALLTHRU */ @@ -39675,14 +40373,14 @@ } else if (FLOAT_MODE_P (mode)) { - *total = ix86_vec_cost (mode, cost->sse_op, true); + *total = ix86_vec_cost (mode, cost->sse_op); return false; } /* FALLTHRU */ case NOT: if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT) - *total = ix86_vec_cost (mode, cost->sse_op, true); + *total = ix86_vec_cost (mode, cost->sse_op); else if (GET_MODE_SIZE (mode) > UNITS_PER_WORD) *total = cost->add * 2; else @@ -39716,14 +40414,14 @@ if (!(SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)) *total = 0; else - *total = ix86_vec_cost (mode, cost->addss, true); + *total = ix86_vec_cost (mode, cost->addss); return false; case FLOAT_TRUNCATE: if (!(SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH)) *total = cost->fadd; else - *total = ix86_vec_cost (mode, cost->addss, true); + *total = ix86_vec_cost (mode, cost->addss); return false; case ABS: @@ -39735,7 +40433,7 @@ else if (X87_FLOAT_MODE_P (mode)) *total = cost->fabs; else if (FLOAT_MODE_P (mode)) - *total = ix86_vec_cost (mode, cost->sse_op, true); + *total = ix86_vec_cost (mode, cost->sse_op); return false; case SQRT: @@ -39745,8 +40443,7 @@ *total = cost->fsqrt; else if (FLOAT_MODE_P (mode)) *total = ix86_vec_cost (mode, - mode == SFmode ? cost->sqrtss : cost->sqrtsd, - true); + mode == SFmode ? cost->sqrtss : cost->sqrtsd); return false; case UNSPEC: @@ -39908,10 +40605,6 @@ for (i = FIRST_MASK_REG; i <= LAST_MASK_REG; i++) reg_alloc_order [pos++] = i; - /* MPX bound registers. */ - for (i = FIRST_BND_REG; i <= LAST_BND_REG; i++) - reg_alloc_order [pos++] = i; - /* x87 registers. */ if (TARGET_SSE_MATH) for (i = FIRST_STACK_REG; i <= LAST_STACK_REG; i++) @@ -39929,10 +40622,8 @@ /* Handle a "callee_pop_aggregate_return" attribute; arguments as in struct attribute_spec handler. */ static tree -ix86_handle_callee_pop_aggregate_return (tree *node, tree name, - tree args, - int, - bool *no_add_attrs) +ix86_handle_callee_pop_aggregate_return (tree *node, tree name, tree args, int, + bool *no_add_attrs) { if (TREE_CODE (*node) != FUNCTION_TYPE && TREE_CODE (*node) != METHOD_TYPE @@ -40054,7 +40745,7 @@ } static tree -ix86_handle_fndecl_attribute (tree *node, tree name, tree, int, +ix86_handle_fndecl_attribute (tree *node, tree name, tree args, int, bool *no_add_attrs) { if (TREE_CODE (*node) != FUNCTION_DECL) @@ -40063,6 +40754,51 @@ name); *no_add_attrs = true; } + + if (is_attribute_p ("indirect_branch", name)) + { + tree cst = TREE_VALUE (args); + if (TREE_CODE (cst) != STRING_CST) + { + warning (OPT_Wattributes, + "%qE attribute requires a string constant argument", + name); + *no_add_attrs = true; + } + else if (strcmp (TREE_STRING_POINTER (cst), "keep") != 0 + && strcmp (TREE_STRING_POINTER (cst), "thunk") != 0 + && strcmp (TREE_STRING_POINTER (cst), "thunk-inline") != 0 + && strcmp (TREE_STRING_POINTER (cst), "thunk-extern") != 0) + { + warning (OPT_Wattributes, + "argument to %qE attribute is not " + "(keep|thunk|thunk-inline|thunk-extern)", name); + *no_add_attrs = true; + } + } + + if (is_attribute_p ("function_return", name)) + { + tree cst = TREE_VALUE (args); + if (TREE_CODE (cst) != STRING_CST) + { + warning (OPT_Wattributes, + "%qE attribute requires a string constant argument", + name); + *no_add_attrs = true; + } + else if (strcmp (TREE_STRING_POINTER (cst), "keep") != 0 + && strcmp (TREE_STRING_POINTER (cst), "thunk") != 0 + && strcmp (TREE_STRING_POINTER (cst), "thunk-inline") != 0 + && strcmp (TREE_STRING_POINTER (cst), "thunk-extern") != 0) + { + warning (OPT_Wattributes, + "argument to %qE attribute is not " + "(keep|thunk|thunk-inline|thunk-extern)", name); + *no_add_attrs = true; + } + } + return NULL_TREE; } @@ -40235,7 +40971,7 @@ emit_note (NOTE_INSN_PROLOGUE_END); /* CET is enabled, insert EB instruction. */ - if ((flag_cf_protection & CF_BRANCH) && TARGET_IBT) + if ((flag_cf_protection & CF_BRANCH)) emit_insn (gen_nop_endbr ()); /* If VCALL_OFFSET, we'll need THIS in a register. Might as well @@ -40438,7 +41174,8 @@ x86_print_call_or_nop (FILE *file, const char *target) { if (flag_nop_mcount) - fprintf (file, "1:\tnopl 0x00(%%eax,%%eax,1)\n"); /* 5 byte nop. */ + /* 5 byte nop: nopl 0(%[re]ax,%[re]ax,1) */ + fprintf (file, "1:" ASM_BYTE "0x0f, 0x1f, 0x44, 0x00, 0x00\n"); else fprintf (file, "1:\tcall\t%s\n", target); } @@ -40450,6 +41187,10 @@ { const char *mcount_name = (flag_fentry ? MCOUNT_NAME_BEFORE_PROLOGUE : MCOUNT_NAME); + + if (cfun->machine->endbr_queued_at_entrance) + fprintf (file, "\t%s\n", TARGET_64BIT ? "endbr64" : "endbr32"); + if (TARGET_64BIT) { #ifndef NO_PROFILE_COUNTERS @@ -40576,8 +41317,9 @@ if (LABEL_P (insn)) { - int align = label_to_alignment (insn); - int max_skip = label_to_max_skip (insn); + align_flags alignment = label_to_alignment (insn); + int align = alignment.levels[0].log; + int max_skip = alignment.levels[0].maxskip; if (max_skip > 15) max_skip = 15; @@ -40845,9 +41587,7 @@ /* Do not separate calls from their debug information. */ for (next = NEXT_INSN (insn); next != NULL; next = NEXT_INSN (next)) - if (NOTE_P (next) - && (NOTE_KIND (next) == NOTE_INSN_VAR_LOCATION - || NOTE_KIND (next) == NOTE_INSN_CALL_ARG_LOCATION)) + if (NOTE_P (next) && NOTE_KIND (next) == NOTE_INSN_VAR_LOCATION) insn = next; else break; @@ -40856,216 +41596,6 @@ } } -/* Given a register number BASE, the lowest of a group of registers, update - regsets IN and OUT with the registers that should be avoided in input - and output operands respectively when trying to avoid generating a modr/m - byte for -fmitigate-rop. */ - -static void -set_rop_modrm_reg_bits (int base, HARD_REG_SET &in, HARD_REG_SET &out) -{ - SET_HARD_REG_BIT (out, base); - SET_HARD_REG_BIT (out, base + 1); - SET_HARD_REG_BIT (in, base + 2); - SET_HARD_REG_BIT (in, base + 3); -} - -/* Called if -fmitigate_rop is in effect. Try to rewrite instructions so - that certain encodings of modr/m bytes do not occur. */ -static void -ix86_mitigate_rop (void) -{ - HARD_REG_SET input_risky; - HARD_REG_SET output_risky; - HARD_REG_SET inout_risky; - - CLEAR_HARD_REG_SET (output_risky); - CLEAR_HARD_REG_SET (input_risky); - SET_HARD_REG_BIT (output_risky, AX_REG); - SET_HARD_REG_BIT (output_risky, CX_REG); - SET_HARD_REG_BIT (input_risky, BX_REG); - SET_HARD_REG_BIT (input_risky, DX_REG); - set_rop_modrm_reg_bits (FIRST_SSE_REG, input_risky, output_risky); - set_rop_modrm_reg_bits (FIRST_REX_INT_REG, input_risky, output_risky); - set_rop_modrm_reg_bits (FIRST_REX_SSE_REG, input_risky, output_risky); - set_rop_modrm_reg_bits (FIRST_EXT_REX_SSE_REG, input_risky, output_risky); - set_rop_modrm_reg_bits (FIRST_MASK_REG, input_risky, output_risky); - set_rop_modrm_reg_bits (FIRST_BND_REG, input_risky, output_risky); - COPY_HARD_REG_SET (inout_risky, input_risky); - IOR_HARD_REG_SET (inout_risky, output_risky); - - df_note_add_problem (); - /* Fix up what stack-regs did. */ - df_insn_rescan_all (); - df_analyze (); - - regrename_init (true); - regrename_analyze (NULL); - - auto_vec<du_head_p> cands; - - for (rtx_insn *insn = get_insns (); insn; insn = NEXT_INSN (insn)) - { - if (!NONDEBUG_INSN_P (insn)) - continue; - - if (GET_CODE (PATTERN (insn)) == USE - || GET_CODE (PATTERN (insn)) == CLOBBER) - continue; - - extract_insn (insn); - - int opno0, opno1; - int modrm = ix86_get_modrm_for_rop (insn, recog_data.operand, - recog_data.n_operands, &opno0, - &opno1); - - if (!ix86_rop_should_change_byte_p (modrm)) - continue; - - insn_rr_info *info = &insn_rr[INSN_UID (insn)]; - - /* This happens when regrename has to fail a block. */ - if (!info->op_info) - continue; - - if (info->op_info[opno0].n_chains != 0) - { - gcc_assert (info->op_info[opno0].n_chains == 1); - du_head_p op0c; - op0c = regrename_chain_from_id (info->op_info[opno0].heads[0]->id); - if (op0c->target_data_1 + op0c->target_data_2 == 0 - && !op0c->cannot_rename) - cands.safe_push (op0c); - - op0c->target_data_1++; - } - if (info->op_info[opno1].n_chains != 0) - { - gcc_assert (info->op_info[opno1].n_chains == 1); - du_head_p op1c; - op1c = regrename_chain_from_id (info->op_info[opno1].heads[0]->id); - if (op1c->target_data_1 + op1c->target_data_2 == 0 - && !op1c->cannot_rename) - cands.safe_push (op1c); - - op1c->target_data_2++; - } - } - - int i; - du_head_p head; - FOR_EACH_VEC_ELT (cands, i, head) - { - int old_reg, best_reg; - HARD_REG_SET unavailable; - - CLEAR_HARD_REG_SET (unavailable); - if (head->target_data_1) - IOR_HARD_REG_SET (unavailable, output_risky); - if (head->target_data_2) - IOR_HARD_REG_SET (unavailable, input_risky); - - int n_uses; - reg_class superclass = regrename_find_superclass (head, &n_uses, - &unavailable); - old_reg = head->regno; - best_reg = find_rename_reg (head, superclass, &unavailable, - old_reg, false); - bool ok = regrename_do_replace (head, best_reg); - gcc_assert (ok); - if (dump_file) - fprintf (dump_file, "Chain %d renamed as %s in %s\n", head->id, - reg_names[best_reg], reg_class_names[superclass]); - - } - - regrename_finish (); - - df_analyze (); - - basic_block bb; - regset_head live; - - INIT_REG_SET (&live); - - FOR_EACH_BB_FN (bb, cfun) - { - rtx_insn *insn; - - COPY_REG_SET (&live, DF_LR_OUT (bb)); - df_simulate_initialize_backwards (bb, &live); - - FOR_BB_INSNS_REVERSE (bb, insn) - { - if (!NONDEBUG_INSN_P (insn)) - continue; - - df_simulate_one_insn_backwards (bb, insn, &live); - - if (GET_CODE (PATTERN (insn)) == USE - || GET_CODE (PATTERN (insn)) == CLOBBER) - continue; - - extract_insn (insn); - constrain_operands_cached (insn, reload_completed); - int opno0, opno1; - int modrm = ix86_get_modrm_for_rop (insn, recog_data.operand, - recog_data.n_operands, &opno0, - &opno1); - if (modrm < 0 - || !ix86_rop_should_change_byte_p (modrm) - || opno0 == opno1) - continue; - - rtx oldreg = recog_data.operand[opno1]; - preprocess_constraints (insn); - const operand_alternative *alt = which_op_alt (); - - int i; - for (i = 0; i < recog_data.n_operands; i++) - if (i != opno1 - && alt[i].earlyclobber - && reg_overlap_mentioned_p (recog_data.operand[i], - oldreg)) - break; - - if (i < recog_data.n_operands) - continue; - - if (dump_file) - fprintf (dump_file, - "attempting to fix modrm byte in insn %d:" - " reg %d class %s", INSN_UID (insn), REGNO (oldreg), - reg_class_names[alt[opno1].cl]); - - HARD_REG_SET unavailable; - REG_SET_TO_HARD_REG_SET (unavailable, &live); - SET_HARD_REG_BIT (unavailable, REGNO (oldreg)); - IOR_COMPL_HARD_REG_SET (unavailable, call_used_reg_set); - IOR_HARD_REG_SET (unavailable, fixed_reg_set); - IOR_HARD_REG_SET (unavailable, output_risky); - IOR_COMPL_HARD_REG_SET (unavailable, - reg_class_contents[alt[opno1].cl]); - - for (i = 0; i < FIRST_PSEUDO_REGISTER; i++) - if (!TEST_HARD_REG_BIT (unavailable, i)) - break; - if (i == FIRST_PSEUDO_REGISTER) - { - if (dump_file) - fprintf (dump_file, ", none available\n"); - continue; - } - if (dump_file) - fprintf (dump_file, " -> %d\n", i); - rtx newreg = gen_rtx_REG (recog_data.operand_mode[opno1], i); - validate_change (insn, recog_data.operand_loc[opno1], newreg, false); - insn = emit_insn_before (gen_move_insn (newreg, oldreg), insn); - } - } -} - /* Implement machine specific optimizations. We implement padding of returns for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */ static void @@ -41075,9 +41605,6 @@ with old MDEP_REORGS that are not CFG based. Recompute it now. */ compute_bb_for_insn (); - if (flag_mitigate_rop) - ix86_mitigate_rop (); - if (TARGET_SEH && current_function_has_exception_handlers ()) ix86_seh_fixup_eh_fallthru (); @@ -41241,7 +41768,7 @@ rtx dup; /* First attempt to recognize VAL as-is. */ - dup = gen_rtx_VEC_DUPLICATE (mode, val); + dup = gen_vec_duplicate (mode, val); insn = emit_insn (gen_rtx_SET (target, dup)); if (recog_memoized (insn) < 0) { @@ -41255,7 +41782,7 @@ reg = force_reg (innermode, val); if (GET_MODE (reg) != innermode) reg = gen_lowpart (innermode, reg); - XEXP (dup, 0) = reg; + SET_SRC (PATTERN (insn)) = gen_vec_duplicate (mode, reg); seq = get_insns (); end_sequence (); if (seq) @@ -41433,6 +41960,7 @@ rtx new_target; rtx x, tmp; bool use_vector_set = false; + rtx (*gen_vec_set_0) (rtx, rtx, rtx) = NULL; switch (mode) { @@ -41457,14 +41985,41 @@ break; case E_V32QImode: case E_V16HImode: + use_vector_set = TARGET_AVX; + break; case E_V8SImode: + use_vector_set = TARGET_AVX; + gen_vec_set_0 = gen_vec_setv8si_0; + break; case E_V8SFmode: + use_vector_set = TARGET_AVX; + gen_vec_set_0 = gen_vec_setv8sf_0; + break; case E_V4DFmode: use_vector_set = TARGET_AVX; + gen_vec_set_0 = gen_vec_setv4df_0; break; case E_V4DImode: /* Use ix86_expand_vector_set in 64bit mode only. */ use_vector_set = TARGET_AVX && TARGET_64BIT; + gen_vec_set_0 = gen_vec_setv4di_0; + break; + case E_V16SImode: + use_vector_set = TARGET_AVX512F && one_var == 0; + gen_vec_set_0 = gen_vec_setv16si_0; + break; + case E_V16SFmode: + use_vector_set = TARGET_AVX512F && one_var == 0; + gen_vec_set_0 = gen_vec_setv16sf_0; + break; + case E_V8DFmode: + use_vector_set = TARGET_AVX512F && one_var == 0; + gen_vec_set_0 = gen_vec_setv8df_0; + break; + case E_V8DImode: + /* Use ix86_expand_vector_set in 64bit mode only. */ + use_vector_set = TARGET_AVX512F && TARGET_64BIT && one_var == 0; + gen_vec_set_0 = gen_vec_setv8di_0; break; default: break; @@ -41472,6 +42027,12 @@ if (use_vector_set) { + if (gen_vec_set_0 && one_var == 0) + { + var = force_reg (GET_MODE_INNER (mode), var); + emit_insn (gen_vec_set_0 (target, CONST0_RTX (mode), var)); + return true; + } emit_insn (gen_rtx_SET (target, CONST0_RTX (mode))); var = force_reg (GET_MODE_INNER (mode), var); ix86_expand_vector_set (mmx_ok, target, var, one_var); @@ -41643,7 +42204,7 @@ else { var = convert_modes (HImode, QImode, var, true); - x = gen_int_mode (INTVAL (x) << 8, HImode); + x = gen_int_mode (UINTVAL (x) << 8, HImode); } if (x != const0_rtx) var = expand_simple_binop (HImode, IOR, var, x, var, @@ -42523,20 +43084,68 @@ break; case E_V32HImode: - if (TARGET_AVX512F && TARGET_AVX512BW) + if (TARGET_AVX512BW) { mmode = SImode; gen_blendm = gen_avx512bw_blendmv32hi; } + else if (TARGET_AVX512F) + { + half_mode = E_V8HImode; + n = 8; + goto quarter; + } break; case E_V64QImode: - if (TARGET_AVX512F && TARGET_AVX512BW) + if (TARGET_AVX512BW) { mmode = DImode; gen_blendm = gen_avx512bw_blendmv64qi; } - break; + else if (TARGET_AVX512F) + { + half_mode = E_V16QImode; + n = 16; + goto quarter; + } + break; + +quarter: + /* Compute offset. */ + i = elt / n; + elt %= n; + + gcc_assert (i <= 3); + + { + /* Extract the quarter. */ + tmp = gen_reg_rtx (V4SImode); + rtx tmp2 = gen_lowpart (V16SImode, target); + rtx mask = gen_reg_rtx (QImode); + + emit_move_insn (mask, constm1_rtx); + emit_insn (gen_avx512f_vextracti32x4_mask (tmp, tmp2, GEN_INT (i), + tmp, mask)); + + tmp2 = gen_reg_rtx (half_mode); + emit_move_insn (tmp2, gen_lowpart (half_mode, tmp)); + tmp = tmp2; + + /* Put val in tmp at elt. */ + ix86_expand_vector_set (false, tmp, val, elt); + + /* Put it back. */ + tmp2 = gen_reg_rtx (V16SImode); + rtx tmp3 = gen_lowpart (V16SImode, target); + mask = gen_reg_rtx (HImode); + emit_move_insn (mask, constm1_rtx); + tmp = gen_lowpart (V4SImode, tmp); + emit_insn (gen_avx512f_vinserti32x4_mask (tmp2, tmp3, tmp, GEN_INT (i), + tmp3, mask)); + emit_move_insn (target, gen_lowpart (mode, tmp2)); + } + return; default: break; @@ -42553,12 +43162,14 @@ where the mask is clear and second input operand otherwise. */ emit_insn (gen_blendm (target, target, tmp, force_reg (mmode, - gen_int_mode (1 << elt, mmode)))); + gen_int_mode (HOST_WIDE_INT_1U << elt, + mmode)))); } else if (use_vec_merge) { tmp = gen_rtx_VEC_DUPLICATE (mode, val); - tmp = gen_rtx_VEC_MERGE (mode, tmp, target, GEN_INT (1 << elt)); + tmp = gen_rtx_VEC_MERGE (mode, tmp, target, + GEN_INT (HOST_WIDE_INT_1U << elt)); emit_insn (gen_rtx_SET (target, tmp)); } else @@ -42567,7 +43178,7 @@ emit_move_insn (mem, target); - tmp = adjust_address (mem, inner_mode, elt*GET_MODE_SIZE (inner_mode)); + tmp = adjust_address (mem, inner_mode, elt * GET_MODE_SIZE (inner_mode)); emit_move_insn (tmp, val); emit_move_insn (target, mem); @@ -43049,9 +43660,6 @@ vec<const char *> &constraints, vec<rtx> &clobbers, HARD_REG_SET &clobbered_regs) { - clobbers.safe_push (gen_rtx_REG (CCFPmode, FPSR_REG)); - SET_HARD_REG_BIT (clobbered_regs, FPSR_REG); - bool saw_asm_flag = false; start_sequence (); @@ -43220,9 +43828,9 @@ enum rtx_code ix86_reverse_condition (enum rtx_code code, machine_mode mode) { - return (mode != CCFPmode && mode != CCFPUmode - ? reverse_condition (code) - : reverse_condition_maybe_unordered (code)); + return (mode == CCFPmode + ? reverse_condition_maybe_unordered (code) + : reverse_condition (code)); } /* Output code to perform an x87 FP register move, from OPERANDS[1] @@ -43270,6 +43878,7 @@ ix86_emit_fp_unordered_jump (rtx label) { rtx reg = gen_reg_rtx (HImode); + rtx_insn *insn; rtx temp; emit_insn (gen_x86_fnstsw_1 (reg)); @@ -43292,10 +43901,9 @@ temp = gen_rtx_IF_THEN_ELSE (VOIDmode, temp, gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx); - temp = gen_rtx_SET (pc_rtx, temp); - - emit_jump_insn (temp); + insn = emit_jump_insn (gen_rtx_SET (pc_rtx, temp)); predict_jump (REG_BR_PROB_BASE * 10 / 100); + JUMP_LABEL (insn) = label; } /* Output code to perform a log1p XFmode calculation. */ @@ -43306,27 +43914,36 @@ rtx_code_label *label2 = gen_label_rtx (); rtx tmp = gen_reg_rtx (XFmode); - rtx tmp2 = gen_reg_rtx (XFmode); - rtx test; + rtx res = gen_reg_rtx (XFmode); + rtx cst, cstln2, cst1; + rtx_insn *insn; + + cst = const_double_from_real_value + (REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode), XFmode); + cstln2 = force_reg (XFmode, standard_80387_constant_rtx (4)); /* fldln2 */ emit_insn (gen_absxf2 (tmp, op1)); - test = gen_rtx_GE (VOIDmode, tmp, - const_double_from_real_value ( - REAL_VALUE_ATOF ("0.29289321881345247561810596348408353", XFmode), - XFmode)); - emit_jump_insn (gen_cbranchxf4 (test, XEXP (test, 0), XEXP (test, 1), label1)); - - emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */ - emit_insn (gen_fyl2xp1xf3_i387 (op0, op1, tmp2)); + + cst = force_reg (XFmode, cst); + ix86_expand_branch (GE, tmp, cst, label1); + predict_jump (REG_BR_PROB_BASE * 10 / 100); + insn = get_last_insn (); + JUMP_LABEL (insn) = label1; + + emit_insn (gen_fyl2xp1xf3_i387 (res, op1, cstln2)); emit_jump (label2); emit_label (label1); - emit_move_insn (tmp, CONST1_RTX (XFmode)); - emit_insn (gen_addxf3 (tmp, op1, tmp)); - emit_move_insn (tmp2, standard_80387_constant_rtx (4)); /* fldln2 */ - emit_insn (gen_fyl2xxf3_i387 (op0, tmp, tmp2)); + LABEL_NUSES (label1) = 1; + + cst1 = force_reg (XFmode, CONST1_RTX (XFmode)); + emit_insn (gen_rtx_SET (tmp, gen_rtx_PLUS (XFmode, op1, cst1))); + emit_insn (gen_fyl2xxf3_i387 (res, tmp, cstln2)); emit_label (label2); + LABEL_NUSES (label2) = 1; + + emit_move_insn (op0, res); } /* Emit code for round calculation. */ @@ -43334,24 +43951,28 @@ { machine_mode inmode = GET_MODE (op1); machine_mode outmode = GET_MODE (op0); - rtx e1, e2, res, tmp, tmp1, half; + rtx e1 = gen_reg_rtx (XFmode); + rtx e2 = gen_reg_rtx (XFmode); rtx scratch = gen_reg_rtx (HImode); rtx flags = gen_rtx_REG (CCNOmode, FLAGS_REG); + rtx half = const_double_from_real_value (dconsthalf, XFmode); + rtx res = gen_reg_rtx (outmode); rtx_code_label *jump_label = gen_label_rtx (); - rtx insn; - rtx (*gen_abs) (rtx, rtx); - rtx (*gen_neg) (rtx, rtx); + rtx (*floor_insn) (rtx, rtx); + rtx (*neg_insn) (rtx, rtx); + rtx_insn *insn; + rtx tmp; switch (inmode) { case E_SFmode: - gen_abs = gen_abssf2; - break; case E_DFmode: - gen_abs = gen_absdf2; + tmp = gen_reg_rtx (XFmode); + + emit_insn (gen_rtx_SET (tmp, gen_rtx_FLOAT_EXTEND (XFmode, op1))); + op1 = tmp; break; case E_XFmode: - gen_abs = gen_absxf2; break; default: gcc_unreachable (); @@ -43360,84 +43981,61 @@ switch (outmode) { case E_SFmode: - gen_neg = gen_negsf2; + floor_insn = gen_frndintxf2_floor; + neg_insn = gen_negsf2; break; case E_DFmode: - gen_neg = gen_negdf2; + floor_insn = gen_frndintxf2_floor; + neg_insn = gen_negdf2; break; case E_XFmode: - gen_neg = gen_negxf2; + floor_insn = gen_frndintxf2_floor; + neg_insn = gen_negxf2; break; case E_HImode: - gen_neg = gen_neghi2; + floor_insn = gen_lfloorxfhi2; + neg_insn = gen_neghi2; break; case E_SImode: - gen_neg = gen_negsi2; + floor_insn = gen_lfloorxfsi2; + neg_insn = gen_negsi2; break; case E_DImode: - gen_neg = gen_negdi2; - break; - default: - gcc_unreachable (); - } - - e1 = gen_reg_rtx (inmode); - e2 = gen_reg_rtx (inmode); - res = gen_reg_rtx (outmode); - - half = const_double_from_real_value (dconsthalf, inmode); + floor_insn = gen_lfloorxfdi2; + neg_insn = gen_negdi2; + break; + default: + gcc_unreachable (); + } /* round(a) = sgn(a) * floor(fabs(a) + 0.5) */ /* scratch = fxam(op1) */ - emit_insn (gen_rtx_SET (scratch, - gen_rtx_UNSPEC (HImode, gen_rtvec (1, op1), - UNSPEC_FXAM))); + emit_insn (gen_fxamxf2_i387 (scratch, op1)); + /* e1 = fabs(op1) */ - emit_insn (gen_abs (e1, op1)); + emit_insn (gen_absxf2 (e1, op1)); /* e2 = e1 + 0.5 */ - half = force_reg (inmode, half); - emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (inmode, e1, half))); + half = force_reg (XFmode, half); + emit_insn (gen_rtx_SET (e2, gen_rtx_PLUS (XFmode, e1, half))); /* res = floor(e2) */ - if (inmode != XFmode) - { - tmp1 = gen_reg_rtx (XFmode); - - emit_insn (gen_rtx_SET (tmp1, gen_rtx_FLOAT_EXTEND (XFmode, e2))); - } - else - tmp1 = e2; - switch (outmode) { case E_SFmode: case E_DFmode: { - rtx tmp0 = gen_reg_rtx (XFmode); - - emit_insn (gen_frndintxf2_floor (tmp0, tmp1)); - + tmp = gen_reg_rtx (XFmode); + + emit_insn (floor_insn (tmp, e2)); emit_insn (gen_rtx_SET (res, - gen_rtx_UNSPEC (outmode, gen_rtvec (1, tmp0), + gen_rtx_UNSPEC (outmode, gen_rtvec (1, tmp), UNSPEC_TRUNC_NOOP))); } break; - case E_XFmode: - emit_insn (gen_frndintxf2_floor (res, tmp1)); - break; - case E_HImode: - emit_insn (gen_lfloorxfhi2 (res, tmp1)); - break; - case E_SImode: - emit_insn (gen_lfloorxfsi2 (res, tmp1)); - break; - case E_DImode: - emit_insn (gen_lfloorxfdi2 (res, tmp1)); - break; - default: - gcc_unreachable (); + default: + emit_insn (floor_insn (res, e2)); } /* flags = signbit(a) */ @@ -43452,7 +44050,7 @@ predict_jump (REG_BR_PROB_BASE * 50 / 100); JUMP_LABEL (insn) = jump_label; - emit_insn (gen_neg (res, res)); + emit_insn (neg_insn (res, res)); emit_label (jump_label); LABEL_NUSES (jump_label) = 1; @@ -43635,6 +44233,15 @@ solaris_elf_asm_comdat_section (name, flags, decl); return; } + + /* Solaris/x86 as uses the same syntax for the SHF_EXCLUDE flags as the + SPARC assembler. One cannot mix single-letter flags and #exclude, so + only emit the latter here. */ + if (flags & SECTION_EXCLUDE) + { + fprintf (asm_out_file, "\t.section\t%s,#exclude\n", name); + return; + } #endif default_elf_asm_named_section (name, flags, decl); @@ -43835,17 +44442,20 @@ ix86_expand_sse_compare_and_jump (enum rtx_code code, rtx op0, rtx op1, bool swap_operands) { - machine_mode fpcmp_mode = ix86_fp_compare_mode (code); + bool unordered_compare = ix86_unordered_fp_compare (code); rtx_code_label *label; - rtx tmp; + rtx tmp, reg; if (swap_operands) std::swap (op0, op1); label = gen_label_rtx (); - tmp = gen_rtx_REG (fpcmp_mode, FLAGS_REG); - emit_insn (gen_rtx_SET (tmp, gen_rtx_COMPARE (fpcmp_mode, op0, op1))); - tmp = gen_rtx_fmt_ee (code, VOIDmode, tmp, const0_rtx); + tmp = gen_rtx_COMPARE (CCFPmode, op0, op1); + if (unordered_compare) + tmp = gen_rtx_UNSPEC (CCFPmode, gen_rtvec (1, tmp), UNSPEC_NOTRAP); + reg = gen_rtx_REG (CCFPmode, FLAGS_REG); + emit_insn (gen_rtx_SET (reg, tmp)); + tmp = gen_rtx_fmt_ee (code, VOIDmode, reg, const0_rtx); tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp, gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx); tmp = emit_jump_insn (gen_rtx_SET (pc_rtx, tmp)); @@ -43956,8 +44566,7 @@ emit_move_insn (op0, ireg); } -/* Expand rint (IEEE round to nearest) rounding OPERAND1 and storing the - result in OPERAND0. */ +/* Expand rint rounding OPERAND1 and storing the result in OPERAND0. */ void ix86_expand_rint (rtx operand0, rtx operand1) { @@ -43965,11 +44574,17 @@ xa = fabs (operand1); if (!isless (xa, 2**52)) return operand1; - xa = xa + 2**52 - 2**52; + two52 = 2**52; + if (flag_rounding_math) + { + two52 = copysign (two52, operand1); + xa = operand1; + } + xa = xa + two52 - two52; return copysign (xa, operand1); */ machine_mode mode = GET_MODE (operand0); - rtx res, xa, TWO52, mask; + rtx res, xa, TWO52, two52, mask; rtx_code_label *label; res = gen_reg_rtx (mode); @@ -43982,8 +44597,16 @@ TWO52 = ix86_gen_TWO52 (mode); label = ix86_expand_sse_compare_and_jump (UNLE, TWO52, xa, false); - xa = expand_simple_binop (mode, PLUS, xa, TWO52, NULL_RTX, 0, OPTAB_DIRECT); - xa = expand_simple_binop (mode, MINUS, xa, TWO52, xa, 0, OPTAB_DIRECT); + two52 = TWO52; + if (flag_rounding_math) + { + two52 = gen_reg_rtx (mode); + ix86_sse_copysign_to_positive (two52, TWO52, res, mask); + xa = res; + } + + xa = expand_simple_binop (mode, PLUS, xa, two52, NULL_RTX, 0, OPTAB_DIRECT); + xa = expand_simple_binop (mode, MINUS, xa, two52, xa, 0, OPTAB_DIRECT); ix86_sse_copysign_to_positive (res, xa, res, mask); @@ -44393,70 +45016,80 @@ /* Table of valid machine attributes. */ static const struct attribute_spec ix86_attribute_table[] = { - /* { name, min_len, max_len, decl_req, type_req, fn_type_req, handler, - affects_type_identity } */ + /* { name, min_len, max_len, decl_req, type_req, fn_type_req, + affects_type_identity, handler, exclude } */ /* Stdcall attribute says callee is responsible for popping arguments if they are not variable. */ - { "stdcall", 0, 0, false, true, true, ix86_handle_cconv_attribute, - true }, + { "stdcall", 0, 0, false, true, true, true, ix86_handle_cconv_attribute, + NULL }, /* Fastcall attribute says callee is responsible for popping arguments if they are not variable. */ - { "fastcall", 0, 0, false, true, true, ix86_handle_cconv_attribute, - true }, + { "fastcall", 0, 0, false, true, true, true, ix86_handle_cconv_attribute, + NULL }, /* Thiscall attribute says callee is responsible for popping arguments if they are not variable. */ - { "thiscall", 0, 0, false, true, true, ix86_handle_cconv_attribute, - true }, + { "thiscall", 0, 0, false, true, true, true, ix86_handle_cconv_attribute, + NULL }, /* Cdecl attribute says the callee is a normal C declaration */ - { "cdecl", 0, 0, false, true, true, ix86_handle_cconv_attribute, - true }, + { "cdecl", 0, 0, false, true, true, true, ix86_handle_cconv_attribute, + NULL }, /* Regparm attribute specifies how many integer arguments are to be passed in registers. */ - { "regparm", 1, 1, false, true, true, ix86_handle_cconv_attribute, - true }, + { "regparm", 1, 1, false, true, true, true, ix86_handle_cconv_attribute, + NULL }, /* Sseregparm attribute says we are using x86_64 calling conventions for FP arguments. */ - { "sseregparm", 0, 0, false, true, true, ix86_handle_cconv_attribute, - true }, + { "sseregparm", 0, 0, false, true, true, true, ix86_handle_cconv_attribute, + NULL }, /* The transactional memory builtins are implicitly regparm or fastcall depending on the ABI. Override the generic do-nothing attribute that these builtins were declared with. */ - { "*tm regparm", 0, 0, false, true, true, ix86_handle_tm_regparm_attribute, - true }, + { "*tm regparm", 0, 0, false, true, true, true, + ix86_handle_tm_regparm_attribute, NULL }, /* force_align_arg_pointer says this function realigns the stack at entry. */ { (const char *)&ix86_force_align_arg_pointer_string, 0, 0, - false, true, true, ix86_handle_force_align_arg_pointer_attribute, false }, + false, true, true, false, ix86_handle_force_align_arg_pointer_attribute, + NULL }, #if TARGET_DLLIMPORT_DECL_ATTRIBUTES - { "dllimport", 0, 0, false, false, false, handle_dll_attribute, false }, - { "dllexport", 0, 0, false, false, false, handle_dll_attribute, false }, - { "shared", 0, 0, true, false, false, ix86_handle_shared_attribute, - false }, -#endif - { "ms_struct", 0, 0, false, false, false, ix86_handle_struct_attribute, - false }, - { "gcc_struct", 0, 0, false, false, false, ix86_handle_struct_attribute, - false }, + { "dllimport", 0, 0, false, false, false, false, handle_dll_attribute, + NULL }, + { "dllexport", 0, 0, false, false, false, false, handle_dll_attribute, + NULL }, + { "shared", 0, 0, true, false, false, false, + ix86_handle_shared_attribute, NULL }, +#endif + { "ms_struct", 0, 0, false, false, false, false, + ix86_handle_struct_attribute, NULL }, + { "gcc_struct", 0, 0, false, false, false, false, + ix86_handle_struct_attribute, NULL }, #ifdef SUBTARGET_ATTRIBUTE_TABLE SUBTARGET_ATTRIBUTE_TABLE, #endif /* ms_abi and sysv_abi calling convention function attributes. */ - { "ms_abi", 0, 0, false, true, true, ix86_handle_abi_attribute, true }, - { "sysv_abi", 0, 0, false, true, true, ix86_handle_abi_attribute, true }, - { "ms_abi va_list", 0, 0, false, false, false, NULL, false }, - { "sysv_abi va_list", 0, 0, false, false, false, NULL, false }, - { "ms_hook_prologue", 0, 0, true, false, false, ix86_handle_fndecl_attribute, - false }, - { "callee_pop_aggregate_return", 1, 1, false, true, true, - ix86_handle_callee_pop_aggregate_return, true }, - { "interrupt", 0, 0, false, true, true, - ix86_handle_interrupt_attribute, false }, - { "no_caller_saved_registers", 0, 0, false, true, true, - ix86_handle_no_caller_saved_registers_attribute, false }, - { "naked", 0, 0, true, false, false, - ix86_handle_fndecl_attribute, false }, + { "ms_abi", 0, 0, false, true, true, true, ix86_handle_abi_attribute, NULL }, + { "sysv_abi", 0, 0, false, true, true, true, ix86_handle_abi_attribute, + NULL }, + { "ms_abi va_list", 0, 0, false, false, false, false, NULL, NULL }, + { "sysv_abi va_list", 0, 0, false, false, false, false, NULL, NULL }, + { "ms_hook_prologue", 0, 0, true, false, false, false, + ix86_handle_fndecl_attribute, NULL }, + { "callee_pop_aggregate_return", 1, 1, false, true, true, true, + ix86_handle_callee_pop_aggregate_return, NULL }, + { "interrupt", 0, 0, false, true, true, false, + ix86_handle_interrupt_attribute, NULL }, + { "no_caller_saved_registers", 0, 0, false, true, true, false, + ix86_handle_no_caller_saved_registers_attribute, NULL }, + { "naked", 0, 0, true, false, false, false, + ix86_handle_fndecl_attribute, NULL }, + { "indirect_branch", 1, 1, true, false, false, false, + ix86_handle_fndecl_attribute, NULL }, + { "function_return", 1, 1, true, false, false, false, + ix86_handle_fndecl_attribute, NULL }, + { "indirect_return", 0, 0, false, true, true, false, + NULL, NULL }, /* End element. */ - { NULL, 0, 0, false, false, false, NULL, false } + { NULL, 0, 0, false, false, false, false, NULL, NULL } }; /* Implement targetm.vectorize.builtin_vectorization_cost. */ @@ -44490,57 +45123,55 @@ case vector_stmt: return ix86_vec_cost (mode, - fp ? ix86_cost->addss : ix86_cost->sse_op, - true); + fp ? ix86_cost->addss : ix86_cost->sse_op); case vector_load: index = sse_store_index (mode); - gcc_assert (index >= 0); - return ix86_vec_cost (mode, - COSTS_N_INSNS (ix86_cost->sse_load[index]) / 2, - true); + /* See PR82713 - we may end up being called on non-vector type. */ + if (index < 0) + index = 2; + return COSTS_N_INSNS (ix86_cost->sse_load[index]) / 2; case vector_store: index = sse_store_index (mode); - return ix86_vec_cost (mode, - COSTS_N_INSNS (ix86_cost->sse_store[index]) / 2, - true); + /* See PR82713 - we may end up being called on non-vector type. */ + if (index < 0) + index = 2; + return COSTS_N_INSNS (ix86_cost->sse_store[index]) / 2; case vec_to_scalar: case scalar_to_vec: - return ix86_vec_cost (mode, ix86_cost->sse_op, true); + return ix86_vec_cost (mode, ix86_cost->sse_op); /* We should have separate costs for unaligned loads and gather/scatter. Do that incrementally. */ case unaligned_load: index = sse_store_index (mode); - return ix86_vec_cost (mode, - COSTS_N_INSNS - (ix86_cost->sse_unaligned_load[index]) / 2, - true); + /* See PR82713 - we may end up being called on non-vector type. */ + if (index < 0) + index = 2; + return COSTS_N_INSNS (ix86_cost->sse_unaligned_load[index]) / 2; case unaligned_store: index = sse_store_index (mode); - return ix86_vec_cost (mode, - COSTS_N_INSNS - (ix86_cost->sse_unaligned_store[index]) / 2, - true); + /* See PR82713 - we may end up being called on non-vector type. */ + if (index < 0) + index = 2; + return COSTS_N_INSNS (ix86_cost->sse_unaligned_store[index]) / 2; case vector_gather_load: return ix86_vec_cost (mode, COSTS_N_INSNS (ix86_cost->gather_static + ix86_cost->gather_per_elt - * TYPE_VECTOR_SUBPARTS (vectype)) / 2, - true); + * TYPE_VECTOR_SUBPARTS (vectype)) / 2); case vector_scatter_store: return ix86_vec_cost (mode, COSTS_N_INSNS (ix86_cost->scatter_static + ix86_cost->scatter_per_elt - * TYPE_VECTOR_SUBPARTS (vectype)) / 2, - true); + * TYPE_VECTOR_SUBPARTS (vectype)) / 2); case cond_branch_taken: return ix86_cost->cond_taken_branch_cost; @@ -44550,11 +45181,21 @@ case vec_perm: case vec_promote_demote: - return ix86_vec_cost (mode, - ix86_cost->sse_op, true); + return ix86_vec_cost (mode, ix86_cost->sse_op); case vec_construct: - return ix86_vec_cost (mode, ix86_cost->sse_op, false); + { + /* N element inserts into SSE vectors. */ + int cost = TYPE_VECTOR_SUBPARTS (vectype) * ix86_cost->sse_op; + /* One vinserti128 for combining two SSE vectors for AVX256. */ + if (GET_MODE_BITSIZE (mode) == 256) + cost += ix86_vec_cost (mode, ix86_cost->addss); + /* One vinserti64x4 and two vinserti128 for combining SSE + and AVX256 vectors to AVX512. */ + else if (GET_MODE_BITSIZE (mode) == 512) + cost += 3 * ix86_vec_cost (mode, ix86_cost->addss); + return cost; + } default: gcc_unreachable (); @@ -44647,6 +45288,42 @@ } /* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D + using movss or movsd. */ +static bool +expand_vec_perm_movs (struct expand_vec_perm_d *d) +{ + machine_mode vmode = d->vmode; + unsigned i, nelt = d->nelt; + rtx x; + + if (d->one_operand_p) + return false; + + if (!(TARGET_SSE && vmode == V4SFmode) + && !(TARGET_SSE2 && vmode == V2DFmode)) + return false; + + /* Only the first element is changed. */ + if (d->perm[0] != nelt && d->perm[0] != 0) + return false; + for (i = 1; i < nelt; ++i) + if (d->perm[i] != i + nelt - d->perm[0]) + return false; + + if (d->testing_p) + return true; + + if (d->perm[0] == nelt) + x = gen_rtx_VEC_MERGE (vmode, d->op1, d->op0, GEN_INT (1)); + else + x = gen_rtx_VEC_MERGE (vmode, d->op0, d->op1, GEN_INT (1)); + + emit_insn (gen_rtx_SET (d->target, x)); + + return true; +} + +/* A subroutine of ix86_expand_vec_perm_builtin_1. Try to implement D in terms of blendp[sd] / pblendw / pblendvb / vpblendd. */ static bool @@ -45388,6 +46065,10 @@ } } + /* Try movss/movsd instructions. */ + if (expand_vec_perm_movs (d)) + return true; + /* Finally, try the fully general two operand permute. */ if (expand_vselect_vconcat (d->target, d->op0, d->op1, d->perm, nelt, d->testing_p)) @@ -46596,7 +47277,7 @@ static bool expand_vec_perm_even_odd_pack (struct expand_vec_perm_d *d) { - rtx op, dop0, dop1, t, rperm[16]; + rtx op, dop0, dop1, t; unsigned i, odd, c, s, nelt = d->nelt; bool end_perm = false; machine_mode half_mode; @@ -46673,9 +47354,7 @@ dop1 = gen_reg_rtx (half_mode); if (odd == 0) { - for (i = 0; i < nelt / 2; i++) - rperm[i] = GEN_INT (c); - t = gen_rtx_CONST_VECTOR (half_mode, gen_rtvec_v (nelt / 2, rperm)); + t = gen_const_vec_duplicate (half_mode, GEN_INT (c)); t = force_reg (half_mode, t); emit_insn (gen_and (dop0, t, gen_lowpart (half_mode, d->op0))); emit_insn (gen_and (dop1, t, gen_lowpart (half_mode, d->op1))); @@ -47278,9 +47957,8 @@ return true; } -/* The guts of ix86_expand_vec_perm_const, also used by the ok hook. - With all of the interface bits taken care of, perform the expansion - in D and return true on success. */ +/* The guts of ix86_vectorize_vec_perm_const. With all of the interface bits + taken care of, perform the expansion in D and return true on success. */ static bool ix86_expand_vec_perm_const_1 (struct expand_vec_perm_d *d) @@ -47415,35 +48093,144 @@ return (which == 3); } -bool -ix86_expand_vec_perm_const (rtx operands[4]) +/* Implement TARGET_VECTORIZE_VEC_PERM_CONST. */ + +static bool +ix86_vectorize_vec_perm_const (machine_mode vmode, rtx target, rtx op0, + rtx op1, const vec_perm_indices &sel) { struct expand_vec_perm_d d; unsigned char perm[MAX_VECT_LEN]; - int i, nelt; + unsigned int i, nelt, which; bool two_args; - rtx sel; - - d.target = operands[0]; - d.op0 = operands[1]; - d.op1 = operands[2]; - sel = operands[3]; - - d.vmode = GET_MODE (d.target); + + d.target = target; + d.op0 = op0; + d.op1 = op1; + + d.vmode = vmode; gcc_assert (VECTOR_MODE_P (d.vmode)); d.nelt = nelt = GET_MODE_NUNITS (d.vmode); - d.testing_p = false; - - gcc_assert (GET_CODE (sel) == CONST_VECTOR); - gcc_assert (XVECLEN (sel, 0) == nelt); + d.testing_p = !target; + + gcc_assert (sel.length () == nelt); gcc_checking_assert (sizeof (d.perm) == sizeof (perm)); - for (i = 0; i < nelt; ++i) - { - rtx e = XVECEXP (sel, 0, i); - int ei = INTVAL (e) & (2 * nelt - 1); - d.perm[i] = ei; - perm[i] = ei; + /* Given sufficient ISA support we can just return true here + for selected vector modes. */ + switch (d.vmode) + { + case E_V16SFmode: + case E_V16SImode: + case E_V8DImode: + case E_V8DFmode: + if (!TARGET_AVX512F) + return false; + /* All implementable with a single vperm[it]2 insn. */ + if (d.testing_p) + return true; + break; + case E_V32HImode: + if (!TARGET_AVX512BW) + return false; + if (d.testing_p) + /* All implementable with a single vperm[it]2 insn. */ + return true; + break; + case E_V64QImode: + if (!TARGET_AVX512BW) + return false; + if (d.testing_p) + /* Implementable with 2 vperm[it]2, 2 vpshufb and 1 or insn. */ + return true; + break; + case E_V8SImode: + case E_V8SFmode: + case E_V4DFmode: + case E_V4DImode: + if (!TARGET_AVX) + return false; + if (d.testing_p && TARGET_AVX512VL) + /* All implementable with a single vperm[it]2 insn. */ + return true; + break; + case E_V16HImode: + if (!TARGET_SSE2) + return false; + if (d.testing_p && TARGET_AVX2) + /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */ + return true; + break; + case E_V32QImode: + if (!TARGET_SSE2) + return false; + if (d.testing_p && TARGET_AVX2) + /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */ + return true; + break; + case E_V8HImode: + case E_V16QImode: + if (!TARGET_SSE2) + return false; + /* Fall through. */ + case E_V4SImode: + case E_V4SFmode: + if (!TARGET_SSE) + return false; + /* All implementable with a single vpperm insn. */ + if (d.testing_p && TARGET_XOP) + return true; + /* All implementable with 2 pshufb + 1 ior. */ + if (d.testing_p && TARGET_SSSE3) + return true; + break; + case E_V2DImode: + case E_V2DFmode: + if (!TARGET_SSE) + return false; + /* All implementable with shufpd or unpck[lh]pd. */ + if (d.testing_p) + return true; + break; + default: + return false; + } + + for (i = which = 0; i < nelt; ++i) + { + unsigned char e = sel[i]; + gcc_assert (e < 2 * nelt); + d.perm[i] = e; + perm[i] = e; + which |= (e < nelt ? 1 : 2); + } + + if (d.testing_p) + { + /* For all elements from second vector, fold the elements to first. */ + if (which == 2) + for (i = 0; i < nelt; ++i) + d.perm[i] -= nelt; + + /* Check whether the mask can be applied to the vector type. */ + d.one_operand_p = (which != 3); + + /* Implementable with shufps or pshufd. */ + if (d.one_operand_p && (d.vmode == V4SFmode || d.vmode == V4SImode)) + return true; + + /* Otherwise we have to go through the motions and see if we can + figure out how to generate the requested permutation. */ + d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1); + d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2); + if (!d.one_operand_p) + d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3); + + start_sequence (); + bool ret = ix86_expand_vec_perm_const_1 (&d); + end_sequence (); + + return ret; } two_args = canonicalize_perm (&d); @@ -47465,114 +48252,6 @@ return false; } -/* Implement targetm.vectorize.vec_perm_const_ok. */ - -static bool -ix86_vectorize_vec_perm_const_ok (machine_mode vmode, vec_perm_indices sel) -{ - struct expand_vec_perm_d d; - unsigned int i, nelt, which; - bool ret; - - d.vmode = vmode; - d.nelt = nelt = GET_MODE_NUNITS (d.vmode); - d.testing_p = true; - - /* Given sufficient ISA support we can just return true here - for selected vector modes. */ - switch (d.vmode) - { - case E_V16SFmode: - case E_V16SImode: - case E_V8DImode: - case E_V8DFmode: - if (TARGET_AVX512F) - /* All implementable with a single vperm[it]2 insn. */ - return true; - break; - case E_V32HImode: - if (TARGET_AVX512BW) - /* All implementable with a single vperm[it]2 insn. */ - return true; - break; - case E_V64QImode: - if (TARGET_AVX512BW) - /* Implementable with 2 vperm[it]2, 2 vpshufb and 1 or insn. */ - return true; - break; - case E_V8SImode: - case E_V8SFmode: - case E_V4DFmode: - case E_V4DImode: - if (TARGET_AVX512VL) - /* All implementable with a single vperm[it]2 insn. */ - return true; - break; - case E_V16HImode: - if (TARGET_AVX2) - /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */ - return true; - break; - case E_V32QImode: - if (TARGET_AVX2) - /* Implementable with 4 vpshufb insns, 2 vpermq and 3 vpor insns. */ - return true; - break; - case E_V4SImode: - case E_V4SFmode: - case E_V8HImode: - case E_V16QImode: - /* All implementable with a single vpperm insn. */ - if (TARGET_XOP) - return true; - /* All implementable with 2 pshufb + 1 ior. */ - if (TARGET_SSSE3) - return true; - break; - case E_V2DImode: - case E_V2DFmode: - /* All implementable with shufpd or unpck[lh]pd. */ - return true; - default: - return false; - } - - /* Extract the values from the vector CST into the permutation - array in D. */ - for (i = which = 0; i < nelt; ++i) - { - unsigned char e = sel[i]; - gcc_assert (e < 2 * nelt); - d.perm[i] = e; - which |= (e < nelt ? 1 : 2); - } - - /* For all elements from second vector, fold the elements to first. */ - if (which == 2) - for (i = 0; i < nelt; ++i) - d.perm[i] -= nelt; - - /* Check whether the mask can be applied to the vector type. */ - d.one_operand_p = (which != 3); - - /* Implementable with shufps or pshufd. */ - if (d.one_operand_p && (d.vmode == V4SFmode || d.vmode == V4SImode)) - return true; - - /* Otherwise we have to go through the motions and see if we can - figure out how to generate the requested permutation. */ - d.target = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 1); - d.op1 = d.op0 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 2); - if (!d.one_operand_p) - d.op1 = gen_raw_REG (d.vmode, LAST_VIRTUAL_REGISTER + 3); - - start_sequence (); - ret = ix86_expand_vec_perm_const_1 (&d); - end_sequence (); - - return ret; -} - void ix86_expand_vec_extract_even_odd (rtx targ, rtx op0, rtx op1, unsigned odd) { @@ -47675,7 +48354,6 @@ op2_h = gen_reg_rtx (qimode); emit_insn (gen_il (op2_l, op2, op2)); emit_insn (gen_ih (op2_h, op2, op2)); - /* FALLTHRU */ op1_l = gen_reg_rtx (qimode); op1_h = gen_reg_rtx (qimode); @@ -48083,33 +48761,12 @@ } /* Return 1 if control tansfer instruction INSN - should be encoded with bnd prefix. - If insn is NULL then return 1 when control - transfer instructions should be prefixed with - bnd by default for current function. */ - -bool -ix86_bnd_prefixed_insn_p (rtx insn) -{ - /* For call insns check special flag. */ - if (insn && CALL_P (insn)) - { - rtx call = get_call_rtx_from (insn); - if (call) - return CALL_EXPR_WITH_BOUNDS_P (call); - } - - /* All other insns are prefixed only if function is instrumented. */ - return chkp_function_instrumented_p (current_function_decl); -} - -/* Return 1 if control tansfer instruction INSN should be encoded with notrack prefix. */ static bool ix86_notrack_prefixed_insn_p (rtx insn) { - if (!insn || !((flag_cf_protection & CF_BRANCH) && TARGET_IBT)) + if (!insn || !((flag_cf_protection & CF_BRANCH))) return false; if (CALL_P (insn)) @@ -48153,39 +48810,70 @@ switch (mode) { + case E_V2DImode: + case E_V4DImode: + /* For 64-bit signed integer X, with SSE4.2 use + pxor t0, t0; pcmpgtq X, t0; pxor t0, X; psubq t0, X. + Otherwise handle it similarly to V4SImode, except use 64 as W instead of + 32 and use logical instead of arithmetic right shift (which is + unimplemented) and subtract. */ + if (TARGET_SSE4_2) + { + tmp0 = gen_reg_rtx (mode); + tmp1 = gen_reg_rtx (mode); + emit_move_insn (tmp1, CONST0_RTX (mode)); + if (mode == E_V2DImode) + emit_insn (gen_sse4_2_gtv2di3 (tmp0, tmp1, input)); + else + emit_insn (gen_avx2_gtv4di3 (tmp0, tmp1, input)); + } + else + { + tmp0 = expand_simple_binop (mode, LSHIFTRT, input, + GEN_INT (GET_MODE_UNIT_BITSIZE (mode) + - 1), NULL, 0, OPTAB_DIRECT); + tmp0 = expand_simple_unop (mode, NEG, tmp0, NULL, false); + } + + tmp1 = expand_simple_binop (mode, XOR, tmp0, input, + NULL, 0, OPTAB_DIRECT); + x = expand_simple_binop (mode, MINUS, tmp1, tmp0, + target, 0, OPTAB_DIRECT); + break; + + case E_V4SImode: /* For 32-bit signed integer X, the best way to calculate the absolute value of X is (((signed) X >> (W-1)) ^ X) - ((signed) X >> (W-1)). */ - case E_V4SImode: - tmp0 = expand_simple_binop (mode, ASHIFTRT, input, - GEN_INT (GET_MODE_UNIT_BITSIZE (mode) - 1), - NULL, 0, OPTAB_DIRECT); - tmp1 = expand_simple_binop (mode, XOR, tmp0, input, - NULL, 0, OPTAB_DIRECT); - x = expand_simple_binop (mode, MINUS, tmp1, tmp0, - target, 0, OPTAB_DIRECT); - break; - + tmp0 = expand_simple_binop (mode, ASHIFTRT, input, + GEN_INT (GET_MODE_UNIT_BITSIZE (mode) - 1), + NULL, 0, OPTAB_DIRECT); + tmp1 = expand_simple_binop (mode, XOR, tmp0, input, + NULL, 0, OPTAB_DIRECT); + x = expand_simple_binop (mode, MINUS, tmp1, tmp0, + target, 0, OPTAB_DIRECT); + break; + + case E_V8HImode: /* For 16-bit signed integer X, the best way to calculate the absolute value of X is max (X, -X), as SSE2 provides the PMAXSW insn. */ - case E_V8HImode: - tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0); - - x = expand_simple_binop (mode, SMAX, tmp0, input, - target, 0, OPTAB_DIRECT); - break; - + tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0); + + x = expand_simple_binop (mode, SMAX, tmp0, input, + target, 0, OPTAB_DIRECT); + break; + + case E_V16QImode: /* For 8-bit signed integer X, the best way to calculate the absolute value of X is min ((unsigned char) X, (unsigned char) (-X)), as SSE2 provides the PMINUB insn. */ - case E_V16QImode: - tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0); - - x = expand_simple_binop (V16QImode, UMIN, tmp0, input, - target, 0, OPTAB_DIRECT); - break; - - default: - gcc_unreachable (); + tmp0 = expand_unop (mode, neg_optab, input, NULL_RTX, 0); + + x = expand_simple_binop (V16QImode, UMIN, tmp0, input, + target, 0, OPTAB_DIRECT); + break; + + default: + gcc_unreachable (); } if (x != target) @@ -48612,27 +49300,63 @@ } } +/* All CPUs prefer to avoid cross-lane operations so perform reductions + upper against lower halves up to SSE reg size. */ + +static machine_mode +ix86_split_reduction (machine_mode mode) +{ + /* Reduce lowpart against highpart until we reach SSE reg width to + avoid cross-lane operations. */ + switch (mode) + { + case E_V8DImode: + case E_V4DImode: + return V2DImode; + case E_V16SImode: + case E_V8SImode: + return V4SImode; + case E_V32HImode: + case E_V16HImode: + return V8HImode; + case E_V64QImode: + case E_V32QImode: + return V16QImode; + case E_V16SFmode: + case E_V8SFmode: + return V4SFmode; + case E_V8DFmode: + case E_V4DFmode: + return V2DFmode; + default: + return mode; + } +} + /* If AVX is enabled then try vectorizing with both 256bit and 128bit vectors. If AVX512F is enabled then try vectorizing with 512bit, 256bit and 128bit vectors. */ -static unsigned int -ix86_autovectorize_vector_sizes (void) -{ - unsigned int bytesizes = 0; - +static void +ix86_autovectorize_vector_sizes (vector_sizes *sizes) +{ if (TARGET_AVX512F && !TARGET_PREFER_AVX256) - bytesizes |= (64 | 32 | 16); + { + sizes->safe_push (64); + sizes->safe_push (32); + sizes->safe_push (16); + } else if (TARGET_AVX && !TARGET_PREFER_AVX128) - bytesizes |= (32 | 16); - - return bytesizes; + { + sizes->safe_push (32); + sizes->safe_push (16); + } } /* Implemenation of targetm.vectorize.get_mask_mode. */ static opt_machine_mode -ix86_get_mask_mode (unsigned nunits, unsigned vector_size) +ix86_get_mask_mode (poly_uint64 nunits, poly_uint64 vector_size) { unsigned elem_size = vector_size / nunits; @@ -48749,9 +49473,151 @@ { unsigned *cost = (unsigned *) data; unsigned retval = 0; + bool scalar_p + = (kind == scalar_stmt || kind == scalar_load || kind == scalar_store); tree vectype = stmt_info ? stmt_vectype (stmt_info) : NULL_TREE; - int stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign); + int stmt_cost = - 1; + + bool fp = false; + machine_mode mode = scalar_p ? SImode : TImode; + + if (vectype != NULL) + { + fp = FLOAT_TYPE_P (vectype); + mode = TYPE_MODE (vectype); + if (scalar_p) + mode = TYPE_MODE (TREE_TYPE (vectype)); + } + + if ((kind == vector_stmt || kind == scalar_stmt) + && stmt_info + && stmt_info->stmt && gimple_code (stmt_info->stmt) == GIMPLE_ASSIGN) + { + tree_code subcode = gimple_assign_rhs_code (stmt_info->stmt); + /*machine_mode inner_mode = mode; + if (VECTOR_MODE_P (mode)) + inner_mode = GET_MODE_INNER (mode);*/ + + switch (subcode) + { + case PLUS_EXPR: + case POINTER_PLUS_EXPR: + case MINUS_EXPR: + if (kind == scalar_stmt) + { + if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH) + stmt_cost = ix86_cost->addss; + else if (X87_FLOAT_MODE_P (mode)) + stmt_cost = ix86_cost->fadd; + else + stmt_cost = ix86_cost->add; + } + else + stmt_cost = ix86_vec_cost (mode, fp ? ix86_cost->addss + : ix86_cost->sse_op); + break; + + case MULT_EXPR: + case WIDEN_MULT_EXPR: + case MULT_HIGHPART_EXPR: + stmt_cost = ix86_multiplication_cost (ix86_cost, mode); + break; + case NEGATE_EXPR: + if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH) + stmt_cost = ix86_cost->sse_op; + else if (X87_FLOAT_MODE_P (mode)) + stmt_cost = ix86_cost->fchs; + else if (VECTOR_MODE_P (mode)) + stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op); + else + stmt_cost = ix86_cost->add; + break; + case TRUNC_DIV_EXPR: + case CEIL_DIV_EXPR: + case FLOOR_DIV_EXPR: + case ROUND_DIV_EXPR: + case TRUNC_MOD_EXPR: + case CEIL_MOD_EXPR: + case FLOOR_MOD_EXPR: + case RDIV_EXPR: + case ROUND_MOD_EXPR: + case EXACT_DIV_EXPR: + stmt_cost = ix86_division_cost (ix86_cost, mode); + break; + + case RSHIFT_EXPR: + case LSHIFT_EXPR: + case LROTATE_EXPR: + case RROTATE_EXPR: + { + tree op2 = gimple_assign_rhs2 (stmt_info->stmt); + stmt_cost = ix86_shift_rotate_cost + (ix86_cost, mode, + TREE_CODE (op2) == INTEGER_CST, + cst_and_fits_in_hwi (op2) ? int_cst_value (op2) : -1, + true, false, false, NULL, NULL); + } + break; + case NOP_EXPR: + /* Only sign-conversions are free. */ + if (tree_nop_conversion_p + (TREE_TYPE (gimple_assign_lhs (stmt_info->stmt)), + TREE_TYPE (gimple_assign_rhs1 (stmt_info->stmt)))) + stmt_cost = 0; + break; + + case BIT_IOR_EXPR: + case ABS_EXPR: + case ABSU_EXPR: + case MIN_EXPR: + case MAX_EXPR: + case BIT_XOR_EXPR: + case BIT_AND_EXPR: + case BIT_NOT_EXPR: + if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH) + stmt_cost = ix86_cost->sse_op; + else if (VECTOR_MODE_P (mode)) + stmt_cost = ix86_vec_cost (mode, ix86_cost->sse_op); + else + stmt_cost = ix86_cost->add; + break; + default: + break; + } + } + + combined_fn cfn; + if ((kind == vector_stmt || kind == scalar_stmt) + && stmt_info + && stmt_info->stmt + && (cfn = gimple_call_combined_fn (stmt_info->stmt)) != CFN_LAST) + switch (cfn) + { + case CFN_FMA: + stmt_cost = ix86_vec_cost (mode, + mode == SFmode ? ix86_cost->fmass + : ix86_cost->fmasd); + break; + default: + break; + } + + /* If we do elementwise loads into a vector then we are bound by + latency and execution resources for the many scalar loads + (AGU and load ports). Try to account for this by scaling the + construction cost by the number of elements involved. */ + if (kind == vec_construct + && stmt_info + && STMT_VINFO_TYPE (stmt_info) == load_vec_info_type + && STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info) == VMAT_ELEMENTWISE + && TREE_CODE (DR_STEP (STMT_VINFO_DATA_REF (stmt_info))) != INTEGER_CST) + { + stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign); + stmt_cost *= TYPE_VECTOR_SUBPARTS (vectype); + } + if (stmt_cost == -1) + stmt_cost = ix86_builtin_vectorization_cost (kind, vectype, misalign); /* Penalize DFmode vector operations for Bonnell. */ if (TARGET_BONNELL && kind == vector_stmt @@ -48769,8 +49635,8 @@ /* We need to multiply all vector stmt cost by 1.7 (estimated cost) for Silvermont as it has out of order integer pipeline and can execute 2 scalar instruction per tick, but has in order SIMD pipeline. */ - if ((TARGET_SILVERMONT || TARGET_INTEL) - && stmt_info && stmt_info->stmt) + if ((TARGET_SILVERMONT || TARGET_GOLDMONT || TARGET_GOLDMONT_PLUS + || TARGET_TREMONT || TARGET_INTEL) && stmt_info && stmt_info->stmt) { tree lhs_op = gimple_get_lhs (stmt_info->stmt); if (lhs_op && TREE_CODE (TREE_TYPE (lhs_op)) == INTEGER_TYPE) @@ -48815,7 +49681,7 @@ || ((val & IX86_HLE_ACQUIRE) && (val & IX86_HLE_RELEASE))) { warning (OPT_Winvalid_memory_model, - "Unknown architecture specific memory model"); + "unknown architecture specific memory model"); return MEMMODEL_SEQ_CST; } strong = (is_mm_acq_rel (model) || is_mm_seq_cst (model)); @@ -48871,7 +49737,7 @@ break; default: warning_at (DECL_SOURCE_LOCATION (node->decl), 0, - "unsupported return type %qT for simd\n", ret_type); + "unsupported return type %qT for simd", ret_type); return 0; } @@ -48893,16 +49759,11 @@ break; default: warning_at (DECL_SOURCE_LOCATION (node->decl), 0, - "unsupported argument type %qT for simd\n", TREE_TYPE (t)); + "unsupported argument type %qT for simd", TREE_TYPE (t)); return 0; } - if (clonei->cilk_elemental) - { - /* Parse here processor clause. If not present, default to 'b'. */ - clonei->vecsize_mangle = 'b'; - } - else if (!TREE_PUBLIC (node->decl)) + if (!TREE_PUBLIC (node->decl)) { /* If the function isn't exported, we can pick up just one ISA for the clones. */ @@ -49092,7 +49953,7 @@ free (bbs); if (mem_count && mem_count <=32) - return 32/mem_count; + return MIN (nunroll, 32 / mem_count); return nunroll; } @@ -49208,73 +50069,6 @@ atomic_feraiseexcept_call); } -/* Return mode to be used for bounds or VOIDmode - if bounds are not supported. */ - -static machine_mode -ix86_mpx_bound_mode () -{ - /* Do not support pointer checker if MPX - is not enabled. */ - if (!TARGET_MPX) - { - if (flag_check_pointer_bounds) - warning (0, "Pointer Checker requires MPX support on this target." - " Use -mmpx options to enable MPX."); - return VOIDmode; - } - - return BNDmode; -} - -/* Return constant used to statically initialize constant bounds. - - This function is used to create special bound values. For now - only INIT bounds and NONE bounds are expected. More special - values may be added later. */ - -static tree -ix86_make_bounds_constant (HOST_WIDE_INT lb, HOST_WIDE_INT ub) -{ - tree low = lb ? build_minus_one_cst (pointer_sized_int_node) - : build_zero_cst (pointer_sized_int_node); - tree high = ub ? build_zero_cst (pointer_sized_int_node) - : build_minus_one_cst (pointer_sized_int_node); - - /* This function is supposed to be used to create INIT and - NONE bounds only. */ - gcc_assert ((lb == 0 && ub == -1) - || (lb == -1 && ub == 0)); - - return build_complex (NULL, low, high); -} - -/* Generate a list of statements STMTS to initialize pointer bounds - variable VAR with bounds LB and UB. Return the number of generated - statements. */ - -static int -ix86_initialize_bounds (tree var, tree lb, tree ub, tree *stmts) -{ - tree bnd_ptr = build_pointer_type (pointer_sized_int_node); - tree lhs, modify, var_p; - - ub = build1 (BIT_NOT_EXPR, pointer_sized_int_node, ub); - var_p = fold_convert (bnd_ptr, build_fold_addr_expr (var)); - - lhs = build1 (INDIRECT_REF, pointer_sized_int_node, var_p); - modify = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, lb); - append_to_statement_list (modify, stmts); - - lhs = build1 (INDIRECT_REF, pointer_sized_int_node, - build2 (POINTER_PLUS_EXPR, bnd_ptr, var_p, - TYPE_SIZE_UNIT (pointer_sized_int_node))); - modify = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, ub); - append_to_statement_list (modify, stmts); - - return 2; -} - #if !TARGET_MACHO && !TARGET_DLLIMPORT_DECL_ATTRIBUTES /* For i386, common symbol is local only for non-PIE binaries. For x86-64, common symbol is local only for non-PIE binaries or linker @@ -49530,6 +50324,19 @@ return FLT_EVAL_METHOD_UNPREDICTABLE; } +/* Implement PUSH_ROUNDING. On 386, we have pushw instruction that + decrements by exactly 2 no matter what the position was, there is no pushb. + + But as CIE data alignment factor on this arch is -4 for 32bit targets + and -8 for 64bit targets, we need to make sure all stack pointer adjustments + are in multiple of 4 for 32bit targets and 8 for 64bit targets. */ + +poly_int64 +ix86_push_rounding (poly_int64 bytes) +{ + return ROUND_UP (bytes, UNITS_PER_WORD); +} + /* Target-specific selftests. */ #if CHECKING_P @@ -50118,12 +50925,14 @@ #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \ ix86_builtin_vectorization_cost -#undef TARGET_VECTORIZE_VEC_PERM_CONST_OK -#define TARGET_VECTORIZE_VEC_PERM_CONST_OK \ - ix86_vectorize_vec_perm_const_ok +#undef TARGET_VECTORIZE_VEC_PERM_CONST +#define TARGET_VECTORIZE_VEC_PERM_CONST ix86_vectorize_vec_perm_const #undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE #define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \ ix86_preferred_simd_mode +#undef TARGET_VECTORIZE_SPLIT_REDUCTION +#define TARGET_VECTORIZE_SPLIT_REDUCTION \ + ix86_split_reduction #undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES #define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \ ix86_autovectorize_vector_sizes @@ -50239,33 +51048,6 @@ #undef TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS #define TARGET_CALL_FUSAGE_CONTAINS_NON_CALLEE_CLOBBERS true -#undef TARGET_LOAD_BOUNDS_FOR_ARG -#define TARGET_LOAD_BOUNDS_FOR_ARG ix86_load_bounds - -#undef TARGET_STORE_BOUNDS_FOR_ARG -#define TARGET_STORE_BOUNDS_FOR_ARG ix86_store_bounds - -#undef TARGET_LOAD_RETURNED_BOUNDS -#define TARGET_LOAD_RETURNED_BOUNDS ix86_load_returned_bounds - -#undef TARGET_STORE_RETURNED_BOUNDS -#define TARGET_STORE_RETURNED_BOUNDS ix86_store_returned_bounds - -#undef TARGET_CHKP_BOUND_MODE -#define TARGET_CHKP_BOUND_MODE ix86_mpx_bound_mode - -#undef TARGET_BUILTIN_CHKP_FUNCTION -#define TARGET_BUILTIN_CHKP_FUNCTION ix86_builtin_mpx_function - -#undef TARGET_CHKP_FUNCTION_VALUE_BOUNDS -#define TARGET_CHKP_FUNCTION_VALUE_BOUNDS ix86_function_value_bounds - -#undef TARGET_CHKP_MAKE_BOUNDS_CONSTANT -#define TARGET_CHKP_MAKE_BOUNDS_CONSTANT ix86_make_bounds_constant - -#undef TARGET_CHKP_INITIALIZE_BOUNDS -#define TARGET_CHKP_INITIALIZE_BOUNDS ix86_initialize_bounds - #undef TARGET_SETUP_INCOMING_VARARG_BOUNDS #define TARGET_SETUP_INCOMING_VARARG_BOUNDS ix86_setup_incoming_vararg_bounds @@ -50285,9 +51067,6 @@ #undef TARGET_CUSTOM_FUNCTION_DESCRIPTORS #define TARGET_CUSTOM_FUNCTION_DESCRIPTORS 1 -#undef TARGET_ADDITIONAL_ALLOCNO_CLASS_P -#define TARGET_ADDITIONAL_ALLOCNO_CLASS_P ix86_additional_allocno_class_p - #undef TARGET_ADDR_SPACE_ZERO_ADDRESS_VALID #define TARGET_ADDR_SPACE_ZERO_ADDRESS_VALID ix86_addr_space_zero_address_valid @@ -50323,6 +51102,12 @@ #undef TARGET_CONSTANT_ALIGNMENT #define TARGET_CONSTANT_ALIGNMENT ix86_constant_alignment +#undef TARGET_EMPTY_RECORD_P +#define TARGET_EMPTY_RECORD_P ix86_is_empty_record + +#undef TARGET_WARN_PARAMETER_PASSING_ABI +#define TARGET_WARN_PARAMETER_PASSING_ABI ix86_warn_parameter_passing_abi + #if CHECKING_P #undef TARGET_RUN_TARGET_SELFTESTS #define TARGET_RUN_TARGET_SELFTESTS selftest::ix86_run_selftests