Mercurial > hg > CbC > CbC_gcc
diff gcc/config/i386/i386.c @ 69:1b10fe6932e1
merge 69
| author | Nobuyasu Oshiro <dimolto@cr.ie.u-ryukyu.ac.jp> |
|---|---|
| date | Sun, 21 Aug 2011 07:53:12 +0900 |
| parents | 326d9e06c2e3 f6334be47118 |
| children | ab0bcb71f44d |
line wrap: on
line diff
--- a/gcc/config/i386/i386.c Tue Dec 14 03:58:33 2010 +0900 +++ b/gcc/config/i386/i386.c Sun Aug 21 07:53:12 2011 +0900 @@ -1,6 +1,6 @@ /* Subroutines used for code generation on IA-32. - Copyright (C) 1988, 1992, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001, - 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 + Copyright (C) 1988, 1992, 1994, 1995, 1996, 1997, 1998, 1999, 2000, + 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011 Free Software Foundation, Inc. This file is part of GCC. @@ -28,7 +28,6 @@ #include "tm_p.h" #include "regs.h" #include "hard-reg-set.h" -#include "real.h" #include "insn-config.h" #include "conditions.h" #include "output.h" @@ -40,6 +39,7 @@ #include "recog.h" #include "expr.h" #include "optabs.h" +#include "diagnostic-core.h" #include "toplev.h" #include "basic-block.h" #include "ggc.h" @@ -53,6 +53,438 @@ #include "tm-constrs.h" #include "params.h" #include "cselib.h" +#include "debug.h" +#include "dwarf2out.h" +#include "sched-int.h" +#include "sbitmap.h" +#include "fibheap.h" + +enum upper_128bits_state +{ + unknown = 0, + unused, + used +}; + +typedef struct block_info_def +{ + /* State of the upper 128bits of AVX registers at exit. */ + enum upper_128bits_state state; + /* TRUE if state of the upper 128bits of AVX registers is unchanged + in this block. */ + bool unchanged; + /* TRUE if block has been processed. */ + bool processed; + /* TRUE if block has been scanned. */ + bool scanned; + /* Previous state of the upper 128bits of AVX registers at entry. */ + enum upper_128bits_state prev; +} *block_info; + +#define BLOCK_INFO(B) ((block_info) (B)->aux) + +enum call_avx256_state +{ + /* Callee returns 256bit AVX register. */ + callee_return_avx256 = -1, + /* Callee returns and passes 256bit AVX register. */ + callee_return_pass_avx256, + /* Callee passes 256bit AVX register. */ + callee_pass_avx256, + /* Callee doesn't return nor passe 256bit AVX register, or no + 256bit AVX register in function return. */ + call_no_avx256, + /* vzeroupper intrinsic. */ + vzeroupper_intrinsic +}; + +/* Check if a 256bit AVX register is referenced in stores. */ + +static void +check_avx256_stores (rtx dest, const_rtx set, void *data) +{ + if ((REG_P (dest) + && VALID_AVX256_REG_MODE (GET_MODE (dest))) + || (GET_CODE (set) == SET + && REG_P (SET_SRC (set)) + && VALID_AVX256_REG_MODE (GET_MODE (SET_SRC (set))))) + { + enum upper_128bits_state *state + = (enum upper_128bits_state *) data; + *state = used; + } +} + +/* Helper function for move_or_delete_vzeroupper_1. Look for vzeroupper + in basic block BB. Delete it if upper 128bit AVX registers are + unused. If it isn't deleted, move it to just before a jump insn. + + STATE is state of the upper 128bits of AVX registers at entry. */ + +static void +move_or_delete_vzeroupper_2 (basic_block bb, + enum upper_128bits_state state) +{ + rtx insn, bb_end; + rtx vzeroupper_insn = NULL_RTX; + rtx pat; + int avx256; + bool unchanged; + + if (BLOCK_INFO (bb)->unchanged) + { + if (dump_file) + fprintf (dump_file, " [bb %i] unchanged: upper 128bits: %d\n", + bb->index, state); + + BLOCK_INFO (bb)->state = state; + return; + } + + if (BLOCK_INFO (bb)->scanned && BLOCK_INFO (bb)->prev == state) + { + if (dump_file) + fprintf (dump_file, " [bb %i] scanned: upper 128bits: %d\n", + bb->index, BLOCK_INFO (bb)->state); + return; + } + + BLOCK_INFO (bb)->prev = state; + + if (dump_file) + fprintf (dump_file, " [bb %i] entry: upper 128bits: %d\n", + bb->index, state); + + unchanged = true; + + /* BB_END changes when it is deleted. */ + bb_end = BB_END (bb); + insn = BB_HEAD (bb); + while (insn != bb_end) + { + insn = NEXT_INSN (insn); + + if (!NONDEBUG_INSN_P (insn)) + continue; + + /* Move vzeroupper before jump/call. */ + if (JUMP_P (insn) || CALL_P (insn)) + { + if (!vzeroupper_insn) + continue; + + if (PREV_INSN (insn) != vzeroupper_insn) + { + if (dump_file) + { + fprintf (dump_file, "Move vzeroupper after:\n"); + print_rtl_single (dump_file, PREV_INSN (insn)); + fprintf (dump_file, "before:\n"); + print_rtl_single (dump_file, insn); + } + reorder_insns_nobb (vzeroupper_insn, vzeroupper_insn, + PREV_INSN (insn)); + } + vzeroupper_insn = NULL_RTX; + continue; + } + + pat = PATTERN (insn); + + /* Check insn for vzeroupper intrinsic. */ + if (GET_CODE (pat) == UNSPEC_VOLATILE + && XINT (pat, 1) == UNSPECV_VZEROUPPER) + { + if (dump_file) + { + /* Found vzeroupper intrinsic. */ + fprintf (dump_file, "Found vzeroupper:\n"); + print_rtl_single (dump_file, insn); + } + } + else + { + /* Check insn for vzeroall intrinsic. */ + if (GET_CODE (pat) == PARALLEL + && GET_CODE (XVECEXP (pat, 0, 0)) == UNSPEC_VOLATILE + && XINT (XVECEXP (pat, 0, 0), 1) == UNSPECV_VZEROALL) + { + state = unused; + unchanged = false; + + /* Delete pending vzeroupper insertion. */ + if (vzeroupper_insn) + { + delete_insn (vzeroupper_insn); + vzeroupper_insn = NULL_RTX; + } + } + else if (state != used) + { + note_stores (pat, check_avx256_stores, &state); + if (state == used) + unchanged = false; + } + continue; + } + + /* Process vzeroupper intrinsic. */ + avx256 = INTVAL (XVECEXP (pat, 0, 0)); + + if (state == unused) + { + /* Since the upper 128bits are cleared, callee must not pass + 256bit AVX register. We only need to check if callee + returns 256bit AVX register. */ + if (avx256 == callee_return_avx256) + { + state = used; + unchanged = false; + } + + /* Remove unnecessary vzeroupper since upper 128bits are + cleared. */ + if (dump_file) + { + fprintf (dump_file, "Delete redundant vzeroupper:\n"); + print_rtl_single (dump_file, insn); + } + delete_insn (insn); + } + else + { + /* Set state to UNUSED if callee doesn't return 256bit AVX + register. */ + if (avx256 != callee_return_pass_avx256) + state = unused; + + if (avx256 == callee_return_pass_avx256 + || avx256 == callee_pass_avx256) + { + /* Must remove vzeroupper since callee passes in 256bit + AVX register. */ + if (dump_file) + { + fprintf (dump_file, "Delete callee pass vzeroupper:\n"); + print_rtl_single (dump_file, insn); + } + delete_insn (insn); + } + else + { + vzeroupper_insn = insn; + unchanged = false; + } + } + } + + BLOCK_INFO (bb)->state = state; + BLOCK_INFO (bb)->unchanged = unchanged; + BLOCK_INFO (bb)->scanned = true; + + if (dump_file) + fprintf (dump_file, " [bb %i] exit: %s: upper 128bits: %d\n", + bb->index, unchanged ? "unchanged" : "changed", + state); +} + +/* Helper function for move_or_delete_vzeroupper. Process vzeroupper + in BLOCK and check its predecessor blocks. Treat UNKNOWN state + as USED if UNKNOWN_IS_UNUSED is true. Return TRUE if the exit + state is changed. */ + +static bool +move_or_delete_vzeroupper_1 (basic_block block, bool unknown_is_unused) +{ + edge e; + edge_iterator ei; + enum upper_128bits_state state, old_state, new_state; + bool seen_unknown; + + if (dump_file) + fprintf (dump_file, " Process [bb %i]: status: %d\n", + block->index, BLOCK_INFO (block)->processed); + + if (BLOCK_INFO (block)->processed) + return false; + + state = unused; + + /* Check all predecessor edges of this block. */ + seen_unknown = false; + FOR_EACH_EDGE (e, ei, block->preds) + { + if (e->src == block) + continue; + switch (BLOCK_INFO (e->src)->state) + { + case unknown: + if (!unknown_is_unused) + seen_unknown = true; + case unused: + break; + case used: + state = used; + goto done; + } + } + + if (seen_unknown) + state = unknown; + +done: + old_state = BLOCK_INFO (block)->state; + move_or_delete_vzeroupper_2 (block, state); + new_state = BLOCK_INFO (block)->state; + + if (state != unknown || new_state == used) + BLOCK_INFO (block)->processed = true; + + /* Need to rescan if the upper 128bits of AVX registers are changed + to USED at exit. */ + if (new_state != old_state) + { + if (new_state == used) + cfun->machine->rescan_vzeroupper_p = 1; + return true; + } + else + return false; +} + +/* Go through the instruction stream looking for vzeroupper. Delete + it if upper 128bit AVX registers are unused. If it isn't deleted, + move it to just before a jump insn. */ + +static void +move_or_delete_vzeroupper (void) +{ + edge e; + edge_iterator ei; + basic_block bb; + fibheap_t worklist, pending, fibheap_swap; + sbitmap visited, in_worklist, in_pending, sbitmap_swap; + int *bb_order; + int *rc_order; + int i; + + /* Set up block info for each basic block. */ + alloc_aux_for_blocks (sizeof (struct block_info_def)); + + /* Process outgoing edges of entry point. */ + if (dump_file) + fprintf (dump_file, "Process outgoing edges of entry point\n"); + + FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR->succs) + { + move_or_delete_vzeroupper_2 (e->dest, + cfun->machine->caller_pass_avx256_p + ? used : unused); + BLOCK_INFO (e->dest)->processed = true; + } + + /* Compute reverse completion order of depth first search of the CFG + so that the data-flow runs faster. */ + rc_order = XNEWVEC (int, n_basic_blocks - NUM_FIXED_BLOCKS); + bb_order = XNEWVEC (int, last_basic_block); + pre_and_rev_post_order_compute (NULL, rc_order, false); + for (i = 0; i < n_basic_blocks - NUM_FIXED_BLOCKS; i++) + bb_order[rc_order[i]] = i; + free (rc_order); + + worklist = fibheap_new (); + pending = fibheap_new (); + visited = sbitmap_alloc (last_basic_block); + in_worklist = sbitmap_alloc (last_basic_block); + in_pending = sbitmap_alloc (last_basic_block); + sbitmap_zero (in_worklist); + + /* Don't check outgoing edges of entry point. */ + sbitmap_ones (in_pending); + FOR_EACH_BB (bb) + if (BLOCK_INFO (bb)->processed) + RESET_BIT (in_pending, bb->index); + else + { + move_or_delete_vzeroupper_1 (bb, false); + fibheap_insert (pending, bb_order[bb->index], bb); + } + + if (dump_file) + fprintf (dump_file, "Check remaining basic blocks\n"); + + while (!fibheap_empty (pending)) + { + fibheap_swap = pending; + pending = worklist; + worklist = fibheap_swap; + sbitmap_swap = in_pending; + in_pending = in_worklist; + in_worklist = sbitmap_swap; + + sbitmap_zero (visited); + + cfun->machine->rescan_vzeroupper_p = 0; + + while (!fibheap_empty (worklist)) + { + bb = (basic_block) fibheap_extract_min (worklist); + RESET_BIT (in_worklist, bb->index); + gcc_assert (!TEST_BIT (visited, bb->index)); + if (!TEST_BIT (visited, bb->index)) + { + edge_iterator ei; + + SET_BIT (visited, bb->index); + + if (move_or_delete_vzeroupper_1 (bb, false)) + FOR_EACH_EDGE (e, ei, bb->succs) + { + if (e->dest == EXIT_BLOCK_PTR + || BLOCK_INFO (e->dest)->processed) + continue; + + if (TEST_BIT (visited, e->dest->index)) + { + if (!TEST_BIT (in_pending, e->dest->index)) + { + /* Send E->DEST to next round. */ + SET_BIT (in_pending, e->dest->index); + fibheap_insert (pending, + bb_order[e->dest->index], + e->dest); + } + } + else if (!TEST_BIT (in_worklist, e->dest->index)) + { + /* Add E->DEST to current round. */ + SET_BIT (in_worklist, e->dest->index); + fibheap_insert (worklist, bb_order[e->dest->index], + e->dest); + } + } + } + } + + if (!cfun->machine->rescan_vzeroupper_p) + break; + } + + free (bb_order); + fibheap_delete (worklist); + fibheap_delete (pending); + sbitmap_free (visited); + sbitmap_free (in_worklist); + sbitmap_free (in_pending); + + if (dump_file) + fprintf (dump_file, "Process remaining basic blocks\n"); + + FOR_EACH_BB (bb) + move_or_delete_vzeroupper_1 (bb, true); + + free_aux_for_blocks (); +} static rtx legitimize_dllimport_symbol (rtx, bool); @@ -81,21 +513,21 @@ COSTS_N_BYTES (2), /* variable shift costs */ COSTS_N_BYTES (3), /* constant shift costs */ {COSTS_N_BYTES (3), /* cost of starting multiply for QI */ - COSTS_N_BYTES (3), /* HI */ - COSTS_N_BYTES (3), /* SI */ - COSTS_N_BYTES (3), /* DI */ - COSTS_N_BYTES (5)}, /* other */ + COSTS_N_BYTES (3), /* HI */ + COSTS_N_BYTES (3), /* SI */ + COSTS_N_BYTES (3), /* DI */ + COSTS_N_BYTES (5)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_BYTES (3), /* cost of a divide/mod for QI */ - COSTS_N_BYTES (3), /* HI */ - COSTS_N_BYTES (3), /* SI */ - COSTS_N_BYTES (3), /* DI */ - COSTS_N_BYTES (5)}, /* other */ + COSTS_N_BYTES (3), /* HI */ + COSTS_N_BYTES (3), /* SI */ + COSTS_N_BYTES (3), /* DI */ + COSTS_N_BYTES (5)}, /* other */ COSTS_N_BYTES (3), /* cost of movsx */ COSTS_N_BYTES (3), /* cost of movzx */ 0, /* "large" insn */ 2, /* MOVE_RATIO */ - 2, /* cost for loading QImode using movzbl */ + 2, /* cost for loading QImode using movzbl */ {2, 2, 2}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -131,17 +563,17 @@ {rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}}, {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}, {rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 1, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 1, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 1, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 1, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; /* Processor costs (relative to an add) */ @@ -152,21 +584,21 @@ COSTS_N_INSNS (3), /* variable shift costs */ COSTS_N_INSNS (2), /* constant shift costs */ {COSTS_N_INSNS (6), /* cost of starting multiply for QI */ - COSTS_N_INSNS (6), /* HI */ - COSTS_N_INSNS (6), /* SI */ - COSTS_N_INSNS (6), /* DI */ - COSTS_N_INSNS (6)}, /* other */ + COSTS_N_INSNS (6), /* HI */ + COSTS_N_INSNS (6), /* SI */ + COSTS_N_INSNS (6), /* DI */ + COSTS_N_INSNS (6)}, /* other */ COSTS_N_INSNS (1), /* cost of multiply per each bit set */ {COSTS_N_INSNS (23), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (23), /* HI */ - COSTS_N_INSNS (23), /* SI */ - COSTS_N_INSNS (23), /* DI */ - COSTS_N_INSNS (23)}, /* other */ + COSTS_N_INSNS (23), /* HI */ + COSTS_N_INSNS (23), /* SI */ + COSTS_N_INSNS (23), /* DI */ + COSTS_N_INSNS (23)}, /* other */ COSTS_N_INSNS (3), /* cost of movsx */ COSTS_N_INSNS (2), /* cost of movzx */ 15, /* "large" insn */ 3, /* MOVE_RATIO */ - 4, /* cost for loading QImode using movzbl */ + 4, /* cost for loading QImode using movzbl */ {2, 4, 2}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -202,17 +634,17 @@ DUMMY_STRINGOP_ALGS}, {{rep_prefix_1_byte, {{-1, rep_prefix_1_byte}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -222,21 +654,21 @@ COSTS_N_INSNS (3), /* variable shift costs */ COSTS_N_INSNS (2), /* constant shift costs */ {COSTS_N_INSNS (12), /* cost of starting multiply for QI */ - COSTS_N_INSNS (12), /* HI */ - COSTS_N_INSNS (12), /* SI */ - COSTS_N_INSNS (12), /* DI */ - COSTS_N_INSNS (12)}, /* other */ + COSTS_N_INSNS (12), /* HI */ + COSTS_N_INSNS (12), /* SI */ + COSTS_N_INSNS (12), /* DI */ + COSTS_N_INSNS (12)}, /* other */ 1, /* cost of multiply per each bit set */ {COSTS_N_INSNS (40), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (40), /* HI */ - COSTS_N_INSNS (40), /* SI */ - COSTS_N_INSNS (40), /* DI */ - COSTS_N_INSNS (40)}, /* other */ + COSTS_N_INSNS (40), /* HI */ + COSTS_N_INSNS (40), /* SI */ + COSTS_N_INSNS (40), /* DI */ + COSTS_N_INSNS (40)}, /* other */ COSTS_N_INSNS (3), /* cost of movsx */ COSTS_N_INSNS (2), /* cost of movzx */ 15, /* "large" insn */ 3, /* MOVE_RATIO */ - 4, /* cost for loading QImode using movzbl */ + 4, /* cost for loading QImode using movzbl */ {2, 4, 2}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -274,17 +706,17 @@ DUMMY_STRINGOP_ALGS}, {{rep_prefix_4_byte, {{-1, rep_prefix_4_byte}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -294,21 +726,21 @@ COSTS_N_INSNS (4), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (11), /* cost of starting multiply for QI */ - COSTS_N_INSNS (11), /* HI */ - COSTS_N_INSNS (11), /* SI */ - COSTS_N_INSNS (11), /* DI */ - COSTS_N_INSNS (11)}, /* other */ + COSTS_N_INSNS (11), /* HI */ + COSTS_N_INSNS (11), /* SI */ + COSTS_N_INSNS (11), /* DI */ + COSTS_N_INSNS (11)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (25), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (25), /* HI */ - COSTS_N_INSNS (25), /* SI */ - COSTS_N_INSNS (25), /* DI */ - COSTS_N_INSNS (25)}, /* other */ + COSTS_N_INSNS (25), /* HI */ + COSTS_N_INSNS (25), /* SI */ + COSTS_N_INSNS (25), /* DI */ + COSTS_N_INSNS (25)}, /* other */ COSTS_N_INSNS (3), /* cost of movsx */ COSTS_N_INSNS (2), /* cost of movzx */ 8, /* "large" insn */ 6, /* MOVE_RATIO */ - 6, /* cost for loading QImode using movzbl */ + 6, /* cost for loading QImode using movzbl */ {2, 4, 2}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -344,17 +776,17 @@ DUMMY_STRINGOP_ALGS}, {{libcall, {{-1, rep_prefix_4_byte}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -364,21 +796,21 @@ COSTS_N_INSNS (1), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (4), /* cost of starting multiply for QI */ - COSTS_N_INSNS (4), /* HI */ - COSTS_N_INSNS (4), /* SI */ - COSTS_N_INSNS (4), /* DI */ - COSTS_N_INSNS (4)}, /* other */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (4), /* SI */ + COSTS_N_INSNS (4), /* DI */ + COSTS_N_INSNS (4)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (17), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (17), /* HI */ - COSTS_N_INSNS (17), /* SI */ - COSTS_N_INSNS (17), /* DI */ - COSTS_N_INSNS (17)}, /* other */ + COSTS_N_INSNS (17), /* HI */ + COSTS_N_INSNS (17), /* SI */ + COSTS_N_INSNS (17), /* DI */ + COSTS_N_INSNS (17)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 8, /* "large" insn */ 6, /* MOVE_RATIO */ - 2, /* cost for loading QImode using movzbl */ + 2, /* cost for loading QImode using movzbl */ {4, 4, 4}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -410,28 +842,28 @@ COSTS_N_INSNS (2), /* cost of FABS instruction. */ COSTS_N_INSNS (2), /* cost of FCHS instruction. */ COSTS_N_INSNS (56), /* cost of FSQRT instruction. */ - /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes (we ensure - the alignment). For small blocks inline loop is still a noticeable win, for bigger - blocks either rep movsl or rep movsb is way to go. Rep movsb has apparently - more expensive startup time in CPU, but after 4K the difference is down in the noise. - */ + /* PentiumPro has optimized rep instructions for blocks aligned by 8 bytes + (we ensure the alignment). For small blocks inline loop is still a + noticeable win, for bigger blocks either rep movsl or rep movsb is + way to go. Rep movsb has apparently more expensive startup time in CPU, + but after 4K the difference is down in the noise. */ {{rep_prefix_4_byte, {{128, loop}, {1024, unrolled_loop}, {8192, rep_prefix_4_byte}, {-1, rep_prefix_1_byte}}}, DUMMY_STRINGOP_ALGS}, {{rep_prefix_4_byte, {{1024, unrolled_loop}, - {8192, rep_prefix_4_byte}, {-1, libcall}}}, + {8192, rep_prefix_4_byte}, {-1, libcall}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -441,21 +873,21 @@ COSTS_N_INSNS (2), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ - COSTS_N_INSNS (4), /* HI */ - COSTS_N_INSNS (7), /* SI */ - COSTS_N_INSNS (7), /* DI */ - COSTS_N_INSNS (7)}, /* other */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (7), /* SI */ + COSTS_N_INSNS (7), /* DI */ + COSTS_N_INSNS (7)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (15), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (23), /* HI */ - COSTS_N_INSNS (39), /* SI */ - COSTS_N_INSNS (39), /* DI */ - COSTS_N_INSNS (39)}, /* other */ + COSTS_N_INSNS (23), /* HI */ + COSTS_N_INSNS (39), /* SI */ + COSTS_N_INSNS (39), /* DI */ + COSTS_N_INSNS (39)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 8, /* "large" insn */ 4, /* MOVE_RATIO */ - 1, /* cost for loading QImode using movzbl */ + 1, /* cost for loading QImode using movzbl */ {1, 1, 1}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -492,17 +924,17 @@ DUMMY_STRINGOP_ALGS}, {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -512,21 +944,21 @@ COSTS_N_INSNS (1), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ - COSTS_N_INSNS (3), /* HI */ - COSTS_N_INSNS (3), /* SI */ - COSTS_N_INSNS (3), /* DI */ - COSTS_N_INSNS (3)}, /* other */ + COSTS_N_INSNS (3), /* HI */ + COSTS_N_INSNS (3), /* SI */ + COSTS_N_INSNS (3), /* DI */ + COSTS_N_INSNS (3)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (18), /* HI */ - COSTS_N_INSNS (18), /* SI */ - COSTS_N_INSNS (18), /* DI */ - COSTS_N_INSNS (18)}, /* other */ + COSTS_N_INSNS (18), /* HI */ + COSTS_N_INSNS (18), /* SI */ + COSTS_N_INSNS (18), /* DI */ + COSTS_N_INSNS (18)}, /* other */ COSTS_N_INSNS (2), /* cost of movsx */ COSTS_N_INSNS (2), /* cost of movzx */ 8, /* "large" insn */ 4, /* MOVE_RATIO */ - 3, /* cost for loading QImode using movzbl */ + 3, /* cost for loading QImode using movzbl */ {4, 5, 4}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -565,17 +997,17 @@ DUMMY_STRINGOP_ALGS}, {{libcall, {{256, rep_prefix_4_byte}, {-1, libcall}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -585,21 +1017,21 @@ COSTS_N_INSNS (1), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (5), /* cost of starting multiply for QI */ - COSTS_N_INSNS (5), /* HI */ - COSTS_N_INSNS (5), /* SI */ - COSTS_N_INSNS (5), /* DI */ - COSTS_N_INSNS (5)}, /* other */ + COSTS_N_INSNS (5), /* HI */ + COSTS_N_INSNS (5), /* SI */ + COSTS_N_INSNS (5), /* DI */ + COSTS_N_INSNS (5)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (26), /* HI */ - COSTS_N_INSNS (42), /* SI */ - COSTS_N_INSNS (74), /* DI */ - COSTS_N_INSNS (74)}, /* other */ + COSTS_N_INSNS (26), /* HI */ + COSTS_N_INSNS (42), /* SI */ + COSTS_N_INSNS (74), /* DI */ + COSTS_N_INSNS (74)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 8, /* "large" insn */ 9, /* MOVE_RATIO */ - 4, /* cost for loading QImode using movzbl */ + 4, /* cost for loading QImode using movzbl */ {3, 4, 3}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -638,17 +1070,17 @@ DUMMY_STRINGOP_ALGS}, {{libcall, {{2048, rep_prefix_4_byte}, {-1, libcall}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -658,21 +1090,21 @@ COSTS_N_INSNS (1), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ - COSTS_N_INSNS (4), /* HI */ - COSTS_N_INSNS (3), /* SI */ - COSTS_N_INSNS (4), /* DI */ - COSTS_N_INSNS (5)}, /* other */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (3), /* SI */ + COSTS_N_INSNS (4), /* DI */ + COSTS_N_INSNS (5)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (26), /* HI */ - COSTS_N_INSNS (42), /* SI */ - COSTS_N_INSNS (74), /* DI */ - COSTS_N_INSNS (74)}, /* other */ + COSTS_N_INSNS (26), /* HI */ + COSTS_N_INSNS (42), /* SI */ + COSTS_N_INSNS (74), /* DI */ + COSTS_N_INSNS (74)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 8, /* "large" insn */ 9, /* MOVE_RATIO */ - 4, /* cost for loading QImode using movzbl */ + 4, /* cost for loading QImode using movzbl */ {3, 4, 3}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -709,48 +1141,48 @@ COSTS_N_INSNS (2), /* cost of FABS instruction. */ COSTS_N_INSNS (2), /* cost of FCHS instruction. */ COSTS_N_INSNS (35), /* cost of FSQRT instruction. */ - /* K8 has optimized REP instruction for medium sized blocks, but for very small - blocks it is better to use loop. For large blocks, libcall can do - nontemporary accesses and beat inline considerably. */ + /* K8 has optimized REP instruction for medium sized blocks, but for very + small blocks it is better to use loop. For large blocks, libcall can + do nontemporary accesses and beat inline considerably. */ {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}}, {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, {{libcall, {{8, loop}, {24, unrolled_loop}, {2048, rep_prefix_4_byte}, {-1, libcall}}}, {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, - 4, /* scalar_stmt_cost. */ - 2, /* scalar load_cost. */ - 2, /* scalar_store_cost. */ - 5, /* vec_stmt_cost. */ - 0, /* vec_to_scalar_cost. */ - 2, /* scalar_to_vec_cost. */ - 2, /* vec_align_load_cost. */ - 3, /* vec_unalign_load_cost. */ - 3, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 2, /* cond_not_taken_branch_cost. */ + 4, /* scalar_stmt_cost. */ + 2, /* scalar load_cost. */ + 2, /* scalar_store_cost. */ + 5, /* vec_stmt_cost. */ + 0, /* vec_to_scalar_cost. */ + 2, /* scalar_to_vec_cost. */ + 2, /* vec_align_load_cost. */ + 3, /* vec_unalign_load_cost. */ + 3, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 2, /* cond_not_taken_branch_cost. */ }; struct processor_costs amdfam10_cost = { - COSTS_N_INSNS (1), /* cost of an add instruction */ - COSTS_N_INSNS (2), /* cost of a lea instruction */ - COSTS_N_INSNS (1), /* variable shift costs */ - COSTS_N_INSNS (1), /* constant shift costs */ - {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ - COSTS_N_INSNS (4), /* HI */ - COSTS_N_INSNS (3), /* SI */ - COSTS_N_INSNS (4), /* DI */ - COSTS_N_INSNS (5)}, /* other */ - 0, /* cost of multiply per each bit set */ - {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (35), /* HI */ - COSTS_N_INSNS (51), /* SI */ - COSTS_N_INSNS (83), /* DI */ - COSTS_N_INSNS (83)}, /* other */ + COSTS_N_INSNS (1), /* cost of an add instruction */ + COSTS_N_INSNS (2), /* cost of a lea instruction */ + COSTS_N_INSNS (1), /* variable shift costs */ + COSTS_N_INSNS (1), /* constant shift costs */ + {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (3), /* SI */ + COSTS_N_INSNS (4), /* DI */ + COSTS_N_INSNS (5)}, /* other */ + 0, /* cost of multiply per each bit set */ + {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */ + COSTS_N_INSNS (35), /* HI */ + COSTS_N_INSNS (51), /* SI */ + COSTS_N_INSNS (83), /* DI */ + COSTS_N_INSNS (83)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 8, /* "large" insn */ 9, /* MOVE_RATIO */ - 4, /* cost for loading QImode using movzbl */ + 4, /* cost for loading QImode using movzbl */ {3, 4, 3}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -771,14 +1203,14 @@ {4, 4, 5}, /* cost of storing SSE registers in SImode, DImode and TImode */ 3, /* MMX or SSE register to integer */ - /* On K8 - MOVD reg64, xmmreg Double FSTORE 4 - MOVD reg32, xmmreg Double FSTORE 4 - On AMDFAM10 - MOVD reg64, xmmreg Double FADD 3 - 1/1 1/1 - MOVD reg32, xmmreg Double FADD 3 - 1/1 1/1 */ + /* On K8: + MOVD reg64, xmmreg Double FSTORE 4 + MOVD reg32, xmmreg Double FSTORE 4 + On AMDFAM10: + MOVD reg64, xmmreg Double FADD 3 + 1/1 1/1 + MOVD reg32, xmmreg Double FADD 3 + 1/1 1/1 */ 64, /* size of l1 cache. */ 512, /* size of l2 cache. */ 64, /* size of prefetch block */ @@ -804,17 +1236,186 @@ {{libcall, {{8, loop}, {24, unrolled_loop}, {2048, rep_prefix_4_byte}, {-1, libcall}}}, {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, - 4, /* scalar_stmt_cost. */ - 2, /* scalar load_cost. */ - 2, /* scalar_store_cost. */ - 6, /* vec_stmt_cost. */ - 0, /* vec_to_scalar_cost. */ - 2, /* scalar_to_vec_cost. */ - 2, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 2, /* vec_store_cost. */ - 2, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 4, /* scalar_stmt_cost. */ + 2, /* scalar load_cost. */ + 2, /* scalar_store_cost. */ + 6, /* vec_stmt_cost. */ + 0, /* vec_to_scalar_cost. */ + 2, /* scalar_to_vec_cost. */ + 2, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 2, /* vec_store_cost. */ + 2, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ +}; + +struct processor_costs bdver1_cost = { + COSTS_N_INSNS (1), /* cost of an add instruction */ + COSTS_N_INSNS (1), /* cost of a lea instruction */ + COSTS_N_INSNS (1), /* variable shift costs */ + COSTS_N_INSNS (1), /* constant shift costs */ + {COSTS_N_INSNS (4), /* cost of starting multiply for QI */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (4), /* SI */ + COSTS_N_INSNS (6), /* DI */ + COSTS_N_INSNS (6)}, /* other */ + 0, /* cost of multiply per each bit set */ + {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */ + COSTS_N_INSNS (35), /* HI */ + COSTS_N_INSNS (51), /* SI */ + COSTS_N_INSNS (83), /* DI */ + COSTS_N_INSNS (83)}, /* other */ + COSTS_N_INSNS (1), /* cost of movsx */ + COSTS_N_INSNS (1), /* cost of movzx */ + 8, /* "large" insn */ + 9, /* MOVE_RATIO */ + 4, /* cost for loading QImode using movzbl */ + {5, 5, 4}, /* cost of loading integer registers + in QImode, HImode and SImode. + Relative to reg-reg move (2). */ + {4, 4, 4}, /* cost of storing integer registers */ + 2, /* cost of reg,reg fld/fst */ + {5, 5, 12}, /* cost of loading fp registers + in SFmode, DFmode and XFmode */ + {4, 4, 8}, /* cost of storing fp registers + in SFmode, DFmode and XFmode */ + 2, /* cost of moving MMX register */ + {4, 4}, /* cost of loading MMX registers + in SImode and DImode */ + {4, 4}, /* cost of storing MMX registers + in SImode and DImode */ + 2, /* cost of moving SSE register */ + {4, 4, 4}, /* cost of loading SSE registers + in SImode, DImode and TImode */ + {4, 4, 4}, /* cost of storing SSE registers + in SImode, DImode and TImode */ + 2, /* MMX or SSE register to integer */ + /* On K8: + MOVD reg64, xmmreg Double FSTORE 4 + MOVD reg32, xmmreg Double FSTORE 4 + On AMDFAM10: + MOVD reg64, xmmreg Double FADD 3 + 1/1 1/1 + MOVD reg32, xmmreg Double FADD 3 + 1/1 1/1 */ + 16, /* size of l1 cache. */ + 2048, /* size of l2 cache. */ + 64, /* size of prefetch block */ + /* New AMD processors never drop prefetches; if they cannot be performed + immediately, they are queued. We set number of simultaneous prefetches + to a large constant to reflect this (it probably is not a good idea not + to limit number of prefetches at all, as their execution also takes some + time). */ + 100, /* number of parallel prefetches */ + 2, /* Branch cost */ + COSTS_N_INSNS (6), /* cost of FADD and FSUB insns. */ + COSTS_N_INSNS (6), /* cost of FMUL instruction. */ + COSTS_N_INSNS (42), /* cost of FDIV instruction. */ + COSTS_N_INSNS (2), /* cost of FABS instruction. */ + COSTS_N_INSNS (2), /* cost of FCHS instruction. */ + COSTS_N_INSNS (52), /* cost of FSQRT instruction. */ + + /* BDVER1 has optimized REP instruction for medium sized blocks, but for + very small blocks it is better to use loop. For large blocks, libcall + can do nontemporary accesses and beat inline considerably. */ + {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}}, + {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, + {{libcall, {{8, loop}, {24, unrolled_loop}, + {2048, rep_prefix_4_byte}, {-1, libcall}}}, + {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, + 6, /* scalar_stmt_cost. */ + 4, /* scalar load_cost. */ + 4, /* scalar_store_cost. */ + 6, /* vec_stmt_cost. */ + 0, /* vec_to_scalar_cost. */ + 2, /* scalar_to_vec_cost. */ + 4, /* vec_align_load_cost. */ + 4, /* vec_unalign_load_cost. */ + 4, /* vec_store_cost. */ + 2, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ +}; + +struct processor_costs btver1_cost = { + COSTS_N_INSNS (1), /* cost of an add instruction */ + COSTS_N_INSNS (2), /* cost of a lea instruction */ + COSTS_N_INSNS (1), /* variable shift costs */ + COSTS_N_INSNS (1), /* constant shift costs */ + {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (3), /* SI */ + COSTS_N_INSNS (4), /* DI */ + COSTS_N_INSNS (5)}, /* other */ + 0, /* cost of multiply per each bit set */ + {COSTS_N_INSNS (19), /* cost of a divide/mod for QI */ + COSTS_N_INSNS (35), /* HI */ + COSTS_N_INSNS (51), /* SI */ + COSTS_N_INSNS (83), /* DI */ + COSTS_N_INSNS (83)}, /* other */ + COSTS_N_INSNS (1), /* cost of movsx */ + COSTS_N_INSNS (1), /* cost of movzx */ + 8, /* "large" insn */ + 9, /* MOVE_RATIO */ + 4, /* cost for loading QImode using movzbl */ + {3, 4, 3}, /* cost of loading integer registers + in QImode, HImode and SImode. + Relative to reg-reg move (2). */ + {3, 4, 3}, /* cost of storing integer registers */ + 4, /* cost of reg,reg fld/fst */ + {4, 4, 12}, /* cost of loading fp registers + in SFmode, DFmode and XFmode */ + {6, 6, 8}, /* cost of storing fp registers + in SFmode, DFmode and XFmode */ + 2, /* cost of moving MMX register */ + {3, 3}, /* cost of loading MMX registers + in SImode and DImode */ + {4, 4}, /* cost of storing MMX registers + in SImode and DImode */ + 2, /* cost of moving SSE register */ + {4, 4, 3}, /* cost of loading SSE registers + in SImode, DImode and TImode */ + {4, 4, 5}, /* cost of storing SSE registers + in SImode, DImode and TImode */ + 3, /* MMX or SSE register to integer */ + /* On K8: + MOVD reg64, xmmreg Double FSTORE 4 + MOVD reg32, xmmreg Double FSTORE 4 + On AMDFAM10: + MOVD reg64, xmmreg Double FADD 3 + 1/1 1/1 + MOVD reg32, xmmreg Double FADD 3 + 1/1 1/1 */ + 32, /* size of l1 cache. */ + 512, /* size of l2 cache. */ + 64, /* size of prefetch block */ + 100, /* number of parallel prefetches */ + 2, /* Branch cost */ + COSTS_N_INSNS (4), /* cost of FADD and FSUB insns. */ + COSTS_N_INSNS (4), /* cost of FMUL instruction. */ + COSTS_N_INSNS (19), /* cost of FDIV instruction. */ + COSTS_N_INSNS (2), /* cost of FABS instruction. */ + COSTS_N_INSNS (2), /* cost of FCHS instruction. */ + COSTS_N_INSNS (35), /* cost of FSQRT instruction. */ + + /* BTVER1 has optimized REP instruction for medium sized blocks, but for + very small blocks it is better to use loop. For large blocks, libcall can + do nontemporary accesses and beat inline considerably. */ + {{libcall, {{6, loop}, {14, unrolled_loop}, {-1, rep_prefix_4_byte}}}, + {libcall, {{16, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, + {{libcall, {{8, loop}, {24, unrolled_loop}, + {2048, rep_prefix_4_byte}, {-1, libcall}}}, + {libcall, {{48, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, + 4, /* scalar_stmt_cost. */ + 2, /* scalar load_cost. */ + 2, /* scalar_store_cost. */ + 6, /* vec_stmt_cost. */ + 0, /* vec_to_scalar_cost. */ + 2, /* scalar_to_vec_cost. */ + 2, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 2, /* vec_store_cost. */ + 2, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -824,21 +1425,21 @@ COSTS_N_INSNS (4), /* variable shift costs */ COSTS_N_INSNS (4), /* constant shift costs */ {COSTS_N_INSNS (15), /* cost of starting multiply for QI */ - COSTS_N_INSNS (15), /* HI */ - COSTS_N_INSNS (15), /* SI */ - COSTS_N_INSNS (15), /* DI */ - COSTS_N_INSNS (15)}, /* other */ + COSTS_N_INSNS (15), /* HI */ + COSTS_N_INSNS (15), /* SI */ + COSTS_N_INSNS (15), /* DI */ + COSTS_N_INSNS (15)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (56), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (56), /* HI */ - COSTS_N_INSNS (56), /* SI */ - COSTS_N_INSNS (56), /* DI */ - COSTS_N_INSNS (56)}, /* other */ + COSTS_N_INSNS (56), /* HI */ + COSTS_N_INSNS (56), /* SI */ + COSTS_N_INSNS (56), /* DI */ + COSTS_N_INSNS (56)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 16, /* "large" insn */ 6, /* MOVE_RATIO */ - 2, /* cost for loading QImode using movzbl */ + 2, /* cost for loading QImode using movzbl */ {4, 5, 4}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -875,17 +1476,17 @@ {{libcall, {{6, loop_1_byte}, {48, loop}, {20480, rep_prefix_4_byte}, {-1, libcall}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -895,21 +1496,21 @@ COSTS_N_INSNS (1), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (10), /* cost of starting multiply for QI */ - COSTS_N_INSNS (10), /* HI */ - COSTS_N_INSNS (10), /* SI */ - COSTS_N_INSNS (10), /* DI */ - COSTS_N_INSNS (10)}, /* other */ + COSTS_N_INSNS (10), /* HI */ + COSTS_N_INSNS (10), /* SI */ + COSTS_N_INSNS (10), /* DI */ + COSTS_N_INSNS (10)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (66), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (66), /* HI */ - COSTS_N_INSNS (66), /* SI */ - COSTS_N_INSNS (66), /* DI */ - COSTS_N_INSNS (66)}, /* other */ + COSTS_N_INSNS (66), /* HI */ + COSTS_N_INSNS (66), /* SI */ + COSTS_N_INSNS (66), /* DI */ + COSTS_N_INSNS (66)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 16, /* "large" insn */ 17, /* MOVE_RATIO */ - 4, /* cost for loading QImode using movzbl */ + 4, /* cost for loading QImode using movzbl */ {4, 4, 4}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -948,90 +1549,17 @@ {-1, libcall}}}, {libcall, {{24, loop}, {64, unrolled_loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ -}; - -static const -struct processor_costs core2_cost = { - COSTS_N_INSNS (1), /* cost of an add instruction */ - COSTS_N_INSNS (1) + 1, /* cost of a lea instruction */ - COSTS_N_INSNS (1), /* variable shift costs */ - COSTS_N_INSNS (1), /* constant shift costs */ - {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ - COSTS_N_INSNS (3), /* HI */ - COSTS_N_INSNS (3), /* SI */ - COSTS_N_INSNS (3), /* DI */ - COSTS_N_INSNS (3)}, /* other */ - 0, /* cost of multiply per each bit set */ - {COSTS_N_INSNS (22), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (22), /* HI */ - COSTS_N_INSNS (22), /* SI */ - COSTS_N_INSNS (22), /* DI */ - COSTS_N_INSNS (22)}, /* other */ - COSTS_N_INSNS (1), /* cost of movsx */ - COSTS_N_INSNS (1), /* cost of movzx */ - 8, /* "large" insn */ - 16, /* MOVE_RATIO */ - 2, /* cost for loading QImode using movzbl */ - {6, 6, 6}, /* cost of loading integer registers - in QImode, HImode and SImode. - Relative to reg-reg move (2). */ - {4, 4, 4}, /* cost of storing integer registers */ - 2, /* cost of reg,reg fld/fst */ - {6, 6, 6}, /* cost of loading fp registers - in SFmode, DFmode and XFmode */ - {4, 4, 4}, /* cost of storing fp registers - in SFmode, DFmode and XFmode */ - 2, /* cost of moving MMX register */ - {6, 6}, /* cost of loading MMX registers - in SImode and DImode */ - {4, 4}, /* cost of storing MMX registers - in SImode and DImode */ - 2, /* cost of moving SSE register */ - {6, 6, 6}, /* cost of loading SSE registers - in SImode, DImode and TImode */ - {4, 4, 4}, /* cost of storing SSE registers - in SImode, DImode and TImode */ - 2, /* MMX or SSE register to integer */ - 32, /* size of l1 cache. */ - 2048, /* size of l2 cache. */ - 128, /* size of prefetch block */ - 8, /* number of parallel prefetches */ - 3, /* Branch cost */ - COSTS_N_INSNS (3), /* cost of FADD and FSUB insns. */ - COSTS_N_INSNS (5), /* cost of FMUL instruction. */ - COSTS_N_INSNS (32), /* cost of FDIV instruction. */ - COSTS_N_INSNS (1), /* cost of FABS instruction. */ - COSTS_N_INSNS (1), /* cost of FCHS instruction. */ - COSTS_N_INSNS (58), /* cost of FSQRT instruction. */ - {{libcall, {{11, loop}, {-1, rep_prefix_4_byte}}}, - {libcall, {{32, loop}, {64, rep_prefix_4_byte}, - {8192, rep_prefix_8_byte}, {-1, libcall}}}}, - {{libcall, {{8, loop}, {15, unrolled_loop}, - {2048, rep_prefix_4_byte}, {-1, libcall}}}, - {libcall, {{24, loop}, {32, unrolled_loop}, - {8192, rep_prefix_8_byte}, {-1, libcall}}}}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; static const @@ -1041,21 +1569,21 @@ COSTS_N_INSNS (1), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ - COSTS_N_INSNS (4), /* HI */ - COSTS_N_INSNS (3), /* SI */ - COSTS_N_INSNS (4), /* DI */ - COSTS_N_INSNS (2)}, /* other */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (3), /* SI */ + COSTS_N_INSNS (4), /* DI */ + COSTS_N_INSNS (2)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (26), /* HI */ - COSTS_N_INSNS (42), /* SI */ - COSTS_N_INSNS (74), /* DI */ - COSTS_N_INSNS (74)}, /* other */ + COSTS_N_INSNS (26), /* HI */ + COSTS_N_INSNS (42), /* SI */ + COSTS_N_INSNS (74), /* DI */ + COSTS_N_INSNS (74)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 8, /* "large" insn */ 17, /* MOVE_RATIO */ - 2, /* cost for loading QImode using movzbl */ + 2, /* cost for loading QImode using movzbl */ {4, 4, 4}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -1089,22 +1617,22 @@ COSTS_N_INSNS (40), /* cost of FSQRT instruction. */ {{libcall, {{11, loop}, {-1, rep_prefix_4_byte}}}, {libcall, {{32, loop}, {64, rep_prefix_4_byte}, - {8192, rep_prefix_8_byte}, {-1, libcall}}}}, + {8192, rep_prefix_8_byte}, {-1, libcall}}}}, {{libcall, {{8, loop}, {15, unrolled_loop}, - {2048, rep_prefix_4_byte}, {-1, libcall}}}, + {2048, rep_prefix_4_byte}, {-1, libcall}}}, {libcall, {{24, loop}, {32, unrolled_loop}, - {8192, rep_prefix_8_byte}, {-1, libcall}}}}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + {8192, rep_prefix_8_byte}, {-1, libcall}}}}, + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; /* Generic64 should produce code tuned for Nocona and K8. */ @@ -1119,21 +1647,21 @@ COSTS_N_INSNS (1), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ - COSTS_N_INSNS (4), /* HI */ - COSTS_N_INSNS (3), /* SI */ - COSTS_N_INSNS (4), /* DI */ - COSTS_N_INSNS (2)}, /* other */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (3), /* SI */ + COSTS_N_INSNS (4), /* DI */ + COSTS_N_INSNS (2)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (26), /* HI */ - COSTS_N_INSNS (42), /* SI */ - COSTS_N_INSNS (74), /* DI */ - COSTS_N_INSNS (74)}, /* other */ + COSTS_N_INSNS (26), /* HI */ + COSTS_N_INSNS (42), /* SI */ + COSTS_N_INSNS (74), /* DI */ + COSTS_N_INSNS (74)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 8, /* "large" insn */ 17, /* MOVE_RATIO */ - 4, /* cost for loading QImode using movzbl */ + 4, /* cost for loading QImode using movzbl */ {4, 4, 4}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -1158,8 +1686,8 @@ 512, /* size of l2 cache. */ 64, /* size of prefetch block */ 6, /* number of parallel prefetches */ - /* Benchmarks shows large regressions on K8 sixtrack benchmark when this value - is increased to perhaps more appropriate value of 5. */ + /* Benchmarks shows large regressions on K8 sixtrack benchmark when this + value is increased to perhaps more appropriate value of 5. */ 3, /* Branch cost */ COSTS_N_INSNS (8), /* cost of FADD and FSUB insns. */ COSTS_N_INSNS (8), /* cost of FMUL instruction. */ @@ -1171,20 +1699,21 @@ {libcall, {{32, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, {DUMMY_STRINGOP_ALGS, {libcall, {{32, loop}, {8192, rep_prefix_8_byte}, {-1, libcall}}}}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ -}; - -/* Generic32 should produce code tuned for Athlon, PPro, Pentium4, Nocona and K8. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ +}; + +/* Generic32 should produce code tuned for PPro, Pentium4, Nocona, + Athlon and K8. */ static const struct processor_costs generic32_cost = { COSTS_N_INSNS (1), /* cost of an add instruction */ @@ -1192,21 +1721,21 @@ COSTS_N_INSNS (1), /* variable shift costs */ COSTS_N_INSNS (1), /* constant shift costs */ {COSTS_N_INSNS (3), /* cost of starting multiply for QI */ - COSTS_N_INSNS (4), /* HI */ - COSTS_N_INSNS (3), /* SI */ - COSTS_N_INSNS (4), /* DI */ - COSTS_N_INSNS (2)}, /* other */ + COSTS_N_INSNS (4), /* HI */ + COSTS_N_INSNS (3), /* SI */ + COSTS_N_INSNS (4), /* DI */ + COSTS_N_INSNS (2)}, /* other */ 0, /* cost of multiply per each bit set */ {COSTS_N_INSNS (18), /* cost of a divide/mod for QI */ - COSTS_N_INSNS (26), /* HI */ - COSTS_N_INSNS (42), /* SI */ - COSTS_N_INSNS (74), /* DI */ - COSTS_N_INSNS (74)}, /* other */ + COSTS_N_INSNS (26), /* HI */ + COSTS_N_INSNS (42), /* SI */ + COSTS_N_INSNS (74), /* DI */ + COSTS_N_INSNS (74)}, /* other */ COSTS_N_INSNS (1), /* cost of movsx */ COSTS_N_INSNS (1), /* cost of movzx */ 8, /* "large" insn */ 17, /* MOVE_RATIO */ - 4, /* cost for loading QImode using movzbl */ + 4, /* cost for loading QImode using movzbl */ {4, 4, 4}, /* cost of loading integer registers in QImode, HImode and SImode. Relative to reg-reg move (2). */ @@ -1242,17 +1771,17 @@ DUMMY_STRINGOP_ALGS}, {{libcall, {{32, loop}, {8192, rep_prefix_4_byte}, {-1, libcall}}}, DUMMY_STRINGOP_ALGS}, - 1, /* scalar_stmt_cost. */ - 1, /* scalar load_cost. */ - 1, /* scalar_store_cost. */ - 1, /* vec_stmt_cost. */ - 1, /* vec_to_scalar_cost. */ - 1, /* scalar_to_vec_cost. */ - 1, /* vec_align_load_cost. */ - 2, /* vec_unalign_load_cost. */ - 1, /* vec_store_cost. */ - 3, /* cond_taken_branch_cost. */ - 1, /* cond_not_taken_branch_cost. */ + 1, /* scalar_stmt_cost. */ + 1, /* scalar load_cost. */ + 1, /* scalar_store_cost. */ + 1, /* vec_stmt_cost. */ + 1, /* vec_to_scalar_cost. */ + 1, /* scalar_to_vec_cost. */ + 1, /* vec_align_load_cost. */ + 2, /* vec_unalign_load_cost. */ + 1, /* vec_store_cost. */ + 3, /* cond_taken_branch_cost. */ + 1, /* cond_not_taken_branch_cost. */ }; const struct processor_costs *ix86_cost = &pentium_cost; @@ -1264,7 +1793,14 @@ #define m_PPRO (1<<PROCESSOR_PENTIUMPRO) #define m_PENT4 (1<<PROCESSOR_PENTIUM4) #define m_NOCONA (1<<PROCESSOR_NOCONA) -#define m_CORE2 (1<<PROCESSOR_CORE2) +#define m_CORE2_32 (1<<PROCESSOR_CORE2_32) +#define m_CORE2_64 (1<<PROCESSOR_CORE2_64) +#define m_COREI7_32 (1<<PROCESSOR_COREI7_32) +#define m_COREI7_64 (1<<PROCESSOR_COREI7_64) +#define m_COREI7 (m_COREI7_32 | m_COREI7_64) +#define m_CORE2I7_32 (m_CORE2_32 | m_COREI7_32) +#define m_CORE2I7_64 (m_CORE2_64 | m_COREI7_64) +#define m_CORE2I7 (m_CORE2I7_32 | m_CORE2I7_64) #define m_ATOM (1<<PROCESSOR_ATOM) #define m_GEODE (1<<PROCESSOR_GEODE) @@ -1274,7 +1810,9 @@ #define m_ATHLON (1<<PROCESSOR_ATHLON) #define m_ATHLON_K8 (m_K8 | m_ATHLON) #define m_AMDFAM10 (1<<PROCESSOR_AMDFAM10) -#define m_AMD_MULTIPLE (m_K8 | m_ATHLON | m_AMDFAM10) +#define m_BDVER1 (1<<PROCESSOR_BDVER1) +#define m_BTVER1 (1<<PROCESSOR_BTVER1) +#define m_AMD_MULTIPLE (m_K8 | m_ATHLON | m_AMDFAM10 | m_BDVER1 | m_BTVER1) #define m_GENERIC32 (1<<PROCESSOR_GENERIC32) #define m_GENERIC64 (1<<PROCESSOR_GENERIC64) @@ -1293,21 +1831,22 @@ negatively, so enabling for Generic64 seems like good code size tradeoff. We can't enable it for 32bit generic because it does not work well with PPro base chips. */ - m_386 | m_K6_GEODE | m_AMD_MULTIPLE | m_CORE2 | m_GENERIC64, + m_386 | m_K6_GEODE | m_AMD_MULTIPLE | m_CORE2I7_64 | m_GENERIC64, /* X86_TUNE_PUSH_MEMORY */ m_386 | m_K6_GEODE | m_AMD_MULTIPLE | m_PENT4 - | m_NOCONA | m_CORE2 | m_GENERIC, + | m_NOCONA | m_CORE2I7 | m_GENERIC, /* X86_TUNE_ZERO_EXTEND_WITH_AND */ m_486 | m_PENT, /* X86_TUNE_UNROLL_STRLEN */ m_486 | m_PENT | m_ATOM | m_PPRO | m_AMD_MULTIPLE | m_K6 - | m_CORE2 | m_GENERIC, + | m_CORE2I7 | m_GENERIC, /* X86_TUNE_DEEP_BRANCH_PREDICTION */ - m_ATOM | m_PPRO | m_K6_GEODE | m_AMD_MULTIPLE | m_PENT4 | m_GENERIC, + m_ATOM | m_PPRO | m_K6_GEODE | m_AMD_MULTIPLE | m_PENT4 + | m_CORE2I7 | m_GENERIC, /* X86_TUNE_BRANCH_PREDICTION_HINTS: Branch hints were put in P4 based on simulation result. But after P4 was made, no performance benefit @@ -1319,13 +1858,13 @@ ~m_386, /* X86_TUNE_USE_SAHF */ - m_ATOM | m_PPRO | m_K6_GEODE | m_K8 | m_AMDFAM10 | m_PENT4 - | m_NOCONA | m_CORE2 | m_GENERIC, + m_ATOM | m_PPRO | m_K6_GEODE | m_K8 | m_AMDFAM10 | m_BDVER1 | m_BTVER1 + | m_PENT4 | m_NOCONA | m_CORE2I7 | m_GENERIC, /* X86_TUNE_MOVX: Enable to zero extend integer registers to avoid partial dependencies. */ m_AMD_MULTIPLE | m_ATOM | m_PPRO | m_PENT4 | m_NOCONA - | m_CORE2 | m_GENERIC | m_GEODE /* m_386 | m_K6 */, + | m_CORE2I7 | m_GENERIC | m_GEODE /* m_386 | m_K6 */, /* X86_TUNE_PARTIAL_REG_STALL: We probably ought to watch for partial register stalls on Generic32 compilation setting as well. However @@ -1338,19 +1877,19 @@ m_PPRO, /* X86_TUNE_PARTIAL_FLAG_REG_STALL */ - m_CORE2 | m_GENERIC, + m_CORE2I7 | m_GENERIC, /* X86_TUNE_USE_HIMODE_FIOP */ m_386 | m_486 | m_K6_GEODE, /* X86_TUNE_USE_SIMODE_FIOP */ - ~(m_PPRO | m_AMD_MULTIPLE | m_PENT | m_ATOM | m_CORE2 | m_GENERIC), + ~(m_PPRO | m_AMD_MULTIPLE | m_PENT | m_ATOM | m_CORE2I7 | m_GENERIC), /* X86_TUNE_USE_MOV0 */ m_K6, /* X86_TUNE_USE_CLTD */ - ~(m_PENT | m_ATOM | m_K6 | m_CORE2 | m_GENERIC), + ~(m_PENT | m_ATOM | m_K6 | m_CORE2I7 | m_GENERIC), /* X86_TUNE_USE_XCHGB: Use xchgb %rh,%rl instead of rolw/rorw $8,rx. */ m_PENT4, @@ -1366,7 +1905,7 @@ /* X86_TUNE_PROMOTE_QIMODE */ m_K6_GEODE | m_PENT | m_ATOM | m_386 | m_486 | m_AMD_MULTIPLE - | m_CORE2 | m_GENERIC /* | m_PENT4 ? */, + | m_CORE2I7 | m_GENERIC /* | m_PENT4 ? */, /* X86_TUNE_FAST_PREFIX */ ~(m_PENT | m_486 | m_386), @@ -1389,29 +1928,29 @@ /* X86_TUNE_PROMOTE_HI_REGS */ m_PPRO, - /* X86_TUNE_ADD_ESP_4: Enable if add/sub is preferred over 1/2 push/pop. */ - m_ATOM | m_AMD_MULTIPLE | m_K6_GEODE | m_PENT4 | m_NOCONA - | m_CORE2 | m_GENERIC, - - /* X86_TUNE_ADD_ESP_8 */ - m_AMD_MULTIPLE | m_ATOM | m_PPRO | m_K6_GEODE | m_386 - | m_486 | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC, - - /* X86_TUNE_SUB_ESP_4 */ - m_AMD_MULTIPLE | m_ATOM | m_PPRO | m_PENT4 | m_NOCONA | m_CORE2 - | m_GENERIC, - - /* X86_TUNE_SUB_ESP_8 */ - m_AMD_MULTIPLE | m_ATOM | m_PPRO | m_386 | m_486 - | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC, + /* X86_TUNE_SINGLE_POP: Enable if single pop insn is preferred + over esp addition. */ + m_386 | m_486 | m_PENT | m_PPRO, + + /* X86_TUNE_DOUBLE_POP: Enable if double pop insn is preferred + over esp addition. */ + m_PENT, + + /* X86_TUNE_SINGLE_PUSH: Enable if single push insn is preferred + over esp subtraction. */ + m_386 | m_486 | m_PENT | m_K6_GEODE, + + /* X86_TUNE_DOUBLE_PUSH. Enable if double push insn is preferred + over esp subtraction. */ + m_PENT | m_K6_GEODE, /* X86_TUNE_INTEGER_DFMODE_MOVES: Enable if integer moves are preferred for DFmode copies */ - ~(m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2 + ~(m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2I7 | m_GENERIC | m_GEODE), /* X86_TUNE_PARTIAL_REG_DEPENDENCY */ - m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC, + m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_CORE2I7 | m_GENERIC, /* X86_TUNE_SSE_PARTIAL_REG_DEPENDENCY: In the Generic model we have a conflict here in between PPro/Pentium4 based chips that thread 128bit @@ -1422,11 +1961,17 @@ shows that disabling this option on P4 brings over 20% SPECfp regression, while enabling it on K8 brings roughly 2.4% regression that can be partly masked by careful scheduling of moves. */ - m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2 | m_GENERIC - | m_AMDFAM10, - - /* X86_TUNE_SSE_UNALIGNED_MOVE_OPTIMAL */ - m_AMDFAM10, + m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2I7 | m_GENERIC + | m_AMDFAM10 | m_BDVER1, + + /* X86_TUNE_SSE_UNALIGNED_LOAD_OPTIMAL */ + m_AMDFAM10 | m_BDVER1 | m_BTVER1 | m_COREI7, + + /* X86_TUNE_SSE_UNALIGNED_STORE_OPTIMAL */ + m_BDVER1 | m_COREI7, + + /* X86_TUNE_SSE_PACKED_SINGLE_INSN_OPTIMAL */ + m_BDVER1, /* X86_TUNE_SSE_SPLIT_REGS: Set for machines where the type and dependencies are resolved on SSE register parts instead of whole registers, so we may @@ -1441,13 +1986,13 @@ m_PPRO | m_PENT4 | m_NOCONA, /* X86_TUNE_MEMORY_MISMATCH_STALL */ - m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_CORE2 | m_GENERIC, + m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_CORE2I7 | m_GENERIC, /* X86_TUNE_PROLOGUE_USING_MOVE */ - m_ATHLON_K8 | m_ATOM | m_PPRO | m_CORE2 | m_GENERIC, + m_ATHLON_K8 | m_ATOM | m_PPRO | m_CORE2I7 | m_GENERIC, /* X86_TUNE_EPILOGUE_USING_MOVE */ - m_ATHLON_K8 | m_ATOM | m_PPRO | m_CORE2 | m_GENERIC, + m_ATHLON_K8 | m_ATOM | m_PPRO | m_CORE2I7 | m_GENERIC, /* X86_TUNE_SHIFT1 */ ~m_486, @@ -1456,38 +2001,41 @@ m_AMD_MULTIPLE, /* X86_TUNE_INTER_UNIT_MOVES */ - ~(m_AMD_MULTIPLE | m_ATOM | m_GENERIC), + ~(m_AMD_MULTIPLE | m_GENERIC), /* X86_TUNE_INTER_UNIT_CONVERSIONS */ - ~(m_AMDFAM10), + ~(m_AMDFAM10 | m_BDVER1), /* X86_TUNE_FOUR_JUMP_LIMIT: Some CPU cores are not able to predict more than 4 branch instructions in the 16 byte window. */ - m_ATOM | m_PPRO | m_AMD_MULTIPLE | m_PENT4 | m_NOCONA | m_CORE2 + m_ATOM | m_PPRO | m_AMD_MULTIPLE | m_PENT4 | m_NOCONA | m_CORE2I7 | m_GENERIC, /* X86_TUNE_SCHEDULE */ - m_PPRO | m_AMD_MULTIPLE | m_K6_GEODE | m_PENT | m_ATOM | m_CORE2 + m_PPRO | m_AMD_MULTIPLE | m_K6_GEODE | m_PENT | m_ATOM | m_CORE2I7 | m_GENERIC, /* X86_TUNE_USE_BT */ - m_AMD_MULTIPLE | m_ATOM | m_CORE2 | m_GENERIC, + m_AMD_MULTIPLE | m_ATOM | m_CORE2I7 | m_GENERIC, /* X86_TUNE_USE_INCDEC */ - ~(m_PENT4 | m_NOCONA | m_GENERIC | m_ATOM), + ~(m_PENT4 | m_NOCONA | m_CORE2I7 | m_GENERIC | m_ATOM), /* X86_TUNE_PAD_RETURNS */ - m_AMD_MULTIPLE | m_CORE2 | m_GENERIC, + m_AMD_MULTIPLE | m_CORE2I7 | m_GENERIC, + + /* X86_TUNE_PAD_SHORT_FUNCTION: Pad short funtion. */ + m_ATOM, /* X86_TUNE_EXT_80387_CONSTANTS */ m_K6_GEODE | m_ATHLON_K8 | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO - | m_CORE2 | m_GENERIC, + | m_CORE2I7 | m_GENERIC, /* X86_TUNE_SHORTEN_X87_SSE */ ~m_K8, /* X86_TUNE_AVOID_VECTOR_DECODE */ - m_K8 | m_GENERIC64, + m_K8 | m_CORE2I7_64 | m_GENERIC64, /* X86_TUNE_PROMOTE_HIMODE_IMUL: Modern CPUs have same latency for HImode and SImode multiply, but 386 and 486 do HImode multiply faster. */ @@ -1495,11 +2043,11 @@ /* X86_TUNE_SLOW_IMUL_IMM32_MEM: Imul of 32-bit constant and memory is vector path on AMD machines. */ - m_K8 | m_GENERIC64 | m_AMDFAM10, + m_K8 | m_CORE2I7_64 | m_GENERIC64 | m_AMDFAM10 | m_BDVER1 | m_BTVER1, /* X86_TUNE_SLOW_IMUL_IMM8: Imul of 8-bit constant is vector path on AMD machines. */ - m_K8 | m_GENERIC64 | m_AMDFAM10, + m_K8 | m_CORE2I7_64 | m_GENERIC64 | m_AMDFAM10 | m_BDVER1 | m_BTVER1, /* X86_TUNE_MOVE_M1_VIA_OR: On pentiums, it is faster to load -1 via OR than a MOV. */ @@ -1516,7 +2064,7 @@ /* X86_TUNE_USE_VECTOR_FP_CONVERTS: Prefer vector packed SSE conversion from FP to FP. */ - m_AMDFAM10 | m_GENERIC, + m_AMDFAM10 | m_CORE2I7 | m_GENERIC, /* X86_TUNE_USE_VECTOR_CONVERTS: Prefer vector packed SSE conversion from integer to FP. */ @@ -1525,11 +2073,15 @@ /* X86_TUNE_FUSE_CMP_AND_BRANCH: Fuse a compare or test instruction with a subsequent conditional jump instruction into a single compare-and-branch uop. */ - m_CORE2, + m_BDVER1, /* X86_TUNE_OPT_AGU: Optimize for Address Generation Unit. This flag will impact LEA instruction selection. */ m_ATOM, + + /* X86_TUNE_VECTORIZE_DOUBLE: Enable double precision vector + instructions. */ + ~m_ATOM, }; /* Feature tests against the various architecture variations. */ @@ -1555,12 +2107,12 @@ }; static const unsigned int x86_accumulate_outgoing_args - = m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2 + = m_AMD_MULTIPLE | m_ATOM | m_PENT4 | m_NOCONA | m_PPRO | m_CORE2I7 | m_GENERIC; static const unsigned int x86_arch_always_fancy_math_387 = m_PENT | m_ATOM | m_PPRO | m_AMD_MULTIPLE | m_PENT4 - | m_NOCONA | m_CORE2 | m_GENERIC; + | m_NOCONA | m_CORE2I7 | m_GENERIC; static enum stringop_alg stringop_alg = no_stringop; @@ -1695,12 +2247,6 @@ -1, -1, -1, -1, -1, -1, -1, -1, /* extended SSE registers */ }; -/* Test and compare insns in i386.md store the information needed to - generate branch and scc insns here. */ - -rtx ix86_compare_op0 = NULL_RTX; -rtx ix86_compare_op1 = NULL_RTX; - /* Define parameter passing and return registers. */ static int const x86_64_int_parameter_registers[6] = @@ -1731,42 +2277,44 @@ Stack grows downward: [arguments] - <- ARG_POINTER + <- ARG_POINTER saved pc - saved frame pointer if frame_pointer_needed - <- HARD_FRAME_POINTER + saved static chain if ix86_static_chain_on_stack + + saved frame pointer if frame_pointer_needed + <- HARD_FRAME_POINTER [saved regs] - + <- regs_save_offset [padding0] [saved SSE regs] - - [padding1] \ - ) - [va_arg registers] ( - > to_allocate <- FRAME_POINTER - [frame] ( - ) - [padding2] / + <- sse_regs_save_offset + [padding1] | + | <- FRAME_POINTER + [va_arg registers] | + | + [frame] | + | + [padding2] | = to_allocate + <- STACK_POINTER */ struct ix86_frame { - int padding0; int nsseregs; int nregs; - int padding1; int va_arg_size; + int red_zone_size; + int outgoing_arguments_size; HOST_WIDE_INT frame; - int padding2; - int outgoing_arguments_size; - int red_zone_size; - - HOST_WIDE_INT to_allocate; + /* The offsets relative to ARG_POINTER. */ HOST_WIDE_INT frame_pointer_offset; HOST_WIDE_INT hard_frame_pointer_offset; HOST_WIDE_INT stack_pointer_offset; + HOST_WIDE_INT hfp_save_offset; + HOST_WIDE_INT reg_save_offset; + HOST_WIDE_INT sse_reg_save_offset; /* When save_regs_using_mov is set, emit prologue using move instead of push instructions. */ @@ -1799,18 +2347,19 @@ static int ix86_regparm; /* -mstackrealign option */ -extern int ix86_force_align_arg_pointer; static const char ix86_force_align_arg_pointer_string[] = "force_align_arg_pointer"; static rtx (*ix86_gen_leave) (void); -static rtx (*ix86_gen_pop1) (rtx); static rtx (*ix86_gen_add3) (rtx, rtx, rtx); static rtx (*ix86_gen_sub3) (rtx, rtx, rtx); static rtx (*ix86_gen_sub3_carry) (rtx, rtx, rtx, rtx, rtx); static rtx (*ix86_gen_one_cmpl2) (rtx, rtx); static rtx (*ix86_gen_monitor) (rtx, rtx, rtx); static rtx (*ix86_gen_andsp) (rtx, rtx, rtx); +static rtx (*ix86_gen_allocate_stack_worker) (rtx, rtx); +static rtx (*ix86_gen_adjust_stack_and_probe) (rtx, rtx, rtx); +static rtx (*ix86_gen_probe_stack_range) (rtx, rtx, rtx); /* Preferred alignment for stack boundary in bits. */ unsigned int ix86_preferred_stack_boundary; @@ -1878,6 +2427,9 @@ static struct machine_function * ix86_init_machine_status (void); static rtx ix86_function_value (const_tree, const_tree, bool); +static bool ix86_function_value_regno_p (const unsigned int); +static unsigned int ix86_function_arg_boundary (enum machine_mode, + const_tree); static rtx ix86_static_chain (const_tree, bool); static int ix86_function_regparm (const_tree, const_tree); static void ix86_compute_frame_layout (struct ix86_frame *); @@ -1885,6 +2437,10 @@ rtx, rtx, int); static void ix86_add_new_builtins (int); static rtx ix86_expand_vec_perm_builtin (tree); +static tree ix86_canonical_va_list_type (tree); +static void predict_jump (int); +static unsigned int split_stack_prologue_scratch_regno (void); +static bool i386_asm_output_addr_const_extra (FILE *, rtx); enum ix86_function_specific_strings { @@ -1910,6 +2466,10 @@ static enum calling_abi ix86_function_abi (const_tree); +#ifndef SUBTARGET32_DEFAULT_CPU +#define SUBTARGET32_DEFAULT_CPU "i386" +#endif + /* The svr4 ABI for the i386 says that records and unions are returned in memory. */ #ifndef DEFAULT_PCC_STRUCT_RETURN @@ -1920,9 +2480,6 @@ static int ix86_tune_defaulted; static int ix86_arch_specified; -/* Bit flags that specify the ISA we are compiling for. */ -int ix86_isa_flags = TARGET_64BIT_DEFAULT | TARGET_SUBTARGET_ISA_DEFAULT; - /* A mask of ix86_isa_flags that includes bit X if X was set or cleared on the command line. */ static int ix86_isa_flags_explicit; @@ -1973,12 +2530,19 @@ #define OPTION_MASK_ISA_ABM_SET \ (OPTION_MASK_ISA_ABM | OPTION_MASK_ISA_POPCNT) +#define OPTION_MASK_ISA_BMI_SET OPTION_MASK_ISA_BMI +#define OPTION_MASK_ISA_TBM_SET OPTION_MASK_ISA_TBM #define OPTION_MASK_ISA_POPCNT_SET OPTION_MASK_ISA_POPCNT #define OPTION_MASK_ISA_CX16_SET OPTION_MASK_ISA_CX16 #define OPTION_MASK_ISA_SAHF_SET OPTION_MASK_ISA_SAHF #define OPTION_MASK_ISA_MOVBE_SET OPTION_MASK_ISA_MOVBE #define OPTION_MASK_ISA_CRC32_SET OPTION_MASK_ISA_CRC32 +#define OPTION_MASK_ISA_FSGSBASE_SET OPTION_MASK_ISA_FSGSBASE +#define OPTION_MASK_ISA_RDRND_SET OPTION_MASK_ISA_RDRND +#define OPTION_MASK_ISA_F16C_SET \ + (OPTION_MASK_ISA_F16C | OPTION_MASK_ISA_AVX_SET) + /* Define a set of ISAs which aren't available when a given ISA is disabled. MMX and SSE ISAs are handled separately. */ @@ -2004,7 +2568,7 @@ (OPTION_MASK_ISA_SSE4_2 | OPTION_MASK_ISA_AVX_UNSET ) #define OPTION_MASK_ISA_AVX_UNSET \ (OPTION_MASK_ISA_AVX | OPTION_MASK_ISA_FMA_UNSET \ - | OPTION_MASK_ISA_FMA4_UNSET) + | OPTION_MASK_ISA_FMA4_UNSET | OPTION_MASK_ISA_F16C_UNSET) #define OPTION_MASK_ISA_FMA_UNSET OPTION_MASK_ISA_FMA /* SSE4 includes both SSE4.1 and SSE4.2. -mno-sse4 should the same @@ -2022,14 +2586,21 @@ #define OPTION_MASK_ISA_AES_UNSET OPTION_MASK_ISA_AES #define OPTION_MASK_ISA_PCLMUL_UNSET OPTION_MASK_ISA_PCLMUL #define OPTION_MASK_ISA_ABM_UNSET OPTION_MASK_ISA_ABM +#define OPTION_MASK_ISA_BMI_UNSET OPTION_MASK_ISA_BMI +#define OPTION_MASK_ISA_TBM_UNSET OPTION_MASK_ISA_TBM #define OPTION_MASK_ISA_POPCNT_UNSET OPTION_MASK_ISA_POPCNT #define OPTION_MASK_ISA_CX16_UNSET OPTION_MASK_ISA_CX16 #define OPTION_MASK_ISA_SAHF_UNSET OPTION_MASK_ISA_SAHF #define OPTION_MASK_ISA_MOVBE_UNSET OPTION_MASK_ISA_MOVBE #define OPTION_MASK_ISA_CRC32_UNSET OPTION_MASK_ISA_CRC32 +#define OPTION_MASK_ISA_FSGSBASE_UNSET OPTION_MASK_ISA_FSGSBASE +#define OPTION_MASK_ISA_RDRND_UNSET OPTION_MASK_ISA_RDRND +#define OPTION_MASK_ISA_F16C_UNSET OPTION_MASK_ISA_F16C + /* Vectorization library interface and handlers. */ -tree (*ix86_veclib_handler)(enum built_in_function, tree, tree) = NULL; +static tree (*ix86_veclib_handler) (enum built_in_function, tree, tree); + static tree ix86_veclibabi_svml (enum built_in_function, tree, tree); static tree ix86_veclibabi_acml (enum built_in_function, tree, tree); @@ -2056,10 +2627,19 @@ {&pentium4_cost, 0, 0, 0, 0, 0}, {&k8_cost, 16, 7, 16, 7, 16}, {&nocona_cost, 0, 0, 0, 0, 0}, - {&core2_cost, 16, 10, 16, 10, 16}, + /* Core 2 32-bit. */ + {&generic32_cost, 16, 10, 16, 10, 16}, + /* Core 2 64-bit. */ + {&generic64_cost, 16, 10, 16, 10, 16}, + /* Core i7 32-bit. */ + {&generic32_cost, 16, 10, 16, 10, 16}, + /* Core i7 64-bit. */ + {&generic64_cost, 16, 10, 16, 10, 16}, {&generic32_cost, 16, 7, 16, 7, 16}, {&generic64_cost, 16, 10, 16, 10, 16}, {&amdfam10_cost, 32, 24, 32, 7, 32}, + {&bdver1_cost, 32, 24, 32, 7, 32}, + {&btver1_cost, 32, 24, 32, 7, 32}, {&atom_cost, 16, 7, 16, 7, 16} }; @@ -2078,6 +2658,7 @@ "prescott", "nocona", "core2", + "corei7", "atom", "geode", "k6", @@ -2086,9 +2667,19 @@ "athlon", "athlon-4", "k8", - "amdfam10" -}; - + "amdfam10", + "bdver1", + "btver1" +}; + +/* Return true if a red-zone is in use. */ + +static inline bool +ix86_using_red_zone (void) +{ + return TARGET_RED_ZONE && !TARGET_64BIT_MS_ABI; +} + /* Implement TARGET_HANDLE_OPTION. */ static bool @@ -2304,6 +2895,32 @@ } return true; + case OPT_mbmi: + if (value) + { + ix86_isa_flags |= OPTION_MASK_ISA_BMI_SET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_BMI_SET; + } + else + { + ix86_isa_flags &= ~OPTION_MASK_ISA_BMI_UNSET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_BMI_UNSET; + } + return true; + + case OPT_mtbm: + if (value) + { + ix86_isa_flags |= OPTION_MASK_ISA_TBM_SET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_TBM_SET; + } + else + { + ix86_isa_flags &= ~OPTION_MASK_ISA_TBM_UNSET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_TBM_UNSET; + } + return true; + case OPT_mpopcnt: if (value) { @@ -2395,12 +3012,51 @@ } return true; - default: - return true; - } -} - -/* Return a string the documents the current -m options. The caller is + case OPT_mfsgsbase: + if (value) + { + ix86_isa_flags |= OPTION_MASK_ISA_FSGSBASE_SET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_FSGSBASE_SET; + } + else + { + ix86_isa_flags &= ~OPTION_MASK_ISA_FSGSBASE_UNSET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_FSGSBASE_UNSET; + } + return true; + + case OPT_mrdrnd: + if (value) + { + ix86_isa_flags |= OPTION_MASK_ISA_RDRND_SET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_RDRND_SET; + } + else + { + ix86_isa_flags &= ~OPTION_MASK_ISA_RDRND_UNSET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_RDRND_UNSET; + } + return true; + + case OPT_mf16c: + if (value) + { + ix86_isa_flags |= OPTION_MASK_ISA_F16C_SET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_F16C_SET; + } + else + { + ix86_isa_flags &= ~OPTION_MASK_ISA_F16C_UNSET; + ix86_isa_flags_explicit |= OPTION_MASK_ISA_F16C_UNSET; + } + return true; + + default: + return true; + } +} + +/* Return a string that documents the current -m options. The caller is responsible for freeing the string. */ static char * @@ -2419,6 +3075,7 @@ { { "-m64", OPTION_MASK_ISA_64BIT }, { "-mfma4", OPTION_MASK_ISA_FMA4 }, + { "-mfma", OPTION_MASK_ISA_FMA }, { "-mxop", OPTION_MASK_ISA_XOP }, { "-mlwp", OPTION_MASK_ISA_LWP }, { "-msse4a", OPTION_MASK_ISA_SSE4A }, @@ -2432,11 +3089,16 @@ { "-m3dnowa", OPTION_MASK_ISA_3DNOW_A }, { "-mmmx", OPTION_MASK_ISA_MMX }, { "-mabm", OPTION_MASK_ISA_ABM }, + { "-mbmi", OPTION_MASK_ISA_BMI }, + { "-mtbm", OPTION_MASK_ISA_TBM }, { "-mpopcnt", OPTION_MASK_ISA_POPCNT }, { "-mmovbe", OPTION_MASK_ISA_MOVBE }, { "-mcrc32", OPTION_MASK_ISA_CRC32 }, { "-maes", OPTION_MASK_ISA_AES }, { "-mpclmul", OPTION_MASK_ISA_PCLMUL }, + { "-mfsgsbase", OPTION_MASK_ISA_FSGSBASE }, + { "-mrdrnd", OPTION_MASK_ISA_RDRND }, + { "-mf16c", OPTION_MASK_ISA_F16C }, }; /* Flag options. */ @@ -2462,6 +3124,9 @@ { "-msseregparm", MASK_SSEREGPARM }, { "-mstack-arg-probe", MASK_STACK_PROBE }, { "-mtls-direct-seg-refs", MASK_TLS_DIRECT_SEG_REFS }, + { "-mvect8-ret-in-mem", MASK_VECT8_RETURNS }, + { "-m8bit-idiv", MASK_USE_8BIT_IDIV }, + { "-mvzeroupper", MASK_VZEROUPPER }, }; const char *opts[ARRAY_SIZE (isa_opts) + ARRAY_SIZE (flag_opts) + 6][2]; @@ -2505,7 +3170,7 @@ if (isa && add_nl_p) { opts[num++][0] = isa_other; - sprintf (isa_other, "(other isa: 0x%x)", isa); + sprintf (isa_other, "(other isa: %#x)", isa); } /* Add flag options. */ @@ -2521,7 +3186,7 @@ if (flags && add_nl_p) { opts[num++][0] = target_other; - sprintf (target_other, "(other flags: 0x%x)", isa); + sprintf (target_other, "(other flags: %#x)", flags); } /* Add -fpmath= option. */ @@ -2587,6 +3252,36 @@ return ret; } +/* Return TRUE if software prefetching is beneficial for the + given CPU. */ + +static bool +software_prefetching_beneficial_p (void) +{ + switch (ix86_tune) + { + case PROCESSOR_GEODE: + case PROCESSOR_K6: + case PROCESSOR_ATHLON: + case PROCESSOR_K8: + case PROCESSOR_AMDFAM10: + case PROCESSOR_BTVER1: + return true; + + default: + return false; + } +} + +/* Return true, if profiling code should be emitted before + prologue. Otherwise it returns false. + Note: For x86 with "hotfix" it is sorried. */ +static bool +ix86_profile_before_prologue (void) +{ + return flag_fentry != 0; +} + /* Function that is callable from the debugger to print the current options. */ void @@ -2607,20 +3302,16 @@ return; } -/* Sometimes certain combinations of command options do not make - sense on a particular target machine. You can define a macro - `OVERRIDE_OPTIONS' to take account of this. This macro, if - defined, is executed once just after all the command options have - been parsed. - - Don't use this macro to turn on various extra optimizations for - `-O'. That is what `OPTIMIZATION_OPTIONS' is for. */ - -void -override_options (bool main_args_p) +/* Override various settings based on options. If MAIN_ARGS_P, the + options are from the command line, otherwise they are from + attributes. */ + +static void +ix86_option_override_internal (bool main_args_p) { int i; unsigned int ix86_arch_mask, ix86_tune_mask; + const bool ix86_tune_specified = (ix86_tune_string != NULL); const char *prefix; const char *suffix; const char *sw; @@ -2653,7 +3344,13 @@ PTA_MOVBE = 1 << 20, PTA_FMA4 = 1 << 21, PTA_XOP = 1 << 22, - PTA_LWP = 1 << 23 + PTA_LWP = 1 << 23, + PTA_FSGSBASE = 1 << 24, + PTA_RDRND = 1 << 25, + PTA_F16C = 1 << 26, + PTA_BMI = 1 << 27, + PTA_TBM = 1 << 28 + /* if this reaches 32, need to widen struct pta flags below */ }; static struct pta @@ -2692,9 +3389,16 @@ {"nocona", PROCESSOR_NOCONA, CPU_NONE, PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_CX16 | PTA_NO_SAHF}, - {"core2", PROCESSOR_CORE2, CPU_CORE2, + {"core2", PROCESSOR_CORE2_64, CPU_CORE2, PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSSE3 | PTA_CX16}, + {"corei7", PROCESSOR_COREI7_64, CPU_COREI7, + PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 + | PTA_SSSE3 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_CX16}, + {"corei7-avx", PROCESSOR_COREI7_64, CPU_COREI7, + PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 + | PTA_SSSE3 | PTA_SSE4_1 | PTA_SSE4_2 | PTA_AVX + | PTA_CX16 | PTA_POPCNT | PTA_AES | PTA_PCLMUL}, {"atom", PROCESSOR_ATOM, CPU_ATOM, PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSSE3 | PTA_CX16 | PTA_MOVBE}, @@ -2742,6 +3446,14 @@ {"barcelona", PROCESSOR_AMDFAM10, CPU_AMDFAM10, PTA_64BIT | PTA_MMX | PTA_3DNOW | PTA_3DNOW_A | PTA_SSE | PTA_SSE2 | PTA_SSE3 | PTA_SSE4A | PTA_CX16 | PTA_ABM}, + {"bdver1", PROCESSOR_BDVER1, CPU_BDVER1, + PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 + | PTA_SSE4A | PTA_CX16 | PTA_ABM | PTA_SSSE3 | PTA_SSE4_1 + | PTA_SSE4_2 | PTA_AES | PTA_PCLMUL | PTA_AVX | PTA_FMA4 + | PTA_XOP | PTA_LWP}, + {"btver1", PROCESSOR_BTVER1, CPU_GENERIC64, + PTA_64BIT | PTA_MMX | PTA_SSE | PTA_SSE2 | PTA_SSE3 + | PTA_SSSE3 | PTA_SSE4A |PTA_ABM | PTA_CX16}, {"generic32", PROCESSOR_GENERIC32, CPU_PENTIUMPRO, 0 /* flags are only used for -march switch. */ }, {"generic64", PROCESSOR_GENERIC64, CPU_GENERIC64, @@ -2777,28 +3489,6 @@ if (TARGET_MACHO && TARGET_64BIT) flag_pic = 2; - /* Set the default values for switches whose default depends on TARGET_64BIT - in case they weren't overwritten by command line options. */ - if (TARGET_64BIT) - { - /* Mach-O doesn't support omitting the frame pointer for now. */ - if (flag_omit_frame_pointer == 2) - flag_omit_frame_pointer = (TARGET_MACHO ? 0 : 1); - if (flag_asynchronous_unwind_tables == 2) - flag_asynchronous_unwind_tables = 1; - if (flag_pcc_struct_return == 2) - flag_pcc_struct_return = 0; - } - else - { - if (flag_omit_frame_pointer == 2) - flag_omit_frame_pointer = 0; - if (flag_asynchronous_unwind_tables == 2) - flag_asynchronous_unwind_tables = 0; - if (flag_pcc_struct_return == 2) - flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN; - } - /* Need to check -mtune=generic first. */ if (ix86_tune_string) { @@ -2821,8 +3511,12 @@ || !strcmp (ix86_tune_string, "generic64"))) ; else if (!strncmp (ix86_tune_string, "generic", 7)) - error ("bad value (%s) for %stune=%s %s", + error ("bad value (%s) for %stune=%s %s", ix86_tune_string, prefix, suffix, sw); + else if (!strcmp (ix86_tune_string, "x86-64")) + warning (OPT_Wdeprecated, "%stune=x86-64%s is deprecated; use " + "%stune=k8%s or %stune=generic%s instead as appropriate", + prefix, suffix, prefix, suffix, prefix, suffix); } else { @@ -2846,6 +3540,7 @@ ix86_tune_string = "generic32"; } } + if (ix86_stringop_string) { if (!strcmp (ix86_stringop_string, "rep_byte")) @@ -2868,23 +3563,12 @@ error ("bad value (%s) for %sstringop-strategy=%s %s", ix86_stringop_string, prefix, suffix, sw); } - if (!strcmp (ix86_tune_string, "x86-64")) - warning (OPT_Wdeprecated, "%stune=x86-64%s is deprecated. Use " - "%stune=k8%s or %stune=generic%s instead as appropriate.", - prefix, suffix, prefix, suffix, prefix, suffix); if (!ix86_arch_string) - ix86_arch_string = TARGET_64BIT ? "x86-64" : "i386"; + ix86_arch_string = TARGET_64BIT ? "x86-64" : SUBTARGET32_DEFAULT_CPU; else ix86_arch_specified = 1; - if (!strcmp (ix86_arch_string, "generic")) - error ("generic CPU can be used only for %stune=%s %s", - prefix, suffix, sw); - if (!strncmp (ix86_arch_string, "generic", 7)) - error ("bad value (%s) for %sarch=%s %s", - ix86_arch_string, prefix, suffix, sw); - /* Validate -mabi= value. */ if (ix86_abi_string) { @@ -3008,6 +3692,12 @@ if (processor_alias_table[i].flags & PTA_ABM && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_ABM)) ix86_isa_flags |= OPTION_MASK_ISA_ABM; + if (processor_alias_table[i].flags & PTA_BMI + && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_BMI)) + ix86_isa_flags |= OPTION_MASK_ISA_BMI; + if (processor_alias_table[i].flags & PTA_TBM + && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_TBM)) + ix86_isa_flags |= OPTION_MASK_ISA_TBM; if (processor_alias_table[i].flags & PTA_CX16 && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_CX16)) ix86_isa_flags |= OPTION_MASK_ISA_CX16; @@ -3026,13 +3716,25 @@ if (processor_alias_table[i].flags & PTA_PCLMUL && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_PCLMUL)) ix86_isa_flags |= OPTION_MASK_ISA_PCLMUL; + if (processor_alias_table[i].flags & PTA_FSGSBASE + && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_FSGSBASE)) + ix86_isa_flags |= OPTION_MASK_ISA_FSGSBASE; + if (processor_alias_table[i].flags & PTA_RDRND + && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_RDRND)) + ix86_isa_flags |= OPTION_MASK_ISA_RDRND; + if (processor_alias_table[i].flags & PTA_F16C + && !(ix86_isa_flags_explicit & OPTION_MASK_ISA_F16C)) + ix86_isa_flags |= OPTION_MASK_ISA_F16C; if (processor_alias_table[i].flags & (PTA_PREFETCH_SSE | PTA_SSE)) x86_prefetch_sse = true; break; } - if (i == pta_size) + if (!strcmp (ix86_arch_string, "generic")) + error ("generic CPU can be used only for %stune=%s %s", + prefix, suffix, sw); + else if (!strncmp (ix86_arch_string, "generic", 7) || i == pta_size) error ("bad value (%s) for %sarch=%s %s", ix86_arch_string, prefix, suffix, sw); @@ -3045,23 +3747,48 @@ { ix86_schedule = processor_alias_table[i].schedule; ix86_tune = processor_alias_table[i].processor; - if (TARGET_64BIT && !(processor_alias_table[i].flags & PTA_64BIT)) + if (TARGET_64BIT) { - if (ix86_tune_defaulted) + if (!(processor_alias_table[i].flags & PTA_64BIT)) { - ix86_tune_string = "x86-64"; - for (i = 0; i < pta_size; i++) - if (! strcmp (ix86_tune_string, - processor_alias_table[i].name)) - break; - ix86_schedule = processor_alias_table[i].schedule; - ix86_tune = processor_alias_table[i].processor; + if (ix86_tune_defaulted) + { + ix86_tune_string = "x86-64"; + for (i = 0; i < pta_size; i++) + if (! strcmp (ix86_tune_string, + processor_alias_table[i].name)) + break; + ix86_schedule = processor_alias_table[i].schedule; + ix86_tune = processor_alias_table[i].processor; + } + else + error ("CPU you selected does not support x86-64 " + "instruction set"); } - else - error ("CPU you selected does not support x86-64 " - "instruction set"); } - /* Intel CPUs have always interpreted SSE prefetch instructions as + else + { + /* Adjust tuning when compiling for 32-bit ABI. */ + switch (ix86_tune) + { + case PROCESSOR_GENERIC64: + ix86_tune = PROCESSOR_GENERIC32; + ix86_schedule = CPU_PENTIUMPRO; + break; + + case PROCESSOR_CORE2_64: + ix86_tune = PROCESSOR_CORE2_32; + break; + + case PROCESSOR_COREI7_64: + ix86_tune = PROCESSOR_COREI7_32; + break; + + default: + break; + } + } + /* Intel CPUs have always interpreted SSE prefetch instructions as NOPs; so, we can enable SSE prefetch instructions even when -mtune (rather than -march) points us to a processor that has them. However, the VIA C3 gives a SIGILL, so we only do that for i686 and @@ -3071,7 +3798,8 @@ x86_prefetch_sse = true; break; } - if (i == pta_size) + + if (ix86_tune_specified && i == pta_size) error ("bad value (%s) for %stune=%s %s", ix86_tune_string, prefix, suffix, sw); @@ -3079,6 +3807,37 @@ for (i = 0; i < X86_TUNE_LAST; ++i) ix86_tune_features[i] = !!(initial_ix86_tune_features[i] & ix86_tune_mask); +#ifndef USE_IX86_FRAME_POINTER +#define USE_IX86_FRAME_POINTER 0 +#endif + +#ifndef USE_X86_64_FRAME_POINTER +#define USE_X86_64_FRAME_POINTER 0 +#endif + + /* Set the default values for switches whose default depends on TARGET_64BIT + in case they weren't overwritten by command line options. */ + if (TARGET_64BIT) + { + if (optimize > 1 && !global_options_set.x_flag_zee) + flag_zee = 1; + if (optimize >= 1 && !global_options_set.x_flag_omit_frame_pointer) + flag_omit_frame_pointer = !USE_X86_64_FRAME_POINTER; + if (flag_asynchronous_unwind_tables == 2) + flag_unwind_tables = flag_asynchronous_unwind_tables = 1; + if (flag_pcc_struct_return == 2) + flag_pcc_struct_return = 0; + } + else + { + if (optimize >= 1 && !global_options_set.x_flag_omit_frame_pointer) + flag_omit_frame_pointer = !(USE_IX86_FRAME_POINTER || optimize_size); + if (flag_asynchronous_unwind_tables == 2) + flag_asynchronous_unwind_tables = !USE_IX86_FRAME_POINTER; + if (flag_pcc_struct_return == 2) + flag_pcc_struct_return = DEFAULT_PCC_STRUCT_RETURN; + } + if (optimize_size) ix86_cost = &ix86_size_cost; else @@ -3191,8 +3950,6 @@ ix86_tls_dialect = TLS_DIALECT_GNU; else if (strcmp (ix86_tls_dialect_string, "gnu2") == 0) ix86_tls_dialect = TLS_DIALECT_GNU2; - else if (strcmp (ix86_tls_dialect_string, "sun") == 0) - ix86_tls_dialect = TLS_DIALECT_SUN; else error ("bad value (%s) for %stls-dialect=%s %s", ix86_tls_dialect_string, prefix, suffix, sw); @@ -3271,10 +4028,19 @@ ix86_preferred_stack_boundary = PREFERRED_STACK_BOUNDARY_DEFAULT; if (ix86_preferred_stack_boundary_string) { + int min = (TARGET_64BIT ? 4 : 2); + int max = (TARGET_SEH ? 4 : 12); + i = atoi (ix86_preferred_stack_boundary_string); - if (i < (TARGET_64BIT ? 4 : 2) || i > 12) - error ("%spreferred-stack-boundary=%d%s is not between %d and 12", - prefix, i, suffix, TARGET_64BIT ? 4 : 2); + if (i < min || i > max) + { + if (min == max) + error ("%spreferred-stack-boundary%s is not supported " + "for this target", prefix, suffix); + else + error ("%spreferred-stack-boundary=%d%s is not between %d and %d", + prefix, i, suffix, min, max); + } else ix86_preferred_stack_boundary = (1 << i) * BITS_PER_UNIT; } @@ -3363,7 +4129,8 @@ prefix, suffix, sw); } - if ((x86_accumulate_outgoing_args & ix86_tune_mask) + if ((!USE_IX86_FRAME_POINTER + || (x86_accumulate_outgoing_args & ix86_tune_mask)) && !(target_flags_explicit & MASK_ACCUMULATE_OUTGOING_ARGS) && !optimize_size) target_flags |= MASK_ACCUMULATE_OUTGOING_ARGS; @@ -3415,42 +4182,57 @@ if (!TARGET_SCHEDULE) flag_schedule_insns_after_reload = flag_schedule_insns = 0; - if (!PARAM_SET_P (PARAM_SIMULTANEOUS_PREFETCHES)) - set_param_value ("simultaneous-prefetches", - ix86_cost->simultaneous_prefetches); - if (!PARAM_SET_P (PARAM_L1_CACHE_LINE_SIZE)) - set_param_value ("l1-cache-line-size", ix86_cost->prefetch_block); - if (!PARAM_SET_P (PARAM_L1_CACHE_SIZE)) - set_param_value ("l1-cache-size", ix86_cost->l1_cache_size); - if (!PARAM_SET_P (PARAM_L2_CACHE_SIZE)) - set_param_value ("l2-cache-size", ix86_cost->l2_cache_size); + maybe_set_param_value (PARAM_SIMULTANEOUS_PREFETCHES, + ix86_cost->simultaneous_prefetches, + global_options.x_param_values, + global_options_set.x_param_values); + maybe_set_param_value (PARAM_L1_CACHE_LINE_SIZE, ix86_cost->prefetch_block, + global_options.x_param_values, + global_options_set.x_param_values); + maybe_set_param_value (PARAM_L1_CACHE_SIZE, ix86_cost->l1_cache_size, + global_options.x_param_values, + global_options_set.x_param_values); + maybe_set_param_value (PARAM_L2_CACHE_SIZE, ix86_cost->l2_cache_size, + global_options.x_param_values, + global_options_set.x_param_values); + + /* Enable sw prefetching at -O3 for CPUS that prefetching is helpful. */ + if (flag_prefetch_loop_arrays < 0 + && HAVE_prefetch + && optimize >= 3 + && software_prefetching_beneficial_p ()) + flag_prefetch_loop_arrays = 1; /* If using typedef char *va_list, signal that __builtin_va_start (&ap, 0) can be optimized to ap = __builtin_next_arg (0). */ - if (!TARGET_64BIT) + if (!TARGET_64BIT && !flag_split_stack) targetm.expand_builtin_va_start = NULL; if (TARGET_64BIT) { ix86_gen_leave = gen_leave_rex64; - ix86_gen_pop1 = gen_popdi1; ix86_gen_add3 = gen_adddi3; ix86_gen_sub3 = gen_subdi3; ix86_gen_sub3_carry = gen_subdi3_carry; ix86_gen_one_cmpl2 = gen_one_cmpldi2; ix86_gen_monitor = gen_sse3_monitor64; ix86_gen_andsp = gen_anddi3; + ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_di; + ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probedi; + ix86_gen_probe_stack_range = gen_probe_stack_rangedi; } else { ix86_gen_leave = gen_leave; - ix86_gen_pop1 = gen_popsi1; ix86_gen_add3 = gen_addsi3; ix86_gen_sub3 = gen_subsi3; ix86_gen_sub3_carry = gen_subsi3_carry; ix86_gen_one_cmpl2 = gen_one_cmplsi2; ix86_gen_monitor = gen_sse3_monitor; ix86_gen_andsp = gen_andsi3; + ix86_gen_allocate_stack_worker = gen_allocate_stack_worker_probe_si; + ix86_gen_adjust_stack_and_probe = gen_adjust_stack_and_probesi; + ix86_gen_probe_stack_range = gen_probe_stack_rangesi; } #ifdef USE_IX86_CLD @@ -3459,15 +4241,91 @@ target_flags |= MASK_CLD & ~target_flags_explicit; #endif + if (!TARGET_64BIT && flag_pic) + { + if (flag_fentry > 0) + sorry ("-mfentry isn%'t supported for 32-bit in combination " + "with -fpic"); + flag_fentry = 0; + } + else if (TARGET_SEH) + { + if (flag_fentry == 0) + sorry ("-mno-fentry isn%'t compatible with SEH"); + flag_fentry = 1; + } + else if (flag_fentry < 0) + { +#if defined(PROFILE_BEFORE_PROLOGUE) + flag_fentry = 1; +#else + flag_fentry = 0; +#endif + } + /* Save the initial options in case the user does function specific options */ if (main_args_p) target_option_default_node = target_option_current_node = build_target_option_node (); + + if (TARGET_AVX) + { + /* When not optimize for size, enable vzeroupper optimization for + TARGET_AVX with -fexpensive-optimizations. */ + if (!optimize_size + && flag_expensive_optimizations + && !(target_flags_explicit & MASK_VZEROUPPER)) + target_flags |= MASK_VZEROUPPER; + } + else + { + /* Disable vzeroupper pass if TARGET_AVX is disabled. */ + target_flags &= ~MASK_VZEROUPPER; + } +} + +/* Return TRUE if VAL is passed in register with 256bit AVX modes. */ + +static bool +function_pass_avx256_p (const_rtx val) +{ + if (!val) + return false; + + if (REG_P (val) && VALID_AVX256_REG_MODE (GET_MODE (val))) + return true; + + if (GET_CODE (val) == PARALLEL) + { + int i; + rtx r; + + for (i = XVECLEN (val, 0) - 1; i >= 0; i--) + { + r = XVECEXP (val, 0, i); + if (GET_CODE (r) == EXPR_LIST + && XEXP (r, 0) + && REG_P (XEXP (r, 0)) + && (GET_MODE (XEXP (r, 0)) == OImode + || VALID_AVX256_REG_MODE (GET_MODE (XEXP (r, 0))))) + return true; + } + } + + return false; +} + +/* Implement the TARGET_OPTION_OVERRIDE hook. */ + +static void +ix86_option_override (void) +{ + ix86_option_override_internal (true); } /* Update register usage after having seen the compiler flags. */ -void +static void ix86_conditional_register_usage (void) { int i; @@ -3551,7 +4409,7 @@ ptr->tune_defaulted = ix86_tune_defaulted; ptr->arch_specified = ix86_arch_specified; ptr->ix86_isa_flags_explicit = ix86_isa_flags_explicit; - ptr->target_flags_explicit = target_flags_explicit; + ptr->ix86_target_flags_explicit = target_flags_explicit; /* The fields are char but the variables are not; make sure the values fit in the fields. */ @@ -3580,7 +4438,7 @@ ix86_tune_defaulted = ptr->tune_defaulted; ix86_arch_specified = ptr->arch_specified; ix86_isa_flags_explicit = ptr->ix86_isa_flags_explicit; - target_flags_explicit = ptr->target_flags_explicit; + target_flags_explicit = ptr->ix86_target_flags_explicit; /* Recreate the arch feature tests if the arch changed */ if (old_arch != ix86_arch) @@ -3608,7 +4466,7 @@ struct cl_target_option *ptr) { char *target_string - = ix86_target_string (ptr->ix86_isa_flags, ptr->target_flags, + = ix86_target_string (ptr->x_ix86_isa_flags, ptr->x_target_flags, NULL, NULL, NULL, false); fprintf (file, "%*sarch = %d (%s)\n", @@ -3673,6 +4531,8 @@ /* isa options */ IX86_ATTR_ISA ("3dnow", OPT_m3dnow), IX86_ATTR_ISA ("abm", OPT_mabm), + IX86_ATTR_ISA ("bmi", OPT_mbmi), + IX86_ATTR_ISA ("tbm", OPT_mtbm), IX86_ATTR_ISA ("aes", OPT_maes), IX86_ATTR_ISA ("avx", OPT_mavx), IX86_ATTR_ISA ("mmx", OPT_mmmx), @@ -3689,6 +4549,9 @@ IX86_ATTR_ISA ("fma4", OPT_mfma4), IX86_ATTR_ISA ("xop", OPT_mxop), IX86_ATTR_ISA ("lwp", OPT_mlwp), + IX86_ATTR_ISA ("fsgsbase", OPT_mfsgsbase), + IX86_ATTR_ISA ("rdrnd", OPT_mrdrnd), + IX86_ATTR_ISA ("f16c", OPT_mf16c), /* string options */ IX86_ATTR_STR ("arch=", IX86_FUNCTION_SPECIFIC_ARCH), @@ -3858,11 +4721,12 @@ if (! ix86_valid_target_attribute_inner_p (args, option_strings)) return NULL_TREE; - /* If the changed options are different from the default, rerun override_options, - and then save the options away. The string options are are attribute options, - and will be undone when we copy the save structure. */ - if (ix86_isa_flags != def->ix86_isa_flags - || target_flags != def->target_flags + /* If the changed options are different from the default, rerun + ix86_option_override_internal, and then save the options away. + The string options are are attribute options, and will be undone + when we copy the save structure. */ + if (ix86_isa_flags != def->x_ix86_isa_flags + || target_flags != def->x_target_flags || option_strings[IX86_FUNCTION_SPECIFIC_ARCH] || option_strings[IX86_FUNCTION_SPECIFIC_TUNE] || option_strings[IX86_FUNCTION_SPECIFIC_FPMATH]) @@ -3886,7 +4750,7 @@ ix86_fpmath_string = "sse,387"; /* Do any overrides, such as arch=xxx, or tune=xxx support. */ - override_options (false); + ix86_option_override_internal (false); /* Add any builtin functions with the new isa if any. */ ix86_add_new_builtins (ix86_isa_flags); @@ -3925,11 +4789,12 @@ /* If the function changed the optimization levels as well as setting target options, start with the optimizations specified. */ if (func_optimize && func_optimize != old_optimize) - cl_optimization_restore (TREE_OPTIMIZATION (func_optimize)); + cl_optimization_restore (&global_options, + TREE_OPTIMIZATION (func_optimize)); /* The target attributes may also change some optimization flags, so update the optimization options if necessary. */ - cl_target_option_save (&cur_target); + cl_target_option_save (&cur_target, &global_options); new_target = ix86_valid_target_attribute_tree (args); new_optimize = build_optimization_node (); @@ -3944,10 +4809,11 @@ DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl) = new_optimize; } - cl_target_option_restore (&cur_target); + cl_target_option_restore (&global_options, &cur_target); if (old_optimize != new_optimize) - cl_optimization_restore (TREE_OPTIMIZATION (old_optimize)); + cl_optimization_restore (&global_options, + TREE_OPTIMIZATION (old_optimize)); return ret; } @@ -3979,12 +4845,12 @@ /* Callee's isa options should a subset of the caller's, i.e. a SSE4 function can inline a SSE2 function but a SSE2 function can't inline a SSE4 function. */ - if ((caller_opts->ix86_isa_flags & callee_opts->ix86_isa_flags) - != callee_opts->ix86_isa_flags) + if ((caller_opts->x_ix86_isa_flags & callee_opts->x_ix86_isa_flags) + != callee_opts->x_ix86_isa_flags) ret = false; /* See if we have the same non-isa options. */ - else if (caller_opts->target_flags != callee_opts->target_flags) + else if (caller_opts->x_target_flags != callee_opts->x_target_flags) ret = false; /* See if arch, tune, etc. are the same. */ @@ -4036,7 +4902,8 @@ else if (new_tree) { - cl_target_option_restore (TREE_TARGET_OPTION (new_tree)); + cl_target_option_restore (&global_options, + TREE_TARGET_OPTION (new_tree)); target_reinit (); } @@ -4045,7 +4912,7 @@ struct cl_target_option *def = TREE_TARGET_OPTION (target_option_current_node); - cl_target_option_restore (def); + cl_target_option_restore (&global_options, def); target_reinit (); } } @@ -4140,9 +5007,6 @@ /* We don't split these for medium model. Place them into default sections and hope for best. */ break; - case SECCAT_EMUTLS_VAR: - case SECCAT_EMUTLS_TMPL: - gcc_unreachable (); } if (sname) { @@ -4200,12 +5064,6 @@ /* We don't split these for medium model. Place them into default sections and hope for best. */ break; - case SECCAT_EMUTLS_VAR: - prefix = targetm.emutls.var_section; - break; - case SECCAT_EMUTLS_TMPL: - prefix = targetm.emutls.tmpl_section; - break; } if (prefix) { @@ -4214,13 +5072,13 @@ name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)); name = targetm.strip_name_encoding (name); - + /* If we're using one_only, then there needs to be a .gnu.linkonce prefix to the section name. */ linkonce = one_only ? ".gnu.linkonce" : ""; - + string = ACONCAT ((linkonce, prefix, ".", name, NULL)); - + DECL_SECTION_NAME (decl) = build_string (strlen (string), string); return; } @@ -4274,33 +5132,47 @@ ASM_OUTPUT_SKIP (file, size ? size : 1); } -void -optimization_options (int level, int size ATTRIBUTE_UNUSED) -{ - /* For -O2 and beyond, turn off -fschedule-insns by default. It tends to - make the problem with not enough registers even worse. */ +static const struct default_options ix86_option_optimization_table[] = + { + /* Turn off -fschedule-insns by default. It tends to make the + problem with not enough registers even worse. */ #ifdef INSN_SCHEDULING - if (level > 1) - flag_schedule_insns = 0; -#endif - + { OPT_LEVELS_ALL, OPT_fschedule_insns, NULL, 0 }, +#endif + +#ifdef SUBTARGET_OPTIMIZATION_OPTIONS + SUBTARGET_OPTIMIZATION_OPTIONS, +#endif + { OPT_LEVELS_NONE, 0, NULL, 0 } + }; + +/* Implement TARGET_OPTION_INIT_STRUCT. */ + +static void +ix86_option_init_struct (struct gcc_options *opts) +{ if (TARGET_MACHO) /* The Darwin libraries never set errno, so we might as well avoid calling them when that's the only reason we would. */ - flag_errno_math = 0; - - /* The default values of these switches depend on the TARGET_64BIT - that is not known at this moment. Mark these values with 2 and - let user the to override these. In case there is no command line option - specifying them, we will set the defaults in override_options. */ - if (optimize >= 1) - flag_omit_frame_pointer = 2; - flag_pcc_struct_return = 2; - flag_asynchronous_unwind_tables = 2; - flag_vect_cost_model = 1; -#ifdef SUBTARGET_OPTIMIZATION_OPTIONS - SUBTARGET_OPTIMIZATION_OPTIONS; -#endif + opts->x_flag_errno_math = 0; + + opts->x_flag_pcc_struct_return = 2; + opts->x_flag_asynchronous_unwind_tables = 2; + opts->x_flag_vect_cost_model = 1; +} + +/* Decide whether we must probe the stack before any space allocation + on this target. It's essentially TARGET_STACK_PROBE except when + -fstack-check causes the stack to be already probed differently. */ + +bool +ix86_target_stack_probe (void) +{ + /* Do not probe the stack twice if static stack checking is enabled. */ + if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK) + return false; + + return TARGET_STACK_PROBE; } /* Decide whether we can make a sibling call to a function. DECL is the @@ -4315,8 +5187,11 @@ /* If we are generating position-independent code, we cannot sibcall optimize any indirect call, or a direct call to a global function, - as the PLT requires %ebx be live. */ - if (!TARGET_64BIT && flag_pic && (!decl || !targetm.binds_local_p (decl))) + as the PLT requires %ebx be live. (Darwin does not have a PLT.) */ + if (!TARGET_MACHO + && !TARGET_64BIT + && flag_pic + && (!decl || !targetm.binds_local_p (decl))) return false; /* If we need to align the outgoing stack, then sibcalling would @@ -4357,7 +5232,14 @@ return false; } else if (VOID_TYPE_P (TREE_TYPE (DECL_RESULT (cfun->decl)))) - ; + { + /* Disable sibcall if we need to generate vzeroupper after + callee returns. */ + if (TARGET_VZEROUPPER + && cfun->machine->callee_return_avx256_p + && !cfun->machine->caller_return_avx256_p) + return false; + } else if (!rtx_equal_p (a, b)) return false; @@ -4391,8 +5273,8 @@ return true; } -/* Handle "cdecl", "stdcall", "fastcall", "regparm" and "sseregparm" - calling convention attributes; +/* Handle "cdecl", "stdcall", "fastcall", "regparm", "thiscall", + and "sseregparm" calling convention attributes; arguments as in struct attribute_spec.handler. */ static tree @@ -4422,6 +5304,11 @@ error ("fastcall and regparm attributes are not compatible"); } + if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node))) + { + error ("regparam and thiscall attributes are not compatible"); + } + cst = TREE_VALUE (args); if (TREE_CODE (cst) != INTEGER_CST) { @@ -4443,7 +5330,8 @@ if (TARGET_64BIT) { /* Do not warn when emulating the MS ABI. */ - if (TREE_CODE (*node) != FUNCTION_TYPE + if ((TREE_CODE (*node) != FUNCTION_TYPE + && TREE_CODE (*node) != METHOD_TYPE) || ix86_function_type_abi (*node) != MS_ABI) warning (OPT_Wattributes, "%qE attribute ignored", name); @@ -4466,6 +5354,10 @@ { error ("fastcall and regparm attributes are not compatible"); } + if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node))) + { + error ("fastcall and thiscall attributes are not compatible"); + } } /* Can combine stdcall with fastcall (redundant), regparm and @@ -4480,6 +5372,10 @@ { error ("stdcall and fastcall attributes are not compatible"); } + if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node))) + { + error ("stdcall and thiscall attributes are not compatible"); + } } /* Can combine cdecl with regparm and sseregparm. */ @@ -4493,6 +5389,28 @@ { error ("fastcall and cdecl attributes are not compatible"); } + if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (*node))) + { + error ("cdecl and thiscall attributes are not compatible"); + } + } + else if (is_attribute_p ("thiscall", name)) + { + if (TREE_CODE (*node) != METHOD_TYPE && pedantic) + warning (OPT_Wattributes, "%qE attribute is used for none class-method", + name); + if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (*node))) + { + error ("stdcall and thiscall attributes are not compatible"); + } + if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (*node))) + { + error ("fastcall and thiscall attributes are not compatible"); + } + if (lookup_attribute ("cdecl", TYPE_ATTRIBUTES (*node))) + { + error ("cdecl and thiscall attributes are not compatible"); + } } /* Can combine sseregparm with all attributes. */ @@ -4526,6 +5444,11 @@ != !lookup_attribute ("sseregparm", TYPE_ATTRIBUTES (type2))) return 0; + /* Check for mismatched thiscall types. */ + if (!lookup_attribute ("thiscall", TYPE_ATTRIBUTES (type1)) + != !lookup_attribute ("thiscall", TYPE_ATTRIBUTES (type2))) + return 0; + /* Check for mismatched return types (cdecl vs stdcall). */ if (!lookup_attribute (rtdstr, TYPE_ATTRIBUTES (type1)) != !lookup_attribute (rtdstr, TYPE_ATTRIBUTES (type2))) @@ -4559,15 +5482,18 @@ if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type))) return 2; + if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (type))) + return 1; + /* Use register calling convention for local functions when possible. */ if (decl && TREE_CODE (decl) == FUNCTION_DECL && optimize - && !profile_flag) + && !(profile_flag && !flag_fentry)) { /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */ struct cgraph_local_info *i = cgraph_local_info (CONST_CAST_TREE (decl)); - if (i && i->local) + if (i && i->local && i->can_change_signature) { int local_regparm, globals = 0, regno; @@ -4582,6 +5508,10 @@ if (local_regparm == 3 && DECL_STATIC_CHAIN (decl)) local_regparm = 2; + /* In 32-bit mode save a register for the split stack. */ + if (!TARGET_64BIT && local_regparm == 3 && flag_split_stack) + local_regparm = 2; + /* Each fixed register usage increases register pressure, so less registers should be used for argument passing. This functionality can be overriden by an explicit @@ -4621,10 +5551,10 @@ if (warn) { if (decl) - error ("Calling %qD with attribute sseregparm without " + error ("calling %qD with attribute sseregparm without " "SSE/SSE2 enabled", decl); else - error ("Calling %qT with attribute sseregparm without " + error ("calling %qT with attribute sseregparm without " "SSE/SSE2 enabled", type); } return 0; @@ -4635,11 +5565,12 @@ /* For local functions, pass up to SSE_REGPARM_MAX SFmode (and DFmode for SSE2) arguments in SSE registers. */ - if (decl && TARGET_SSE_MATH && optimize && !profile_flag) + if (decl && TARGET_SSE_MATH && optimize + && !(profile_flag && !flag_fentry)) { /* FIXME: remove this CONST_CAST when cgraph.[ch] is constified. */ struct cgraph_local_info *i = cgraph_local_info (CONST_CAST_TREE(decl)); - if (i && i->local) + if (i && i->local && i->can_change_signature) return TARGET_SSE2 ? 2 : 1; } @@ -4662,6 +5593,19 @@ return REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR), 0); } +static bool +ix86_keep_aggregate_return_pointer (tree fntype) +{ + tree attr; + + attr = lookup_attribute ("callee_pop_aggregate_return", + TYPE_ATTRIBUTES (fntype)); + if (attr) + return (TREE_INT_CST_LOW (TREE_VALUE (TREE_VALUE (attr))) == 0); + + return KEEP_AGGREGATE_RETURN_POINTER != 0; +} + /* Value is the number of bytes of arguments automatically popped when returning from a subroutine call. FUNDECL is the declaration node of the function (as a tree), @@ -4679,7 +5623,7 @@ The attribute stdcall is equivalent to RTD on a per module basis. */ -int +static int ix86_return_pops_args (tree fundecl, tree funtype, int size) { int rtd; @@ -4696,7 +5640,8 @@ /* Stdcall and fastcall functions will pop the stack if not variable args. */ if (lookup_attribute ("stdcall", TYPE_ATTRIBUTES (funtype)) - || lookup_attribute ("fastcall", TYPE_ATTRIBUTES (funtype))) + || lookup_attribute ("fastcall", TYPE_ATTRIBUTES (funtype)) + || lookup_attribute ("thiscall", TYPE_ATTRIBUTES (funtype))) rtd = 1; if (rtd && ! stdarg_p (funtype)) @@ -4705,7 +5650,7 @@ /* Lose any fake structure return argument if it is passed on the stack. */ if (aggregate_value_p (TREE_TYPE (funtype), fundecl) - && !KEEP_AGGREGATE_RETURN_POINTER) + && !ix86_keep_aggregate_return_pointer (funtype)) { int nregs = ix86_function_regparm (funtype, fundecl); if (nregs == 0) @@ -4820,20 +5765,15 @@ } static bool -ix86_function_ms_hook_prologue (const_tree fntype) -{ - if (!TARGET_64BIT) - { - if (lookup_attribute ("ms_hook_prologue", DECL_ATTRIBUTES (fntype))) - { - if (decl_function_context (fntype) != NULL_TREE) - { - error_at (DECL_SOURCE_LOCATION (fntype), - "ms_hook_prologue is not compatible with nested function"); - } - - return true; - } +ix86_function_ms_hook_prologue (const_tree fn) +{ + if (fn && lookup_attribute ("ms_hook_prologue", DECL_ATTRIBUTES (fn))) + { + if (decl_function_context (fn) != NULL_TREE) + error_at (DECL_SOURCE_LOCATION (fn), + "ms_hook_prologue is not compatible with nested function"); + else + return true; } return false; } @@ -4856,12 +5796,55 @@ return cfun->machine->call_abi; } +/* Write the extra assembler code needed to declare a function properly. */ + +void +ix86_asm_output_function_label (FILE *asm_out_file, const char *fname, + tree decl) +{ + bool is_ms_hook = ix86_function_ms_hook_prologue (decl); + + if (is_ms_hook) + { + int i, filler_count = (TARGET_64BIT ? 32 : 16); + unsigned int filler_cc = 0xcccccccc; + + for (i = 0; i < filler_count; i += 4) + fprintf (asm_out_file, ASM_LONG " %#x\n", filler_cc); + } + +#ifdef SUBTARGET_ASM_UNWIND_INIT + SUBTARGET_ASM_UNWIND_INIT (asm_out_file); +#endif + + ASM_OUTPUT_LABEL (asm_out_file, fname); + + /* Output magic byte marker, if hot-patch attribute is set. */ + if (is_ms_hook) + { + if (TARGET_64BIT) + { + /* leaq [%rsp + 0], %rsp */ + asm_fprintf (asm_out_file, ASM_BYTE + "0x48, 0x8d, 0xa4, 0x24, 0x00, 0x00, 0x00, 0x00\n"); + } + else + { + /* movl.s %edi, %edi + push %ebp + movl.s %esp, %ebp */ + asm_fprintf (asm_out_file, ASM_BYTE + "0x8b, 0xff, 0x55, 0x8b, 0xec\n"); + } + } +} + /* regclass.c */ extern void init_regs (void); /* Implementation of call abi switching target hook. Specific to FNDECL - the specific call register sets are set. See also CONDITIONAL_REGISTER_USAGE - for more details. */ + the specific call register sets are set. See also + ix86_conditional_register_usage for more details. */ void ix86_call_abi_override (const_tree fndecl) { @@ -4890,15 +5873,53 @@ init_cumulative_args (CUMULATIVE_ARGS *cum, /* Argument info to initialize */ tree fntype, /* tree ptr for function decl */ rtx libname, /* SYMBOL_REF of library name or 0 */ - tree fndecl) -{ - struct cgraph_local_info *i = fndecl ? cgraph_local_info (fndecl) : NULL; + tree fndecl, + int caller) +{ + struct cgraph_local_info *i; + tree fnret_type; + memset (cum, 0, sizeof (*cum)); + /* Initialize for the current callee. */ + if (caller) + { + cfun->machine->callee_pass_avx256_p = false; + cfun->machine->callee_return_avx256_p = false; + } + if (fndecl) - cum->call_abi = ix86_function_abi (fndecl); - else - cum->call_abi = ix86_function_type_abi (fntype); + { + i = cgraph_local_info (fndecl); + cum->call_abi = ix86_function_abi (fndecl); + fnret_type = TREE_TYPE (TREE_TYPE (fndecl)); + } + else + { + i = NULL; + cum->call_abi = ix86_function_type_abi (fntype); + if (fntype) + fnret_type = TREE_TYPE (fntype); + else + fnret_type = NULL; + } + + if (TARGET_VZEROUPPER && fnret_type) + { + rtx fnret_value = ix86_function_value (fnret_type, fntype, + false); + if (function_pass_avx256_p (fnret_value)) + { + /* The return value of this function uses 256bit AVX modes. */ + if (caller) + cfun->machine->callee_return_avx256_p = true; + else + cfun->machine->caller_return_avx256_p = true; + } + } + + cum->caller = caller; + /* Set up the number of registers to use for passing arguments. */ if (cum->call_abi == MS_ABI && !ACCUMULATE_OUTGOING_ARGS) @@ -4907,19 +5928,18 @@ cum->nregs = ix86_regparm; if (TARGET_64BIT) { - if (cum->call_abi != ix86_abi) - cum->nregs = (ix86_abi != SYSV_ABI - ? X86_64_REGPARM_MAX : X86_64_MS_REGPARM_MAX); + cum->nregs = (cum->call_abi == SYSV_ABI + ? X86_64_REGPARM_MAX + : X86_64_MS_REGPARM_MAX); } if (TARGET_SSE) { cum->sse_nregs = SSE_REGPARM_MAX; if (TARGET_64BIT) { - if (cum->call_abi != ix86_abi) - cum->sse_nregs = (ix86_abi != SYSV_ABI - ? X86_64_SSE_REGPARM_MAX - : X86_64_MS_SSE_REGPARM_MAX); + cum->sse_nregs = (cum->call_abi == SYSV_ABI + ? X86_64_SSE_REGPARM_MAX + : X86_64_MS_SSE_REGPARM_MAX); } } if (TARGET_MMX) @@ -4934,7 +5954,7 @@ va_start so for local functions maybe_vaarg can be made aggressive helping K&R code. FIXME: once typesytem is fixed, we won't need this code anymore. */ - if (i && i->local) + if (i && i->local && i->can_change_signature) fntype = TREE_TYPE (fndecl); cum->maybe_vaarg = (fntype ? (!prototype_p (fntype) || stdarg_p (fntype)) @@ -4959,7 +5979,12 @@ else look for regparm information. */ if (fntype) { - if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype))) + if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (fntype))) + { + cum->nregs = 1; + cum->fastcall = 1; /* Same first register as in fastcall. */ + } + else if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype))) { cum->nregs = 2; cum->fastcall = 1; @@ -4989,7 +6014,7 @@ NULL. */ static enum machine_mode -type_natural_mode (const_tree type, CUMULATIVE_ARGS *cum) +type_natural_mode (const_tree type, const CUMULATIVE_ARGS *cum) { enum machine_mode mode = TYPE_MODE (type); @@ -5017,7 +6042,7 @@ static bool warnedavx; if (cum - && !warnedavx + && !warnedavx && cum->warn_avx) { warnedavx = true; @@ -5161,7 +6186,7 @@ { case RECORD_TYPE: /* And now merge the fields of structure. */ - for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field)) + for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) { if (TREE_CODE (field) == FIELD_DECL) { @@ -5198,12 +6223,12 @@ == NULL_TREE)) { static bool warned; - + if (!warned && warn_psabi) { warned = true; inform (input_location, - "The ABI of passing struct with" + "the ABI of passing struct with" " a flexible array member has" " changed in GCC 4.4"); } @@ -5249,7 +6274,7 @@ case QUAL_UNION_TYPE: /* Unions are similar to RECORD_TYPE but offset is always 0. */ - for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field)) + for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) { if (TREE_CODE (field) == FIELD_DECL) { @@ -5319,7 +6344,7 @@ { warned = true; inform (input_location, - "The ABI of passing union with long double" + "the ABI of passing union with long double" " has changed in GCC 4.4"); } return 0; @@ -5434,7 +6459,7 @@ { warned = true; inform (input_location, - "The ABI of passing structure with complex float" + "the ABI of passing structure with complex float" " member has changed in GCC 4.4"); } classes[1] = X86_64_SSESF_CLASS; @@ -5620,7 +6645,7 @@ case X86_64_SSESF_CLASS: case X86_64_SSEDF_CLASS: if (mode != BLKmode) - return gen_reg_or_parallel (mode, orig_mode, + return gen_reg_or_parallel (mode, orig_mode, SSE_REGNO (sse_regno)); break; case X86_64_X87_CLASS: @@ -5746,7 +6771,8 @@ static void function_arg_advance_32 (CUMULATIVE_ARGS *cum, enum machine_mode mode, - tree type, HOST_WIDE_INT bytes, HOST_WIDE_INT words) + const_tree type, HOST_WIDE_INT bytes, + HOST_WIDE_INT words) { switch (mode) { @@ -5834,7 +6860,7 @@ static void function_arg_advance_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode, - tree type, HOST_WIDE_INT words, int named) + const_tree type, HOST_WIDE_INT words, bool named) { int int_nregs, sse_nregs; @@ -5842,9 +6868,8 @@ if (!named && VALID_AVX256_REG_MODE (mode)) return; - if (!examine_argument (mode, type, 0, &int_nregs, &sse_nregs)) - cum->words += words; - else if (sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs) + if (examine_argument (mode, type, 0, &int_nregs, &sse_nregs) + && sse_nregs <= cum->sse_nregs && int_nregs <= cum->nregs) { cum->nregs -= int_nregs; cum->sse_nregs -= sse_nregs; @@ -5852,7 +6877,11 @@ cum->sse_regno += sse_nregs; } else - cum->words += words; + { + int align = ix86_function_arg_boundary (mode, type) / BITS_PER_WORD; + cum->words = (cum->words + align - 1) & ~(align - 1); + cum->words += words; + } } static void @@ -5870,9 +6899,13 @@ } } -void -function_arg_advance (CUMULATIVE_ARGS *cum, enum machine_mode mode, - tree type, int named) +/* Update the data in CUM to advance over an argument of mode MODE and + data type TYPE. (TYPE is null for libcalls where that information + may not be available.) */ + +static void +ix86_function_arg_advance (CUMULATIVE_ARGS *cum, enum machine_mode mode, + const_tree type, bool named) { HOST_WIDE_INT bytes, words; @@ -5907,8 +6940,8 @@ (otherwise it is an extra parameter matching an ellipsis). */ static rtx -function_arg_32 (CUMULATIVE_ARGS *cum, enum machine_mode mode, - enum machine_mode orig_mode, tree type, +function_arg_32 (const CUMULATIVE_ARGS *cum, enum machine_mode mode, + enum machine_mode orig_mode, const_tree type, HOST_WIDE_INT bytes, HOST_WIDE_INT words) { static bool warnedsse, warnedmmx; @@ -6024,19 +7057,15 @@ } static rtx -function_arg_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode, - enum machine_mode orig_mode, tree type, int named) +function_arg_64 (const CUMULATIVE_ARGS *cum, enum machine_mode mode, + enum machine_mode orig_mode, const_tree type, bool named) { /* Handle a hidden AL argument containing number of registers for varargs x86-64 functions. */ if (mode == VOIDmode) return GEN_INT (cum->maybe_vaarg ? (cum->sse_nregs < 0 - ? (cum->call_abi == ix86_abi - ? SSE_REGPARM_MAX - : (ix86_abi != SYSV_ABI - ? X86_64_SSE_REGPARM_MAX - : X86_64_MS_SSE_REGPARM_MAX)) + ? X86_64_SSE_REGPARM_MAX : cum->sse_regno) : -1); @@ -6064,8 +7093,8 @@ } static rtx -function_arg_ms_64 (CUMULATIVE_ARGS *cum, enum machine_mode mode, - enum machine_mode orig_mode, int named, +function_arg_ms_64 (const CUMULATIVE_ARGS *cum, enum machine_mode mode, + enum machine_mode orig_mode, bool named, HOST_WIDE_INT bytes) { unsigned int regno; @@ -6111,12 +7140,23 @@ return gen_reg_or_parallel (mode, orig_mode, regno); } -rtx -function_arg (CUMULATIVE_ARGS *cum, enum machine_mode omode, - tree type, int named) +/* Return where to put the arguments to a function. + Return zero to push the argument on the stack, or a hard register in which to store the argument. + + MODE is the argument's machine mode. TYPE is the data type of the + argument. It is null for libcalls where that information may not be + available. CUM gives information about the preceding args and about + the function being called. NAMED is nonzero if this argument is a + named parameter (otherwise it is an extra parameter matching an + ellipsis). */ + +static rtx +ix86_function_arg (CUMULATIVE_ARGS *cum, enum machine_mode omode, + const_tree type, bool named) { enum machine_mode mode = omode; HOST_WIDE_INT bytes, words; + rtx arg; if (mode == BLKmode) bytes = int_size_in_bytes (type); @@ -6130,11 +7170,22 @@ mode = type_natural_mode (type, cum); if (TARGET_64BIT && (cum ? cum->call_abi : ix86_abi) == MS_ABI) - return function_arg_ms_64 (cum, mode, omode, named, bytes); + arg = function_arg_ms_64 (cum, mode, omode, named, bytes); else if (TARGET_64BIT) - return function_arg_64 (cum, mode, omode, type, named); - else - return function_arg_32 (cum, mode, omode, type, bytes, words); + arg = function_arg_64 (cum, mode, omode, type, named); + else + arg = function_arg_32 (cum, mode, omode, type, bytes, words); + + if (TARGET_VZEROUPPER && function_pass_avx256_p (arg)) + { + /* This argument uses 256bit AVX modes. */ + if (cum->caller) + cfun->machine->callee_pass_avx256_p = true; + else + cfun->machine->caller_pass_avx256_p = true; + } + + return arg; } /* A C expression that indicates when an argument must be passed by @@ -6180,10 +7231,12 @@ return 0; } -/* Return true when TYPE should be 128bit aligned for 32bit argument passing - ABI. */ -static bool -contains_aligned_value_p (tree type) +/* Return true when TYPE should be 128bit aligned for 32bit argument + passing ABI. XXX: This function is obsolete and is only used for + checking psABI compatibility with previous versions of GCC. */ + +static bool +ix86_compat_aligned_value_p (const_tree type) { enum machine_mode mode = TYPE_MODE (type); if (((TARGET_SSE && SSE_REG_MODE_P (mode)) @@ -6207,10 +7260,10 @@ tree field; /* Walk all the structure fields. */ - for (field = TYPE_FIELDS (type); field; field = TREE_CHAIN (field)) + for (field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field)) { if (TREE_CODE (field) == FIELD_DECL - && contains_aligned_value_p (TREE_TYPE (field))) + && ix86_compat_aligned_value_p (TREE_TYPE (field))) return true; } break; @@ -6218,7 +7271,7 @@ case ARRAY_TYPE: /* Just for use if some languages passes arrays by value. */ - if (contains_aligned_value_p (TREE_TYPE (type))) + if (ix86_compat_aligned_value_p (TREE_TYPE (type))) return true; break; @@ -6229,25 +7282,14 @@ return false; } -/* Gives the alignment boundary, in bits, of an argument with the - specified mode and type. */ - -int -ix86_function_arg_boundary (enum machine_mode mode, tree type) -{ - int align; - if (type) - { - /* Since canonical type is used for call, we convert it to - canonical type if needed. */ - if (!TYPE_STRUCTURAL_EQUALITY_P (type)) - type = TYPE_CANONICAL (type); - align = TYPE_ALIGN (type); - } - else - align = GET_MODE_ALIGNMENT (mode); - if (align < PARM_BOUNDARY) - align = PARM_BOUNDARY; +/* Return the alignment boundary for MODE and TYPE with alignment ALIGN. + XXX: This function is obsolete and is only used for checking psABI + compatibility with previous versions of GCC. */ + +static unsigned int +ix86_compat_function_arg_boundary (enum machine_mode mode, + const_tree type, unsigned int align) +{ /* In 32bit, only _Decimal128 and __float128 are aligned to their natural boundaries. */ if (!TARGET_64BIT && mode != TDmode && mode != TFmode) @@ -6266,7 +7308,7 @@ } else { - if (!contains_aligned_value_p (type)) + if (!ix86_compat_aligned_value_p (type)) align = PARM_BOUNDARY; } } @@ -6275,10 +7317,117 @@ return align; } +/* Return true when TYPE should be 128bit aligned for 32bit argument + passing ABI. */ + +static bool +ix86_contains_aligned_value_p (const_tree type) +{ + enum machine_mode mode = TYPE_MODE (type); + + if (mode == XFmode || mode == XCmode) + return false; + + if (TYPE_ALIGN (type) < 128) + return false; + + if (AGGREGATE_TYPE_P (type)) + { + /* Walk the aggregates recursively. */ + switch (TREE_CODE (type)) + { + case RECORD_TYPE: + case UNION_TYPE: + case QUAL_UNION_TYPE: + { + tree field; + + /* Walk all the structure fields. */ + for (field = TYPE_FIELDS (type); + field; + field = DECL_CHAIN (field)) + { + if (TREE_CODE (field) == FIELD_DECL + && ix86_contains_aligned_value_p (TREE_TYPE (field))) + return true; + } + break; + } + + case ARRAY_TYPE: + /* Just for use if some languages passes arrays by value. */ + if (ix86_contains_aligned_value_p (TREE_TYPE (type))) + return true; + break; + + default: + gcc_unreachable (); + } + } + else + return TYPE_ALIGN (type) >= 128; + + return false; +} + +/* Gives the alignment boundary, in bits, of an argument with the + specified mode and type. */ + +static unsigned int +ix86_function_arg_boundary (enum machine_mode mode, const_tree type) +{ + unsigned int align; + if (type) + { + /* Since the main variant type is used for call, we convert it to + the main variant type. */ + type = TYPE_MAIN_VARIANT (type); + align = TYPE_ALIGN (type); + } + else + align = GET_MODE_ALIGNMENT (mode); + if (align < PARM_BOUNDARY) + align = PARM_BOUNDARY; + else + { + static bool warned; + unsigned int saved_align = align; + + if (!TARGET_64BIT) + { + /* i386 ABI defines XFmode arguments to be 4 byte aligned. */ + if (!type) + { + if (mode == XFmode || mode == XCmode) + align = PARM_BOUNDARY; + } + else if (!ix86_contains_aligned_value_p (type)) + align = PARM_BOUNDARY; + + if (align < 128) + align = PARM_BOUNDARY; + } + + if (warn_psabi + && !warned + && align != ix86_compat_function_arg_boundary (mode, type, + saved_align)) + { + warned = true; + inform (input_location, + "The ABI for passing parameters with %d-byte" + " alignment has changed in GCC 4.6", + align / BITS_PER_UNIT); + } + } + + return align; +} + /* Return true if N is a possible register number of function value. */ -bool -ix86_function_value_regno_p (int regno) +static bool +ix86_function_value_regno_p (const unsigned int regno) { switch (regno) { @@ -6462,59 +7611,59 @@ /* Return true iff type is returned in memory. */ -static int ATTRIBUTE_UNUSED +static bool ATTRIBUTE_UNUSED return_in_memory_32 (const_tree type, enum machine_mode mode) { HOST_WIDE_INT size; if (mode == BLKmode) - return 1; + return true; size = int_size_in_bytes (type); if (MS_AGGREGATE_RETURN && AGGREGATE_TYPE_P (type) && size <= 8) - return 0; + return false; if (VECTOR_MODE_P (mode) || mode == TImode) { /* User-created vectors small enough to fit in EAX. */ if (size < 8) - return 0; + return false; /* MMX/3dNow values are returned in MM0, - except when it doesn't exits. */ + except when it doesn't exits or the ABI prescribes otherwise. */ if (size == 8) - return (TARGET_MMX ? 0 : 1); + return !TARGET_MMX || TARGET_VECT8_RETURNS; /* SSE values are returned in XMM0, except when it doesn't exist. */ if (size == 16) - return (TARGET_SSE ? 0 : 1); + return !TARGET_SSE; /* AVX values are returned in YMM0, except when it doesn't exist. */ if (size == 32) - return TARGET_AVX ? 0 : 1; + return !TARGET_AVX; } if (mode == XFmode) - return 0; + return false; if (size > 12) - return 1; + return true; /* OImode shouldn't be used directly. */ gcc_assert (mode != OImode); - return 0; -} - -static int ATTRIBUTE_UNUSED + return false; +} + +static bool ATTRIBUTE_UNUSED return_in_memory_64 (const_tree type, enum machine_mode mode) { int needed_intregs, needed_sseregs; return !examine_argument (mode, type, 1, &needed_intregs, &needed_sseregs); } -static int ATTRIBUTE_UNUSED +static bool ATTRIBUTE_UNUSED return_in_memory_ms_64 (const_tree type, enum machine_mode mode) { HOST_WIDE_INT size = int_size_in_bytes (type); @@ -6522,10 +7671,10 @@ /* __m128 is returned in xmm0. */ if ((SCALAR_INT_MODE_P (mode) || VECTOR_MODE_P (mode)) && !COMPLEX_MODE_P (mode) && (GET_MODE_SIZE (mode) == 16 || size == 16)) - return 0; + return false; /* Otherwise, the size must be exactly in [1248]. */ - return (size != 1 && size != 2 && size != 4 && size != 8); + return size != 1 && size != 2 && size != 4 && size != 8; } static bool @@ -6535,7 +7684,7 @@ return SUBTARGET_RETURN_IN_MEMORY (type, fntype); #else const enum machine_mode mode = type_natural_mode (type, NULL); - + if (TARGET_64BIT) { if (ix86_function_type_abi (fntype) == MS_ABI) @@ -6548,43 +7697,6 @@ #endif } -/* Return false iff TYPE is returned in memory. This version is used - on Solaris 10. It is similar to the generic ix86_return_in_memory, - but differs notably in that when MMX is available, 8-byte vectors - are returned in memory, rather than in MMX registers. */ - -bool -ix86_sol10_return_in_memory (const_tree type, const_tree fntype ATTRIBUTE_UNUSED) -{ - int size; - enum machine_mode mode = type_natural_mode (type, NULL); - - if (TARGET_64BIT) - return return_in_memory_64 (type, mode); - - if (mode == BLKmode) - return 1; - - size = int_size_in_bytes (type); - - if (VECTOR_MODE_P (mode)) - { - /* Return in memory only if MMX registers *are* available. This - seems backwards, but it is consistent with the existing - Solaris x86 ABI. */ - if (size == 8) - return TARGET_MMX; - if (size == 16) - return !TARGET_SSE; - } - else if (mode == TImode) - return !TARGET_SSE; - else if (mode == XFmode) - return 0; - - return size > 12; -} - /* When returning SSE vector types, we have a choice of either (1) being abi incompatible with a -march switch, or (2) generating an error. @@ -6646,7 +7758,7 @@ if (!TARGET_64BIT || abi == MS_ABI) return build_pointer_type (char_type_node); - record = (*lang_hooks.types.make_type) (RECORD_TYPE); + record = lang_hooks.types.make_type (RECORD_TYPE); type_decl = build_decl (BUILTINS_LOCATION, TYPE_DECL, get_identifier ("__va_list_tag"), record); @@ -6671,12 +7783,12 @@ DECL_FIELD_CONTEXT (f_ovf) = record; DECL_FIELD_CONTEXT (f_sav) = record; - TREE_CHAIN (record) = type_decl; + TYPE_STUB_DECL (record) = type_decl; TYPE_NAME (record) = type_decl; TYPE_FIELDS (record) = f_gpr; - TREE_CHAIN (f_gpr) = f_fpr; - TREE_CHAIN (f_fpr) = f_ovf; - TREE_CHAIN (f_ovf) = f_sav; + DECL_CHAIN (f_gpr) = f_fpr; + DECL_CHAIN (f_fpr) = f_ovf; + DECL_CHAIN (f_ovf) = f_sav; layout_type (record); @@ -6735,17 +7847,8 @@ setup_incoming_varargs_64 (CUMULATIVE_ARGS *cum) { rtx save_area, mem; - rtx label; - rtx label_ref; - rtx tmp_reg; - rtx nsse_reg; alias_set_type set; - int i; - int regparm = ix86_regparm; - - if (cum->call_abi != ix86_abi) - regparm = (ix86_abi != SYSV_ABI - ? X86_64_REGPARM_MAX : X86_64_MS_REGPARM_MAX); + int i, max; /* GPR size of varargs save area. */ if (cfun->va_list_gpr_size) @@ -6755,7 +7858,7 @@ /* FPR size of varargs save area. We don't need it if we don't pass anything in SSE registers. */ - if (cum->sse_nregs && cfun->va_list_fpr_size) + if (TARGET_SSE && cfun->va_list_fpr_size) ix86_varargs_fpr_size = X86_64_SSE_REGPARM_MAX * 16; else ix86_varargs_fpr_size = 0; @@ -6766,10 +7869,11 @@ save_area = frame_pointer_rtx; set = get_varargs_alias_set (); - for (i = cum->regno; - i < regparm - && i < cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD; - i++) + max = cum->regno + cfun->va_list_gpr_size / UNITS_PER_WORD; + if (max > X86_64_REGPARM_MAX) + max = X86_64_REGPARM_MAX; + + for (i = cum->regno; i < max; i++) { mem = gen_rtx_MEM (Pmode, plus_constant (save_area, i * UNITS_PER_WORD)); @@ -6781,58 +7885,42 @@ if (ix86_varargs_fpr_size) { + enum machine_mode smode; + rtx label, test; + /* Now emit code to save SSE registers. The AX parameter contains number - of SSE parameter registers used to call this function. We use - sse_prologue_save insn template that produces computed jump across - SSE saves. We need some preparation work to get this working. */ + of SSE parameter registers used to call this function, though all we + actually check here is the zero/non-zero status. */ label = gen_label_rtx (); - label_ref = gen_rtx_LABEL_REF (Pmode, label); - - /* Compute address to jump to : - label - eax*4 + nnamed_sse_arguments*4 Or - label - eax*5 + nnamed_sse_arguments*5 for AVX. */ - tmp_reg = gen_reg_rtx (Pmode); - nsse_reg = gen_reg_rtx (Pmode); - emit_insn (gen_zero_extendqidi2 (nsse_reg, gen_rtx_REG (QImode, AX_REG))); - emit_insn (gen_rtx_SET (VOIDmode, tmp_reg, - gen_rtx_MULT (Pmode, nsse_reg, - GEN_INT (4)))); - - /* vmovaps is one byte longer than movaps. */ - if (TARGET_AVX) - emit_insn (gen_rtx_SET (VOIDmode, tmp_reg, - gen_rtx_PLUS (Pmode, tmp_reg, - nsse_reg))); - - if (cum->sse_regno) - emit_move_insn - (nsse_reg, - gen_rtx_CONST (DImode, - gen_rtx_PLUS (DImode, - label_ref, - GEN_INT (cum->sse_regno - * (TARGET_AVX ? 5 : 4))))); - else - emit_move_insn (nsse_reg, label_ref); - emit_insn (gen_subdi3 (nsse_reg, nsse_reg, tmp_reg)); - - /* Compute address of memory block we save into. We always use pointer - pointing 127 bytes after first byte to store - this is needed to keep - instruction size limited by 4 bytes (5 bytes for AVX) with one - byte displacement. */ - tmp_reg = gen_reg_rtx (Pmode); - emit_insn (gen_rtx_SET (VOIDmode, tmp_reg, - plus_constant (save_area, - ix86_varargs_gpr_size + 127))); - mem = gen_rtx_MEM (BLKmode, plus_constant (tmp_reg, -127)); - MEM_NOTRAP_P (mem) = 1; - set_mem_alias_set (mem, set); - set_mem_align (mem, BITS_PER_WORD); - - /* And finally do the dirty job! */ - emit_insn (gen_sse_prologue_save (mem, nsse_reg, - GEN_INT (cum->sse_regno), label)); + test = gen_rtx_EQ (VOIDmode, gen_rtx_REG (QImode, AX_REG), const0_rtx); + emit_jump_insn (gen_cbranchqi4 (test, XEXP (test, 0), XEXP (test, 1), + label)); + + /* ??? If !TARGET_SSE_TYPELESS_STORES, would we perform better if + we used movdqa (i.e. TImode) instead? Perhaps even better would + be if we could determine the real mode of the data, via a hook + into pass_stdarg. Ignore all that for now. */ + smode = V4SFmode; + if (crtl->stack_alignment_needed < GET_MODE_ALIGNMENT (smode)) + crtl->stack_alignment_needed = GET_MODE_ALIGNMENT (smode); + + max = cum->sse_regno + cfun->va_list_fpr_size / 16; + if (max > X86_64_SSE_REGPARM_MAX) + max = X86_64_SSE_REGPARM_MAX; + + for (i = cum->sse_regno; i < max; ++i) + { + mem = plus_constant (save_area, i * 16 + ix86_varargs_gpr_size); + mem = gen_rtx_MEM (smode, mem); + MEM_NOTRAP_P (mem) = 1; + set_mem_alias_set (mem, set); + set_mem_align (mem, GET_MODE_ALIGNMENT (smode)); + + emit_move_insn (mem, gen_rtx_REG (smode, SSE_REGNO (i))); + } + + emit_label (label); } } @@ -6878,7 +7966,7 @@ For stdargs, we do want to skip the last named argument. */ next_cum = *cum; if (stdarg_p (fntype)) - function_arg_advance (&next_cum, mode, type, 1); + ix86_function_arg_advance (&next_cum, mode, type, true); if (cum->call_abi == MS_ABI) setup_incoming_varargs_ms_64 (&next_cum); @@ -6910,24 +7998,75 @@ tree f_gpr, f_fpr, f_ovf, f_sav; tree gpr, fpr, ovf, sav, t; tree type; + rtx ovf_rtx; + + if (flag_split_stack + && cfun->machine->split_stack_varargs_pointer == NULL_RTX) + { + unsigned int scratch_regno; + + /* When we are splitting the stack, we can't refer to the stack + arguments using internal_arg_pointer, because they may be on + the old stack. The split stack prologue will arrange to + leave a pointer to the old stack arguments in a scratch + register, which we here copy to a pseudo-register. The split + stack prologue can't set the pseudo-register directly because + it (the prologue) runs before any registers have been saved. */ + + scratch_regno = split_stack_prologue_scratch_regno (); + if (scratch_regno != INVALID_REGNUM) + { + rtx reg, seq; + + reg = gen_reg_rtx (Pmode); + cfun->machine->split_stack_varargs_pointer = reg; + + start_sequence (); + emit_move_insn (reg, gen_rtx_REG (Pmode, scratch_regno)); + seq = get_insns (); + end_sequence (); + + push_topmost_sequence (); + emit_insn_after (seq, entry_of_function ()); + pop_topmost_sequence (); + } + } /* Only 64bit target needs something special. */ if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist))) { - std_expand_builtin_va_start (valist, nextarg); + if (cfun->machine->split_stack_varargs_pointer == NULL_RTX) + std_expand_builtin_va_start (valist, nextarg); + else + { + rtx va_r, next; + + va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE); + next = expand_binop (ptr_mode, add_optab, + cfun->machine->split_stack_varargs_pointer, + crtl->args.arg_offset_rtx, + NULL_RTX, 0, OPTAB_LIB_WIDEN); + convert_move (va_r, next, 0); + } return; } f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node)); - f_fpr = TREE_CHAIN (f_gpr); - f_ovf = TREE_CHAIN (f_fpr); - f_sav = TREE_CHAIN (f_ovf); - - valist = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (valist)), valist); - gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), valist, f_gpr, NULL_TREE); - fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), valist, f_fpr, NULL_TREE); - ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), valist, f_ovf, NULL_TREE); - sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), valist, f_sav, NULL_TREE); + f_fpr = DECL_CHAIN (f_gpr); + f_ovf = DECL_CHAIN (f_fpr); + f_sav = DECL_CHAIN (f_ovf); + + valist = build_simple_mem_ref (valist); + TREE_TYPE (valist) = TREE_TYPE (sysv_va_list_type_node); + /* The following should be folded into the MEM_REF offset. */ + gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), unshare_expr (valist), + f_gpr, NULL_TREE); + fpr = build3 (COMPONENT_REF, TREE_TYPE (f_fpr), unshare_expr (valist), + f_fpr, NULL_TREE); + ovf = build3 (COMPONENT_REF, TREE_TYPE (f_ovf), unshare_expr (valist), + f_ovf, NULL_TREE); + sav = build3 (COMPONENT_REF, TREE_TYPE (f_sav), unshare_expr (valist), + f_sav, NULL_TREE); /* Count number of gp and fp argument registers used. */ words = crtl->args.info.words; @@ -6954,7 +8093,11 @@ /* Find the overflow area. */ type = TREE_TYPE (ovf); - t = make_tree (type, crtl->args.internal_arg_pointer); + if (cfun->machine->split_stack_varargs_pointer == NULL_RTX) + ovf_rtx = crtl->args.internal_arg_pointer; + else + ovf_rtx = cfun->machine->split_stack_varargs_pointer; + t = make_tree (type, ovf_rtx); if (words != 0) t = build2 (POINTER_PLUS_EXPR, type, t, size_int (words * UNITS_PER_WORD)); @@ -6993,16 +8136,16 @@ int indirect_p = 0; tree ptrtype; enum machine_mode nat_mode; - int arg_boundary; + unsigned int arg_boundary; /* Only 64bit target needs something special. */ if (!TARGET_64BIT || is_va_list_char_pointer (TREE_TYPE (valist))) return std_gimplify_va_arg_expr (valist, type, pre_p, post_p); f_gpr = TYPE_FIELDS (TREE_TYPE (sysv_va_list_type_node)); - f_fpr = TREE_CHAIN (f_gpr); - f_ovf = TREE_CHAIN (f_fpr); - f_sav = TREE_CHAIN (f_ovf); + f_fpr = DECL_CHAIN (f_gpr); + f_ovf = DECL_CHAIN (f_fpr); + f_sav = DECL_CHAIN (f_ovf); gpr = build3 (COMPONENT_REF, TREE_TYPE (f_gpr), build_va_arg_indirect_ref (valist), f_gpr, NULL_TREE); @@ -7139,7 +8282,7 @@ } if (need_temp) { - int i; + int i, prev_size = 0; tree temp = create_tmp_var (type, "va_arg_tmp"); /* addr = &temp; */ @@ -7151,13 +8294,31 @@ rtx slot = XVECEXP (container, 0, i); rtx reg = XEXP (slot, 0); enum machine_mode mode = GET_MODE (reg); - tree piece_type = lang_hooks.types.type_for_mode (mode, 1); - tree addr_type = build_pointer_type (piece_type); - tree daddr_type = build_pointer_type_for_mode (piece_type, - ptr_mode, true); + tree piece_type; + tree addr_type; + tree daddr_type; tree src_addr, src; int src_offset; tree dest_addr, dest; + int cur_size = GET_MODE_SIZE (mode); + + gcc_assert (prev_size <= INTVAL (XEXP (slot, 1))); + prev_size = INTVAL (XEXP (slot, 1)); + if (prev_size + cur_size > size) + { + cur_size = size - prev_size; + mode = mode_for_size (cur_size * BITS_PER_UNIT, MODE_INT, 1); + if (mode == BLKmode) + mode = QImode; + } + piece_type = lang_hooks.types.type_for_mode (mode, 1); + if (mode == GET_MODE (reg)) + addr_type = build_pointer_type (piece_type); + else + addr_type = build_pointer_type_for_mode (piece_type, ptr_mode, + true); + daddr_type = build_pointer_type_for_mode (piece_type, ptr_mode, + true); if (SSE_REGNO_P (REGNO (reg))) { @@ -7172,14 +8333,26 @@ src_addr = fold_convert (addr_type, src_addr); src_addr = fold_build2 (POINTER_PLUS_EXPR, addr_type, src_addr, size_int (src_offset)); - src = build_va_arg_indirect_ref (src_addr); dest_addr = fold_convert (daddr_type, addr); dest_addr = fold_build2 (POINTER_PLUS_EXPR, daddr_type, dest_addr, - size_int (INTVAL (XEXP (slot, 1)))); - dest = build_va_arg_indirect_ref (dest_addr); - - gimplify_assign (dest, src, pre_p); + size_int (prev_size)); + if (cur_size == GET_MODE_SIZE (mode)) + { + src = build_va_arg_indirect_ref (src_addr); + dest = build_va_arg_indirect_ref (dest_addr); + + gimplify_assign (dest, src, pre_p); + } + else + { + tree copy + = build_call_expr (implicit_built_in_decls[BUILT_IN_MEMCPY], + 3, dest_addr, src_addr, + size_int (cur_size)); + gimplify_and_add (copy, pre_p); + } + prev_size += cur_size; } } @@ -7208,13 +8381,12 @@ alignment is beyond MAX_SUPPORTED_STACK_ALIGNMENT, it will be aligned at MAX_SUPPORTED_STACK_ALIGNMENT. We will match callee here with caller. */ - arg_boundary = FUNCTION_ARG_BOUNDARY (VOIDmode, type); + arg_boundary = ix86_function_arg_boundary (VOIDmode, type); if ((unsigned int) arg_boundary > MAX_SUPPORTED_STACK_ALIGNMENT) arg_boundary = MAX_SUPPORTED_STACK_ALIGNMENT; /* Care for on-stack alignment if needed. */ - if (arg_boundary <= 64 - || integer_zerop (TYPE_SIZE (type))) + if (arg_boundary <= 64 || size == 0) t = ovf; else { @@ -7226,6 +8398,7 @@ size_int (-align)); t = fold_convert (TREE_TYPE (ovf), t); } + gimplify_expr (&t, pre_p, NULL, is_gimple_val, fb_rvalue); gimplify_assign (addr, t, pre_p); @@ -7244,10 +8417,10 @@ return build_va_arg_indirect_ref (addr); } -/* Return nonzero if OPNUM's MEM should be matched +/* Return true if OPNUM's MEM should be matched in movabs* patterns. */ -int +bool ix86_check_movabs (rtx insn, int opnum) { rtx set, mem; @@ -7260,7 +8433,7 @@ while (GET_CODE (mem) == SUBREG) mem = SUBREG_REG (mem); gcc_assert (MEM_P (mem)); - return (volatile_ok || !MEM_VOLATILE_P (mem)); + return volatile_ok || !MEM_VOLATILE_P (mem); } /* Initialize the table of extra 80387 mathematical constants. */ @@ -7289,8 +8462,8 @@ ext_80387_constants_init = 1; } -/* Return true if the constant is something that can be loaded with - a special instruction. */ +/* Return non-zero if the constant is something that + can be loaded with a special instruction. */ int standard_80387_constant_p (rtx x) @@ -7434,15 +8607,27 @@ case MODE_V4SF: return TARGET_AVX ? "vxorps\t%0, %0, %0" : "xorps\t%0, %0"; case MODE_V2DF: - return TARGET_AVX ? "vxorpd\t%0, %0, %0" : "xorpd\t%0, %0"; + if (TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL) + return TARGET_AVX ? "vxorps\t%0, %0, %0" : "xorps\t%0, %0"; + else + return TARGET_AVX ? "vxorpd\t%0, %0, %0" : "xorpd\t%0, %0"; case MODE_TI: - return TARGET_AVX ? "vpxor\t%0, %0, %0" : "pxor\t%0, %0"; + if (TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL) + return TARGET_AVX ? "vxorps\t%0, %0, %0" : "xorps\t%0, %0"; + else + return TARGET_AVX ? "vpxor\t%0, %0, %0" : "pxor\t%0, %0"; case MODE_V8SF: return "vxorps\t%x0, %x0, %x0"; case MODE_V4DF: - return "vxorpd\t%x0, %x0, %x0"; + if (TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL) + return "vxorps\t%x0, %x0, %x0"; + else + return "vxorpd\t%x0, %x0, %x0"; case MODE_OI: - return "vpxor\t%x0, %x0, %x0"; + if (TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL) + return "vxorps\t%x0, %x0, %x0"; + else + return "vpxor\t%x0, %x0, %x0"; default: break; } @@ -7454,16 +8639,16 @@ gcc_unreachable (); } -/* Returns 1 if OP contains a symbol reference */ - -int +/* Returns true if OP contains a symbol reference */ + +bool symbolic_reference_mentioned_p (rtx op) { const char *fmt; int i; if (GET_CODE (op) == SYMBOL_REF || GET_CODE (op) == LABEL_REF) - return 1; + return true; fmt = GET_RTX_FORMAT (GET_CODE (op)); for (i = GET_RTX_LENGTH (GET_CODE (op)) - 1; i >= 0; i--) @@ -7474,23 +8659,23 @@ for (j = XVECLEN (op, i) - 1; j >= 0; j--) if (symbolic_reference_mentioned_p (XVECEXP (op, i, j))) - return 1; + return true; } else if (fmt[i] == 'e' && symbolic_reference_mentioned_p (XEXP (op, i))) - return 1; - } - - return 0; -} - -/* Return 1 if it is appropriate to emit `ret' instructions in the + return true; + } + + return false; +} + +/* Return true if it is appropriate to emit `ret' instructions in the body of a function. Do this only if the epilogue is simple, needing a couple of insns. Prior to reloading, we can't tell how many registers - must be saved, so return 0 then. Return 0 if there is no frame + must be saved, so return false then. Return false if there is no frame marker to de-allocate. */ -int +bool ix86_can_use_return_insn_p (void) { struct ix86_frame frame; @@ -7498,15 +8683,14 @@ if (! reload_completed || frame_pointer_needed) return 0; - /* Don't allow more than 32 pop, since that's all we can do + /* Don't allow more than 32k pop, since that's all we can do with one instruction. */ - if (crtl->args.pops_args - && crtl->args.size >= 32768) + if (crtl->args.pops_args && crtl->args.size >= 32768) return 0; ix86_compute_frame_layout (&frame); - return frame.to_allocate == 0 && frame.padding0 == 0 - && (frame.nregs + frame.nsseregs) == 0; + return (frame.stack_pointer_offset == UNITS_PER_WORD + && (frame.nregs + frame.nsseregs) == 0); } /* Value should be nonzero if functions must have frame pointers. @@ -7526,15 +8710,15 @@ if (SUBTARGET_FRAME_POINTER_REQUIRED) return true; - /* In override_options, TARGET_OMIT_LEAF_FRAME_POINTER turns off - the frame pointer by default. Turn it back on now if we've not - got a leaf function. */ + /* In ix86_option_override_internal, TARGET_OMIT_LEAF_FRAME_POINTER + turns off the frame pointer by default. Turn it back on now if + we've not got a leaf function. */ if (TARGET_OMIT_LEAF_FRAME_POINTER && (!current_function_is_leaf || ix86_current_function_calls_tls_descriptor)) return true; - if (crtl->profile) + if (crtl->profile && !flag_fentry) return true; return false; @@ -7576,21 +8760,30 @@ /* This function generates code for -fpic that loads %ebx with the return address of the caller and then returns. */ -void -ix86_file_end (void) +static void +ix86_code_end (void) { rtx xops[2]; int regno; - for (regno = 0; regno < 8; ++regno) + for (regno = AX_REG; regno <= SP_REG; regno++) { char name[32]; - - if (! ((pic_labels_used >> regno) & 1)) + tree decl; + + if (!(pic_labels_used & (1 << regno))) continue; get_pc_thunk_name (name, regno); + decl = build_decl (BUILTINS_LOCATION, FUNCTION_DECL, + get_identifier (name), + build_function_type (void_type_node, void_list_node)); + DECL_RESULT (decl) = build_decl (BUILTINS_LOCATION, RESULT_DECL, + NULL_TREE, void_type_node); + TREE_PUBLIC (decl) = 1; + TREE_STATIC (decl) = 1; + #if TARGET_MACHO if (TARGET_MACHO) { @@ -7599,26 +8792,20 @@ assemble_name (asm_out_file, name); fputs ("\n\t.private_extern\t", asm_out_file); assemble_name (asm_out_file, name); - fputs ("\n", asm_out_file); + putc ('\n', asm_out_file); ASM_OUTPUT_LABEL (asm_out_file, name); + DECL_WEAK (decl) = 1; } else #endif if (USE_HIDDEN_LINKONCE) { - tree decl; - - decl = build_decl (BUILTINS_LOCATION, - FUNCTION_DECL, get_identifier (name), - error_mark_node); - TREE_PUBLIC (decl) = 1; - TREE_STATIC (decl) = 1; DECL_COMDAT_GROUP (decl) = DECL_ASSEMBLER_NAME (decl); - (*targetm.asm_out.unique_section) (decl, 0); + targetm.asm_out.unique_section (decl, 0); switch_to_section (get_named_section (decl, NULL, 0)); - (*targetm.asm_out.globalize_label) (asm_out_file, name); + targetm.asm_out.globalize_label (asm_out_file, name); fputs ("\t.hidden\t", asm_out_file); assemble_name (asm_out_file, name); putc ('\n', asm_out_file); @@ -7630,14 +8817,36 @@ ASM_OUTPUT_LABEL (asm_out_file, name); } + DECL_INITIAL (decl) = make_node (BLOCK); + current_function_decl = decl; + init_function_start (decl); + first_function_block_is_cold = false; + /* Make sure unwind info is emitted for the thunk if needed. */ + final_start_function (emit_barrier (), asm_out_file, 1); + + /* Pad stack IP move with 4 instructions (two NOPs count + as one instruction). */ + if (TARGET_PAD_SHORT_FUNCTION) + { + int i = 8; + + while (i--) + fputs ("\tnop\n", asm_out_file); + } + xops[0] = gen_rtx_REG (Pmode, regno); xops[1] = gen_rtx_MEM (Pmode, stack_pointer_rtx); output_asm_insn ("mov%z0\t{%1, %0|%0, %1}", xops); - output_asm_insn ("ret", xops); - } - - if (NEED_INDICATE_EXEC_STACK) - file_end_indicate_exec_stack (); + fputs ("\tret\n", asm_out_file); + final_end_function (); + init_insn_lengths (); + free_after_compilation (cfun); + set_cfun (NULL); + current_function_decl = NULL; + } + + if (flag_split_stack) + file_end_indicate_split_stack (); } /* Emit code for the SET_GOT patterns. */ @@ -7674,7 +8883,24 @@ if (!flag_pic) output_asm_insn ("mov%z0\t{%2, %0|%0, %2}", xops); else - output_asm_insn ("call\t%a2", xops); + { + output_asm_insn ("call\t%a2", xops); +#ifdef DWARF2_UNWIND_INFO + /* The call to next label acts as a push. */ + if (dwarf2out_do_frame ()) + { + rtx insn; + start_sequence (); + insn = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, + gen_rtx_PLUS (Pmode, + stack_pointer_rtx, + GEN_INT (-4)))); + RTX_FRAME_RELATED_P (insn) = 1; + dwarf2out_frame_debug (insn, true); + end_sequence (); + } +#endif + } #if TARGET_MACHO /* Output the Mach-O "canonical" label name ("Lxx$pb") here too. This @@ -7683,11 +8909,31 @@ ASM_OUTPUT_LABEL (asm_out_file, MACHOPIC_FUNCTION_BASE_NAME); #endif - (*targetm.asm_out.internal_label) (asm_out_file, "L", - CODE_LABEL_NUMBER (XEXP (xops[2], 0))); + targetm.asm_out.internal_label (asm_out_file, "L", + CODE_LABEL_NUMBER (XEXP (xops[2], 0))); if (flag_pic) - output_asm_insn ("pop%z0\t%0", xops); + { + output_asm_insn ("pop%z0\t%0", xops); +#ifdef DWARF2_UNWIND_INFO + /* The pop is a pop and clobbers dest, but doesn't restore it + for unwind info purposes. */ + if (dwarf2out_do_frame ()) + { + rtx insn; + start_sequence (); + insn = emit_insn (gen_rtx_SET (VOIDmode, dest, const0_rtx)); + dwarf2out_frame_debug (insn, true); + insn = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, + gen_rtx_PLUS (Pmode, + stack_pointer_rtx, + GEN_INT (4)))); + RTX_FRAME_RELATED_P (insn) = 1; + dwarf2out_frame_debug (insn, true); + end_sequence (); + } +#endif + } } else { @@ -7695,6 +8941,12 @@ get_pc_thunk_name (name, REGNO (dest)); pic_labels_used |= 1 << REGNO (dest); +#ifdef DWARF2_UNWIND_INFO + /* Ensure all queued register saves are flushed before the + call. */ + if (dwarf2out_do_frame ()) + dwarf2out_flush_queued_reg_saves (); +#endif xops[2] = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (name)); xops[2] = gen_rtx_MEM (QImode, xops[2]); output_asm_insn ("call\t%X2", xops); @@ -7725,8 +8977,11 @@ static rtx gen_push (rtx arg) { - if (ix86_cfa_state->reg == stack_pointer_rtx) - ix86_cfa_state->offset += UNITS_PER_WORD; + struct machine_function *m = cfun->machine; + + if (m->fs.cfa_reg == stack_pointer_rtx) + m->fs.cfa_offset += UNITS_PER_WORD; + m->fs.sp_offset += UNITS_PER_WORD; return gen_rtx_SET (VOIDmode, gen_rtx_MEM (Pmode, @@ -7735,13 +8990,26 @@ arg); } +/* Generate an "pop" pattern for input ARG. */ + +static rtx +gen_pop (rtx arg) +{ + return gen_rtx_SET (VOIDmode, + arg, + gen_rtx_MEM (Pmode, + gen_rtx_POST_INC (Pmode, + stack_pointer_rtx))); +} + /* Return >= 0 if there is an unused call-clobbered register available for the entire function. */ static unsigned int ix86_select_alt_pic_regnum (void) { - if (current_function_is_leaf && !crtl->profile + if (current_function_is_leaf + && !crtl->profile && !ix86_current_function_calls_tls_descriptor) { int i, drap; @@ -7881,6 +9149,38 @@ return stack_realign_fp ? hard_frame_pointer_rtx : virtual_stack_vars_rtx; } +/* On the x86 -fsplit-stack and -fstack-protector both use the same + field in the TCB, so they can not be used together. */ + +static bool +ix86_supports_split_stack (bool report ATTRIBUTE_UNUSED, + struct gcc_options *opts ATTRIBUTE_UNUSED) +{ + bool ret = true; + +#ifndef TARGET_THREAD_SPLIT_STACK_OFFSET + if (report) + error ("%<-fsplit-stack%> currently only supported on GNU/Linux"); + ret = false; +#else + if (!HAVE_GAS_CFI_PERSONALITY_DIRECTIVE) + { + if (report) + error ("%<-fsplit-stack%> requires " + "assembler support for CFI directives"); + ret = false; + } +#endif + + return ret; +} + +/* When using -fsplit-stack, the allocation routines set a field in + the TCB to the bottom of the stack plus this much space, measured + in bytes. */ + +#define SPLIT_STACK_AVAILABLE 256 + /* Fill structure ix86_frame about frame of currently computed function. */ static void @@ -7890,6 +9190,7 @@ HOST_WIDE_INT offset; unsigned int preferred_alignment; HOST_WIDE_INT size = get_frame_size (); + HOST_WIDE_INT to_allocate; frame->nregs = ix86_nsaved_regs (); frame->nsseregs = ix86_nsaved_sseregs (); @@ -7898,8 +9199,10 @@ preferred_alignment = crtl->preferred_stack_boundary / BITS_PER_UNIT; /* MS ABI seem to require stack alignment to be always 16 except for function - prologues. */ - if (ix86_cfun_abi () == MS_ABI && preferred_alignment < 16) + prologues and leaf. */ + if ((ix86_cfun_abi () == MS_ABI && preferred_alignment < 16) + && (!current_function_is_leaf || cfun->calls_alloca != 0 + || ix86_current_function_calls_tls_descriptor)) { preferred_alignment = 16; stack_alignment_needed = 16; @@ -7911,16 +9214,25 @@ gcc_assert (preferred_alignment >= STACK_BOUNDARY / BITS_PER_UNIT); gcc_assert (preferred_alignment <= stack_alignment_needed); + /* For SEH we have to limit the amount of code movement into the prologue. + At present we do this via a BLOCKAGE, at which point there's very little + scheduling that can be done, which means that there's very little point + in doing anything except PUSHs. */ + if (TARGET_SEH) + cfun->machine->use_fast_prologue_epilogue = false; + /* During reload iteration the amount of registers saved can change. Recompute the value as needed. Do not recompute when amount of registers didn't change as reload does multiple calls to the function and does not expect the decision to change within single iteration. */ - if (!optimize_function_for_size_p (cfun) - && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs) + else if (!optimize_function_for_size_p (cfun) + && cfun->machine->use_fast_prologue_epilogue_nregs != frame->nregs) { int count = frame->nregs; + struct cgraph_node *node = cgraph_node (current_function_decl); cfun->machine->use_fast_prologue_epilogue_nregs = count; + /* The fast prologue uses move instead of push to save registers. This is significantly longer, but also executes faster as modern hardware can execute the moves in parallel, but can't do that for push/pop. @@ -7933,9 +9245,9 @@ slow to use many of them. */ if (count) count = (count - 1) * FAST_PROLOGUE_INSN_COUNT; - if (cfun->function_frequency < FUNCTION_FREQUENCY_NORMAL + if (node->frequency < NODE_FREQUENCY_NORMAL || (flag_branch_probabilities - && cfun->function_frequency < FUNCTION_FREQUENCY_HOT)) + && node->frequency < NODE_FREQUENCY_HOT)) cfun->machine->use_fast_prologue_epilogue = false; else cfun->machine->use_fast_prologue_epilogue @@ -7947,6 +9259,11 @@ else frame->save_regs_using_mov = false; + /* If static stack checking is enabled and done with probes, the registers + need to be saved before allocating the frame. */ + if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK) + frame->save_regs_using_mov = false; + /* Skip return address. */ offset = UNITS_PER_WORD; @@ -7957,35 +9274,46 @@ /* Skip saved base pointer. */ if (frame_pointer_needed) offset += UNITS_PER_WORD; - + frame->hfp_save_offset = offset; + + /* The traditional frame pointer location is at the top of the frame. */ frame->hard_frame_pointer_offset = offset; - /* Set offset to aligned because the realigned frame starts from - here. */ - if (stack_realign_fp) - offset = (offset + stack_alignment_needed -1) & -stack_alignment_needed; - /* Register save area */ offset += frame->nregs * UNITS_PER_WORD; - - /* Align SSE reg save area. */ + frame->reg_save_offset = offset; + + /* Align and set SSE register save area. */ if (frame->nsseregs) - frame->padding0 = ((offset + 16 - 1) & -16) - offset; - else - frame->padding0 = 0; - - /* SSE register save area. */ - offset += frame->padding0 + frame->nsseregs * 16; + { + /* The only ABI that has saved SSE registers (Win64) also has a + 16-byte aligned default stack, and thus we don't need to be + within the re-aligned local stack frame to save them. */ + gcc_assert (INCOMING_STACK_BOUNDARY >= 128); + offset = (offset + 16 - 1) & -16; + offset += frame->nsseregs * 16; + } + frame->sse_reg_save_offset = offset; + + /* The re-aligned stack starts here. Values before this point are not + directly comparable with values below this point. In order to make + sure that no value happens to be the same before and after, force + the alignment computation below to add a non-zero value. */ + if (stack_realign_fp) + offset = (offset + stack_alignment_needed) & -stack_alignment_needed; /* Va-arg area */ frame->va_arg_size = ix86_varargs_gpr_size + ix86_varargs_fpr_size; offset += frame->va_arg_size; /* Align start of frame for local function. */ - frame->padding1 = ((offset + stack_alignment_needed - 1) - & -stack_alignment_needed) - offset; - - offset += frame->padding1; + if (stack_realign_fp + || offset != frame->sse_reg_save_offset + || size != 0 + || !current_function_is_leaf + || cfun->calls_alloca + || ix86_current_function_calls_tls_descriptor) + offset = (offset + stack_alignment_needed - 1) & -stack_alignment_needed; /* Frame pointer points here. */ frame->frame_pointer_offset = offset; @@ -8011,31 +9339,24 @@ or using alloca. */ if (!current_function_is_leaf || cfun->calls_alloca || ix86_current_function_calls_tls_descriptor) - frame->padding2 = ((offset + preferred_alignment - 1) - & -preferred_alignment) - offset; - else - frame->padding2 = 0; - - offset += frame->padding2; + offset = (offset + preferred_alignment - 1) & -preferred_alignment; /* We've reached end of stack frame. */ frame->stack_pointer_offset = offset; /* Size prologue needs to allocate. */ - frame->to_allocate = - (size + frame->padding1 + frame->padding2 - + frame->outgoing_arguments_size + frame->va_arg_size); - - if ((!frame->to_allocate && frame->nregs <= 1) - || (TARGET_64BIT && frame->to_allocate >= (HOST_WIDE_INT) 0x80000000)) + to_allocate = offset - frame->sse_reg_save_offset; + + if ((!to_allocate && frame->nregs <= 1) + || (TARGET_64BIT && to_allocate >= (HOST_WIDE_INT) 0x80000000)) frame->save_regs_using_mov = false; - if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE + if (ix86_using_red_zone () && current_function_sp_is_unchanging && current_function_is_leaf && !ix86_current_function_calls_tls_descriptor) { - frame->red_zone_size = frame->to_allocate; + frame->red_zone_size = to_allocate; if (frame->save_regs_using_mov) frame->red_zone_size += frame->nregs * UNITS_PER_WORD; if (frame->red_zone_size > RED_ZONE_SIZE - RED_ZONE_RESERVE) @@ -8043,8 +9364,126 @@ } else frame->red_zone_size = 0; - frame->to_allocate -= frame->red_zone_size; frame->stack_pointer_offset -= frame->red_zone_size; + + /* The SEH frame pointer location is near the bottom of the frame. + This is enforced by the fact that the difference between the + stack pointer and the frame pointer is limited to 240 bytes in + the unwind data structure. */ + if (TARGET_SEH) + { + HOST_WIDE_INT diff; + + /* If we can leave the frame pointer where it is, do so. */ + diff = frame->stack_pointer_offset - frame->hard_frame_pointer_offset; + if (diff > 240 || (diff & 15) != 0) + { + /* Ideally we'd determine what portion of the local stack frame + (within the constraint of the lowest 240) is most heavily used. + But without that complication, simply bias the frame pointer + by 128 bytes so as to maximize the amount of the local stack + frame that is addressable with 8-bit offsets. */ + frame->hard_frame_pointer_offset = frame->stack_pointer_offset - 128; + } + } +} + +/* This is semi-inlined memory_address_length, but simplified + since we know that we're always dealing with reg+offset, and + to avoid having to create and discard all that rtl. */ + +static inline int +choose_baseaddr_len (unsigned int regno, HOST_WIDE_INT offset) +{ + int len = 4; + + if (offset == 0) + { + /* EBP and R13 cannot be encoded without an offset. */ + len = (regno == BP_REG || regno == R13_REG); + } + else if (IN_RANGE (offset, -128, 127)) + len = 1; + + /* ESP and R12 must be encoded with a SIB byte. */ + if (regno == SP_REG || regno == R12_REG) + len++; + + return len; +} + +/* Return an RTX that points to CFA_OFFSET within the stack frame. + The valid base registers are taken from CFUN->MACHINE->FS. */ + +static rtx +choose_baseaddr (HOST_WIDE_INT cfa_offset) +{ + const struct machine_function *m = cfun->machine; + rtx base_reg = NULL; + HOST_WIDE_INT base_offset = 0; + + if (m->use_fast_prologue_epilogue) + { + /* Choose the base register most likely to allow the most scheduling + opportunities. Generally FP is valid througout the function, + while DRAP must be reloaded within the epilogue. But choose either + over the SP due to increased encoding size. */ + + if (m->fs.fp_valid) + { + base_reg = hard_frame_pointer_rtx; + base_offset = m->fs.fp_offset - cfa_offset; + } + else if (m->fs.drap_valid) + { + base_reg = crtl->drap_reg; + base_offset = 0 - cfa_offset; + } + else if (m->fs.sp_valid) + { + base_reg = stack_pointer_rtx; + base_offset = m->fs.sp_offset - cfa_offset; + } + } + else + { + HOST_WIDE_INT toffset; + int len = 16, tlen; + + /* Choose the base register with the smallest address encoding. + With a tie, choose FP > DRAP > SP. */ + if (m->fs.sp_valid) + { + base_reg = stack_pointer_rtx; + base_offset = m->fs.sp_offset - cfa_offset; + len = choose_baseaddr_len (STACK_POINTER_REGNUM, base_offset); + } + if (m->fs.drap_valid) + { + toffset = 0 - cfa_offset; + tlen = choose_baseaddr_len (REGNO (crtl->drap_reg), toffset); + if (tlen <= len) + { + base_reg = crtl->drap_reg; + base_offset = toffset; + len = tlen; + } + } + if (m->fs.fp_valid) + { + toffset = m->fs.fp_offset - cfa_offset; + tlen = choose_baseaddr_len (HARD_FRAME_POINTER_REGNUM, toffset); + if (tlen <= len) + { + base_reg = hard_frame_pointer_rtx; + base_offset = toffset; + len = tlen; + } + } + } + gcc_assert (base_reg != NULL); + + return plus_constant (base_reg, base_offset); } /* Emit code to save registers in the prologue. */ @@ -8063,60 +9502,113 @@ } } -/* Emit code to save registers using MOV insns. First register - is restored from POINTER + OFFSET. */ -static void -ix86_emit_save_regs_using_mov (rtx pointer, HOST_WIDE_INT offset) +/* Emit a single register save at CFA - CFA_OFFSET. */ + +static void +ix86_emit_save_reg_using_mov (enum machine_mode mode, unsigned int regno, + HOST_WIDE_INT cfa_offset) +{ + struct machine_function *m = cfun->machine; + rtx reg = gen_rtx_REG (mode, regno); + rtx mem, addr, base, insn; + + addr = choose_baseaddr (cfa_offset); + mem = gen_frame_mem (mode, addr); + + /* For SSE saves, we need to indicate the 128-bit alignment. */ + set_mem_align (mem, GET_MODE_ALIGNMENT (mode)); + + insn = emit_move_insn (mem, reg); + RTX_FRAME_RELATED_P (insn) = 1; + + base = addr; + if (GET_CODE (base) == PLUS) + base = XEXP (base, 0); + gcc_checking_assert (REG_P (base)); + + /* When saving registers into a re-aligned local stack frame, avoid + any tricky guessing by dwarf2out. */ + if (m->fs.realigned) + { + gcc_checking_assert (stack_realign_drap); + + if (regno == REGNO (crtl->drap_reg)) + { + /* A bit of a hack. We force the DRAP register to be saved in + the re-aligned stack frame, which provides us with a copy + of the CFA that will last past the prologue. Install it. */ + gcc_checking_assert (cfun->machine->fs.fp_valid); + addr = plus_constant (hard_frame_pointer_rtx, + cfun->machine->fs.fp_offset - cfa_offset); + mem = gen_rtx_MEM (mode, addr); + add_reg_note (insn, REG_CFA_DEF_CFA, mem); + } + else + { + /* The frame pointer is a stable reference within the + aligned frame. Use it. */ + gcc_checking_assert (cfun->machine->fs.fp_valid); + addr = plus_constant (hard_frame_pointer_rtx, + cfun->machine->fs.fp_offset - cfa_offset); + mem = gen_rtx_MEM (mode, addr); + add_reg_note (insn, REG_CFA_EXPRESSION, + gen_rtx_SET (VOIDmode, mem, reg)); + } + } + + /* The memory may not be relative to the current CFA register, + which means that we may need to generate a new pattern for + use by the unwind info. */ + else if (base != m->fs.cfa_reg) + { + addr = plus_constant (m->fs.cfa_reg, m->fs.cfa_offset - cfa_offset); + mem = gen_rtx_MEM (mode, addr); + add_reg_note (insn, REG_CFA_OFFSET, gen_rtx_SET (VOIDmode, mem, reg)); + } +} + +/* Emit code to save registers using MOV insns. + First register is stored at CFA - CFA_OFFSET. */ +static void +ix86_emit_save_regs_using_mov (HOST_WIDE_INT cfa_offset) { unsigned int regno; - rtx insn; for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++) if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, true)) { - insn = emit_move_insn (adjust_address (gen_rtx_MEM (Pmode, pointer), - Pmode, offset), - gen_rtx_REG (Pmode, regno)); - RTX_FRAME_RELATED_P (insn) = 1; - offset += UNITS_PER_WORD; + ix86_emit_save_reg_using_mov (Pmode, regno, cfa_offset); + cfa_offset -= UNITS_PER_WORD; } } -/* Emit code to save registers using MOV insns. First register - is restored from POINTER + OFFSET. */ -static void -ix86_emit_save_sse_regs_using_mov (rtx pointer, HOST_WIDE_INT offset) +/* Emit code to save SSE registers using MOV insns. + First register is stored at CFA - CFA_OFFSET. */ +static void +ix86_emit_save_sse_regs_using_mov (HOST_WIDE_INT cfa_offset) { unsigned int regno; - rtx insn; - rtx mem; for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++) if (SSE_REGNO_P (regno) && ix86_save_reg (regno, true)) { - mem = adjust_address (gen_rtx_MEM (TImode, pointer), TImode, offset); - set_mem_align (mem, 128); - insn = emit_move_insn (mem, gen_rtx_REG (TImode, regno)); - RTX_FRAME_RELATED_P (insn) = 1; - offset += 16; + ix86_emit_save_reg_using_mov (V4SFmode, regno, cfa_offset); + cfa_offset -= 16; } } static GTY(()) rtx queued_cfa_restores; /* Add a REG_CFA_RESTORE REG note to INSN or queue them until next stack - manipulation insn. Don't add it if the previously - saved value will be left untouched within stack red-zone till return, - as unwinders can find the same value in the register and - on the stack. */ - -static void -ix86_add_cfa_restore_note (rtx insn, rtx reg, HOST_WIDE_INT red_offset) -{ - if (TARGET_RED_ZONE - && !TARGET_64BIT_MS_ABI - && red_offset + RED_ZONE_SIZE >= 0 - && crtl->args.pops_args < 65536) + manipulation insn. The value is on the stack at CFA - CFA_OFFSET. + Don't add the note if the previously saved value will be left untouched + within stack red-zone till return, as unwinders can find the same value + in the register and on the stack. */ + +static void +ix86_add_cfa_restore_note (rtx insn, rtx reg, HOST_WIDE_INT cfa_offset) +{ + if (cfa_offset <= cfun->machine->fs.red_zone_offset) return; if (insn) @@ -8155,28 +9647,35 @@ pro_epilogue_adjust_stack (rtx dest, rtx src, rtx offset, int style, bool set_cfa) { + struct machine_function *m = cfun->machine; rtx insn; + bool add_frame_related_expr = false; if (! TARGET_64BIT) - insn = emit_insn (gen_pro_epilogue_adjust_stack_1 (dest, src, offset)); + insn = gen_pro_epilogue_adjust_stack_si_add (dest, src, offset); else if (x86_64_immediate_operand (offset, DImode)) - insn = emit_insn (gen_pro_epilogue_adjust_stack_rex64 (dest, src, offset)); - else - { - rtx r11; + insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, offset); + else + { + rtx tmp; /* r11 is used by indirect sibcall return as well, set before the - epilogue and used after the epilogue. ATM indirect sibcall - shouldn't be used together with huge frame sizes in one - function because of the frame_size check in sibcall.c. */ - gcc_assert (style); - r11 = gen_rtx_REG (DImode, R11_REG); - insn = emit_insn (gen_rtx_SET (DImode, r11, offset)); + epilogue and used after the epilogue. */ + if (style) + tmp = gen_rtx_REG (DImode, R11_REG); + else + { + gcc_assert (src != hard_frame_pointer_rtx + && dest != hard_frame_pointer_rtx); + tmp = hard_frame_pointer_rtx; + } + insn = emit_insn (gen_rtx_SET (DImode, tmp, offset)); if (style < 0) - RTX_FRAME_RELATED_P (insn) = 1; - insn = emit_insn (gen_pro_epilogue_adjust_stack_rex64_2 (dest, src, r11, - offset)); - } - + add_frame_related_expr = true; + + insn = gen_pro_epilogue_adjust_stack_di_add (dest, src, tmp); + } + + insn = emit_insn (insn); if (style >= 0) ix86_add_queued_cfa_restore_notes (insn); @@ -8184,17 +9683,53 @@ { rtx r; - gcc_assert (ix86_cfa_state->reg == src); - ix86_cfa_state->offset += INTVAL (offset); - ix86_cfa_state->reg = dest; - + gcc_assert (m->fs.cfa_reg == src); + m->fs.cfa_offset += INTVAL (offset); + m->fs.cfa_reg = dest; + r = gen_rtx_PLUS (Pmode, src, offset); r = gen_rtx_SET (VOIDmode, dest, r); add_reg_note (insn, REG_CFA_ADJUST_CFA, r); RTX_FRAME_RELATED_P (insn) = 1; } else if (style < 0) - RTX_FRAME_RELATED_P (insn) = 1; + { + RTX_FRAME_RELATED_P (insn) = 1; + if (add_frame_related_expr) + { + rtx r = gen_rtx_PLUS (Pmode, src, offset); + r = gen_rtx_SET (VOIDmode, dest, r); + add_reg_note (insn, REG_FRAME_RELATED_EXPR, r); + } + } + + if (dest == stack_pointer_rtx) + { + HOST_WIDE_INT ooffset = m->fs.sp_offset; + bool valid = m->fs.sp_valid; + + if (src == hard_frame_pointer_rtx) + { + valid = m->fs.fp_valid; + ooffset = m->fs.fp_offset; + } + else if (src == crtl->drap_reg) + { + valid = m->fs.drap_valid; + ooffset = 0; + } + else + { + /* Else there are two possibilities: SP itself, which we set + up as the default above. Or EH_RETURN_STACKADJ_RTX, which is + taken care of this by hand along the eh_return path. */ + gcc_checking_assert (src == stack_pointer_rtx + || offset == const0_rtx); + } + + m->fs.sp_offset = ooffset - INTVAL (offset); + m->fs.sp_valid = valid; + } } /* Find an available register to be used as dynamic realign argument @@ -8208,7 +9743,7 @@ Return: the regno of chosen register. */ -static unsigned int +static unsigned int find_drap_reg (void) { tree decl = cfun->decl; @@ -8232,11 +9767,13 @@ register in such case. */ if (DECL_STATIC_CHAIN (decl) || crtl->tail_call_emit) return DI_REG; - + /* Reuse static chain register if it isn't used for parameter passing. */ if (ix86_function_regparm (TREE_TYPE (decl), decl) <= 2 && !lookup_attribute ("fastcall", + TYPE_ATTRIBUTES (TREE_TYPE (decl))) + && !lookup_attribute ("thiscall", TYPE_ATTRIBUTES (TREE_TYPE (decl)))) return CX_REG; else @@ -8255,7 +9792,7 @@ if (ix86_user_incoming_stack_boundary) incoming_stack_boundary = ix86_user_incoming_stack_boundary; /* In 32bit, use MIN_STACK_BOUNDARY for incoming stack boundary - if -mstackrealign is used, it isn't used for sibcall check and + if -mstackrealign is used, it isn't used for sibcall check and estimated stack alignment is 128bit. */ else if (!sibcall && !TARGET_64BIT @@ -8329,9 +9866,13 @@ drap_vreg = copy_to_reg (arg_ptr); seq = get_insns (); end_sequence (); - + insn = emit_insn_before (seq, NEXT_INSN (entry_of_function ())); - RTX_FRAME_RELATED_P (insn) = 1; + if (!optimize) + { + add_reg_note (insn, REG_CFA_SET_VDRAP, drap_vreg); + RTX_FRAME_RELATED_P (insn) = 1; + } return drap_vreg; } else @@ -8346,12 +9887,384 @@ return virtual_incoming_args_rtx; } +struct scratch_reg { + rtx reg; + bool saved; +}; + +/* Return a short-lived scratch register for use on function entry. + In 32-bit mode, it is valid only after the registers are saved + in the prologue. This register must be released by means of + release_scratch_register_on_entry once it is dead. */ + +static void +get_scratch_register_on_entry (struct scratch_reg *sr) +{ + int regno; + + sr->saved = false; + + if (TARGET_64BIT) + { + /* We always use R11 in 64-bit mode. */ + regno = R11_REG; + } + else + { + tree decl = current_function_decl, fntype = TREE_TYPE (decl); + bool fastcall_p + = lookup_attribute ("fastcall", TYPE_ATTRIBUTES (fntype)) != NULL_TREE; + bool static_chain_p = DECL_STATIC_CHAIN (decl); + int regparm = ix86_function_regparm (fntype, decl); + int drap_regno + = crtl->drap_reg ? REGNO (crtl->drap_reg) : INVALID_REGNUM; + + /* 'fastcall' sets regparm to 2, uses ecx/edx for arguments and eax + for the static chain register. */ + if ((regparm < 1 || (fastcall_p && !static_chain_p)) + && drap_regno != AX_REG) + regno = AX_REG; + else if (regparm < 2 && drap_regno != DX_REG) + regno = DX_REG; + /* ecx is the static chain register. */ + else if (regparm < 3 && !fastcall_p && !static_chain_p + && drap_regno != CX_REG) + regno = CX_REG; + else if (ix86_save_reg (BX_REG, true)) + regno = BX_REG; + /* esi is the static chain register. */ + else if (!(regparm == 3 && static_chain_p) + && ix86_save_reg (SI_REG, true)) + regno = SI_REG; + else if (ix86_save_reg (DI_REG, true)) + regno = DI_REG; + else + { + regno = (drap_regno == AX_REG ? DX_REG : AX_REG); + sr->saved = true; + } + } + + sr->reg = gen_rtx_REG (Pmode, regno); + if (sr->saved) + { + rtx insn = emit_insn (gen_push (sr->reg)); + RTX_FRAME_RELATED_P (insn) = 1; + } +} + +/* Release a scratch register obtained from the preceding function. */ + +static void +release_scratch_register_on_entry (struct scratch_reg *sr) +{ + if (sr->saved) + { + rtx x, insn = emit_insn (gen_pop (sr->reg)); + + /* The RTX_FRAME_RELATED_P mechanism doesn't know about pop. */ + RTX_FRAME_RELATED_P (insn) = 1; + x = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (UNITS_PER_WORD)); + x = gen_rtx_SET (VOIDmode, stack_pointer_rtx, x); + add_reg_note (insn, REG_FRAME_RELATED_EXPR, x); + } +} + +#define PROBE_INTERVAL (1 << STACK_CHECK_PROBE_INTERVAL_EXP) + +/* Emit code to adjust the stack pointer by SIZE bytes while probing it. */ + +static void +ix86_adjust_stack_and_probe (const HOST_WIDE_INT size) +{ + /* We skip the probe for the first interval + a small dope of 4 words and + probe that many bytes past the specified size to maintain a protection + area at the botton of the stack. */ + const int dope = 4 * UNITS_PER_WORD; + rtx size_rtx = GEN_INT (size); + + /* See if we have a constant small number of probes to generate. If so, + that's the easy case. The run-time loop is made up of 11 insns in the + generic case while the compile-time loop is made up of 3+2*(n-1) insns + for n # of intervals. */ + if (size <= 5 * PROBE_INTERVAL) + { + HOST_WIDE_INT i, adjust; + bool first_probe = true; + + /* Adjust SP and probe at PROBE_INTERVAL + N * PROBE_INTERVAL for + values of N from 1 until it exceeds SIZE. If only one probe is + needed, this will not generate any code. Then adjust and probe + to PROBE_INTERVAL + SIZE. */ + for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL) + { + if (first_probe) + { + adjust = 2 * PROBE_INTERVAL + dope; + first_probe = false; + } + else + adjust = PROBE_INTERVAL; + + emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, + plus_constant (stack_pointer_rtx, -adjust))); + emit_stack_probe (stack_pointer_rtx); + } + + if (first_probe) + adjust = size + PROBE_INTERVAL + dope; + else + adjust = size + PROBE_INTERVAL - i; + + emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, + plus_constant (stack_pointer_rtx, -adjust))); + emit_stack_probe (stack_pointer_rtx); + + /* Adjust back to account for the additional first interval. */ + emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, + plus_constant (stack_pointer_rtx, + PROBE_INTERVAL + dope))); + } + + /* Otherwise, do the same as above, but in a loop. Note that we must be + extra careful with variables wrapping around because we might be at + the very top (or the very bottom) of the address space and we have + to be able to handle this case properly; in particular, we use an + equality test for the loop condition. */ + else + { + HOST_WIDE_INT rounded_size; + struct scratch_reg sr; + + get_scratch_register_on_entry (&sr); + + + /* Step 1: round SIZE to the previous multiple of the interval. */ + + rounded_size = size & -PROBE_INTERVAL; + + + /* Step 2: compute initial and final value of the loop counter. */ + + /* SP = SP_0 + PROBE_INTERVAL. */ + emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, + plus_constant (stack_pointer_rtx, + - (PROBE_INTERVAL + dope)))); + + /* LAST_ADDR = SP_0 + PROBE_INTERVAL + ROUNDED_SIZE. */ + emit_move_insn (sr.reg, GEN_INT (-rounded_size)); + emit_insn (gen_rtx_SET (VOIDmode, sr.reg, + gen_rtx_PLUS (Pmode, sr.reg, + stack_pointer_rtx))); + + + /* Step 3: the loop + + while (SP != LAST_ADDR) + { + SP = SP + PROBE_INTERVAL + probe at SP + } + + adjusts SP and probes to PROBE_INTERVAL + N * PROBE_INTERVAL for + values of N from 1 until it is equal to ROUNDED_SIZE. */ + + emit_insn (ix86_gen_adjust_stack_and_probe (sr.reg, sr.reg, size_rtx)); + + + /* Step 4: adjust SP and probe at PROBE_INTERVAL + SIZE if we cannot + assert at compile-time that SIZE is equal to ROUNDED_SIZE. */ + + if (size != rounded_size) + { + emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, + plus_constant (stack_pointer_rtx, + rounded_size - size))); + emit_stack_probe (stack_pointer_rtx); + } + + /* Adjust back to account for the additional first interval. */ + emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, + plus_constant (stack_pointer_rtx, + PROBE_INTERVAL + dope))); + + release_scratch_register_on_entry (&sr); + } + + gcc_assert (cfun->machine->fs.cfa_reg != stack_pointer_rtx); + cfun->machine->fs.sp_offset += size; + + /* Make sure nothing is scheduled before we are done. */ + emit_insn (gen_blockage ()); +} + +/* Adjust the stack pointer up to REG while probing it. */ + +const char * +output_adjust_stack_and_probe (rtx reg) +{ + static int labelno = 0; + char loop_lab[32], end_lab[32]; + rtx xops[2]; + + ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno); + ASM_GENERATE_INTERNAL_LABEL (end_lab, "LPSRE", labelno++); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab); + + /* Jump to END_LAB if SP == LAST_ADDR. */ + xops[0] = stack_pointer_rtx; + xops[1] = reg; + output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops); + fputs ("\tje\t", asm_out_file); + assemble_name_raw (asm_out_file, end_lab); + fputc ('\n', asm_out_file); + + /* SP = SP + PROBE_INTERVAL. */ + xops[1] = GEN_INT (PROBE_INTERVAL); + output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops); + + /* Probe at SP. */ + xops[1] = const0_rtx; + output_asm_insn ("or%z0\t{%1, (%0)|DWORD PTR [%0], %1}", xops); + + fprintf (asm_out_file, "\tjmp\t"); + assemble_name_raw (asm_out_file, loop_lab); + fputc ('\n', asm_out_file); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, end_lab); + + return ""; +} + +/* Emit code to probe a range of stack addresses from FIRST to FIRST+SIZE, + inclusive. These are offsets from the current stack pointer. */ + +static void +ix86_emit_probe_stack_range (HOST_WIDE_INT first, HOST_WIDE_INT size) +{ + /* See if we have a constant small number of probes to generate. If so, + that's the easy case. The run-time loop is made up of 7 insns in the + generic case while the compile-time loop is made up of n insns for n # + of intervals. */ + if (size <= 7 * PROBE_INTERVAL) + { + HOST_WIDE_INT i; + + /* Probe at FIRST + N * PROBE_INTERVAL for values of N from 1 until + it exceeds SIZE. If only one probe is needed, this will not + generate any code. Then probe at FIRST + SIZE. */ + for (i = PROBE_INTERVAL; i < size; i += PROBE_INTERVAL) + emit_stack_probe (plus_constant (stack_pointer_rtx, -(first + i))); + + emit_stack_probe (plus_constant (stack_pointer_rtx, -(first + size))); + } + + /* Otherwise, do the same as above, but in a loop. Note that we must be + extra careful with variables wrapping around because we might be at + the very top (or the very bottom) of the address space and we have + to be able to handle this case properly; in particular, we use an + equality test for the loop condition. */ + else + { + HOST_WIDE_INT rounded_size, last; + struct scratch_reg sr; + + get_scratch_register_on_entry (&sr); + + + /* Step 1: round SIZE to the previous multiple of the interval. */ + + rounded_size = size & -PROBE_INTERVAL; + + + /* Step 2: compute initial and final value of the loop counter. */ + + /* TEST_OFFSET = FIRST. */ + emit_move_insn (sr.reg, GEN_INT (-first)); + + /* LAST_OFFSET = FIRST + ROUNDED_SIZE. */ + last = first + rounded_size; + + + /* Step 3: the loop + + while (TEST_ADDR != LAST_ADDR) + { + TEST_ADDR = TEST_ADDR + PROBE_INTERVAL + probe at TEST_ADDR + } + + probes at FIRST + N * PROBE_INTERVAL for values of N from 1 + until it is equal to ROUNDED_SIZE. */ + + emit_insn (ix86_gen_probe_stack_range (sr.reg, sr.reg, GEN_INT (-last))); + + + /* Step 4: probe at FIRST + SIZE if we cannot assert at compile-time + that SIZE is equal to ROUNDED_SIZE. */ + + if (size != rounded_size) + emit_stack_probe (plus_constant (gen_rtx_PLUS (Pmode, + stack_pointer_rtx, + sr.reg), + rounded_size - size)); + + release_scratch_register_on_entry (&sr); + } + + /* Make sure nothing is scheduled before we are done. */ + emit_insn (gen_blockage ()); +} + +/* Probe a range of stack addresses from REG to END, inclusive. These are + offsets from the current stack pointer. */ + +const char * +output_probe_stack_range (rtx reg, rtx end) +{ + static int labelno = 0; + char loop_lab[32], end_lab[32]; + rtx xops[3]; + + ASM_GENERATE_INTERNAL_LABEL (loop_lab, "LPSRL", labelno); + ASM_GENERATE_INTERNAL_LABEL (end_lab, "LPSRE", labelno++); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, loop_lab); + + /* Jump to END_LAB if TEST_ADDR == LAST_ADDR. */ + xops[0] = reg; + xops[1] = end; + output_asm_insn ("cmp%z0\t{%1, %0|%0, %1}", xops); + fputs ("\tje\t", asm_out_file); + assemble_name_raw (asm_out_file, end_lab); + fputc ('\n', asm_out_file); + + /* TEST_ADDR = TEST_ADDR + PROBE_INTERVAL. */ + xops[1] = GEN_INT (PROBE_INTERVAL); + output_asm_insn ("sub%z0\t{%1, %0|%0, %1}", xops); + + /* Probe at TEST_ADDR. */ + xops[0] = stack_pointer_rtx; + xops[1] = reg; + xops[2] = const0_rtx; + output_asm_insn ("or%z0\t{%2, (%0,%1)|DWORD PTR [%0+%1], %2}", xops); + + fprintf (asm_out_file, "\tjmp\t"); + assemble_name_raw (asm_out_file, loop_lab); + fputc ('\n', asm_out_file); + + ASM_OUTPUT_INTERNAL_LABEL (asm_out_file, end_lab); + + return ""; +} + /* Finalize stack_realign_needed flag, which will guide prologue/epilogue to be generated in correct form. */ -static void +static void ix86_finalize_stack_realign_flags (void) { - /* Check if stack realign is really needed after reload, and + /* Check if stack realign is really needed after reload, and stores result in cfun */ unsigned int incoming_stack_boundary = (crtl->parm_stack_boundary > ix86_incoming_stack_boundary @@ -8379,28 +10292,45 @@ void ix86_expand_prologue (void) { - rtx insn; + struct machine_function *m = cfun->machine; + rtx insn, t; bool pic_reg_used; struct ix86_frame frame; HOST_WIDE_INT allocate; - int gen_frame_pointer = frame_pointer_needed; + bool int_registers_saved; ix86_finalize_stack_realign_flags (); /* DRAP should not coexist with stack_realign_fp */ gcc_assert (!(crtl->drap_reg && stack_realign_fp)); + memset (&m->fs, 0, sizeof (m->fs)); + /* Initialize CFA state for before the prologue. */ - ix86_cfa_state->reg = stack_pointer_rtx; - ix86_cfa_state->offset = INCOMING_FRAME_SP_OFFSET; + m->fs.cfa_reg = stack_pointer_rtx; + m->fs.cfa_offset = INCOMING_FRAME_SP_OFFSET; + + /* Track SP offset to the CFA. We continue tracking this after we've + swapped the CFA register away from SP. In the case of re-alignment + this is fudged; we're interested to offsets within the local frame. */ + m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET; + m->fs.sp_valid = true; ix86_compute_frame_layout (&frame); - if (ix86_function_ms_hook_prologue (current_function_decl)) - { - rtx push, mov; - - /* Make sure the function starts with + if (!TARGET_64BIT && ix86_function_ms_hook_prologue (current_function_decl)) + { + /* We should have already generated an error for any use of + ms_hook on a nested function. */ + gcc_checking_assert (!ix86_static_chain_on_stack); + + /* Check if profiling is active and we shall use profiling before + prologue variant. If so sorry. */ + if (crtl->profile && flag_fentry != 0) + sorry ("ms_hook_prologue attribute isn%'t compatible " + "with -mfentry for 32-bit"); + + /* In ix86_asm_output_function_label we emitted: 8b ff movl.s %edi,%edi 55 push %ebp 8b ec movl.s %esp,%ebp @@ -8408,41 +10338,64 @@ This matches the hookable function prologue in Win32 API functions in Microsoft Windows XP Service Pack 2 and newer. Wine uses this to enable Windows apps to hook the Win32 API - functions provided by Wine. */ - insn = emit_insn (gen_vswapmov (gen_rtx_REG (SImode, DI_REG), - gen_rtx_REG (SImode, DI_REG))); - push = emit_insn (gen_push (hard_frame_pointer_rtx)); - mov = emit_insn (gen_vswapmov (hard_frame_pointer_rtx, - stack_pointer_rtx)); - - if (frame_pointer_needed && !(crtl->drap_reg - && crtl->stack_realign_needed)) - { - /* The push %ebp and movl.s %esp, %ebp already set up - the frame pointer. No need to do this again. */ - gen_frame_pointer = 0; + functions provided by Wine. + + What that means is that we've already set up the frame pointer. */ + + if (frame_pointer_needed + && !(crtl->drap_reg && crtl->stack_realign_needed)) + { + rtx push, mov; + + /* We've decided to use the frame pointer already set up. + Describe this to the unwinder by pretending that both + push and mov insns happen right here. + + Putting the unwind info here at the end of the ms_hook + is done so that we can make absolutely certain we get + the required byte sequence at the start of the function, + rather than relying on an assembler that can produce + the exact encoding required. + + However it does mean (in the unpatched case) that we have + a 1 insn window where the asynchronous unwind info is + incorrect. However, if we placed the unwind info at + its correct location we would have incorrect unwind info + in the patched case. Which is probably all moot since + I don't expect Wine generates dwarf2 unwind info for the + system libraries that use this feature. */ + + insn = emit_insn (gen_blockage ()); + + push = gen_push (hard_frame_pointer_rtx); + mov = gen_rtx_SET (VOIDmode, hard_frame_pointer_rtx, + stack_pointer_rtx); RTX_FRAME_RELATED_P (push) = 1; RTX_FRAME_RELATED_P (mov) = 1; - if (ix86_cfa_state->reg == stack_pointer_rtx) - ix86_cfa_state->reg = hard_frame_pointer_rtx; - } - else - /* If the frame pointer is not needed, pop %ebp again. This - could be optimized for cases where ebp needs to be backed up - for some other reason. If stack realignment is needed, pop - the base pointer again, align the stack, and later regenerate - the frame pointer setup. The frame pointer generated by the - hook prologue is not aligned, so it can't be used. */ - insn = emit_insn ((*ix86_gen_pop1) (hard_frame_pointer_rtx)); + + RTX_FRAME_RELATED_P (insn) = 1; + add_reg_note (insn, REG_FRAME_RELATED_EXPR, + gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, push, mov))); + + /* Note that gen_push incremented m->fs.cfa_offset, even + though we didn't emit the push insn here. */ + m->fs.cfa_reg = hard_frame_pointer_rtx; + m->fs.fp_offset = m->fs.cfa_offset; + m->fs.fp_valid = true; + } + else + { + /* The frame pointer is not needed so pop %ebp again. + This leaves us with a pristine state. */ + emit_insn (gen_pop (hard_frame_pointer_rtx)); + } } /* The first insn of a function that accepts its static chain on the stack is to push the register that would be filled in by a direct call. This insn will be skipped by the trampoline. */ - if (ix86_static_chain_on_stack) - { - rtx t; - + else if (ix86_static_chain_on_stack) + { insn = emit_insn (gen_push (ix86_static_chain (cfun->decl, false))); emit_insn (gen_blockage ()); @@ -8457,66 +10410,88 @@ /* Emit prologue code to adjust stack alignment and setup DRAP, in case of DRAP is needed and stack realignment is really needed after reload */ - if (crtl->drap_reg && crtl->stack_realign_needed) - { - rtx x, y; + if (stack_realign_drap) + { int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT; - int param_ptr_offset = UNITS_PER_WORD; - - if (ix86_static_chain_on_stack) - param_ptr_offset += UNITS_PER_WORD; - if (!call_used_regs[REGNO (crtl->drap_reg)]) - param_ptr_offset += UNITS_PER_WORD; - - gcc_assert (stack_realign_drap); - - /* Grab the argument pointer. */ - x = plus_constant (stack_pointer_rtx, param_ptr_offset); - y = crtl->drap_reg; - - /* Only need to push parameter pointer reg if it is caller - saved reg */ + + /* Only need to push parameter pointer reg if it is caller saved. */ if (!call_used_regs[REGNO (crtl->drap_reg)]) { /* Push arg pointer reg */ - insn = emit_insn (gen_push (y)); + insn = emit_insn (gen_push (crtl->drap_reg)); RTX_FRAME_RELATED_P (insn) = 1; } - insn = emit_insn (gen_rtx_SET (VOIDmode, y, x)); - RTX_FRAME_RELATED_P (insn) = 1; - ix86_cfa_state->reg = crtl->drap_reg; + /* Grab the argument pointer. */ + t = plus_constant (stack_pointer_rtx, m->fs.sp_offset); + insn = emit_insn (gen_rtx_SET (VOIDmode, crtl->drap_reg, t)); + RTX_FRAME_RELATED_P (insn) = 1; + m->fs.cfa_reg = crtl->drap_reg; + m->fs.cfa_offset = 0; /* Align the stack. */ - insn = emit_insn ((*ix86_gen_andsp) (stack_pointer_rtx, - stack_pointer_rtx, - GEN_INT (-align_bytes))); + insn = emit_insn (ix86_gen_andsp (stack_pointer_rtx, + stack_pointer_rtx, + GEN_INT (-align_bytes))); RTX_FRAME_RELATED_P (insn) = 1; /* Replicate the return address on the stack so that return address can be reached via (argp - 1) slot. This is needed to implement macro RETURN_ADDR_RTX and intrinsic function expand_builtin_return_addr etc. */ - x = crtl->drap_reg; - x = gen_frame_mem (Pmode, - plus_constant (x, -UNITS_PER_WORD)); - insn = emit_insn (gen_push (x)); + t = plus_constant (crtl->drap_reg, -UNITS_PER_WORD); + t = gen_frame_mem (Pmode, t); + insn = emit_insn (gen_push (t)); RTX_FRAME_RELATED_P (insn) = 1; - } - - /* Note: AT&T enter does NOT have reversed args. Enter is probably - slower on all targets. Also sdb doesn't like it. */ - - if (gen_frame_pointer) - { + + /* For the purposes of frame and register save area addressing, + we've started over with a new frame. */ + m->fs.sp_offset = INCOMING_FRAME_SP_OFFSET; + m->fs.realigned = true; + } + + if (frame_pointer_needed && !m->fs.fp_valid) + { + /* Note: AT&T enter does NOT have reversed args. Enter is probably + slower on all targets. Also sdb doesn't like it. */ insn = emit_insn (gen_push (hard_frame_pointer_rtx)); RTX_FRAME_RELATED_P (insn) = 1; - insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx); - RTX_FRAME_RELATED_P (insn) = 1; - - if (ix86_cfa_state->reg == stack_pointer_rtx) - ix86_cfa_state->reg = hard_frame_pointer_rtx; + if (m->fs.sp_offset == frame.hard_frame_pointer_offset) + { + insn = emit_move_insn (hard_frame_pointer_rtx, stack_pointer_rtx); + RTX_FRAME_RELATED_P (insn) = 1; + + if (m->fs.cfa_reg == stack_pointer_rtx) + m->fs.cfa_reg = hard_frame_pointer_rtx; + m->fs.fp_offset = m->fs.sp_offset; + m->fs.fp_valid = true; + } + } + + int_registers_saved = (frame.nregs == 0); + + if (!int_registers_saved) + { + /* If saving registers via PUSH, do so now. */ + if (!frame.save_regs_using_mov) + { + ix86_emit_save_regs (); + int_registers_saved = true; + gcc_assert (m->fs.sp_offset == frame.reg_save_offset); + } + + /* When using red zone we may start register saving before allocating + the stack frame saving one cycle of the prologue. However, avoid + doing this if we have to probe the stack; at least on x86_64 the + stack probe can turn into a call that clobbers a red zone location. */ + else if (ix86_using_red_zone () + && (! TARGET_STACK_PROBE + || frame.stack_pointer_offset < CHECK_STACK_LIMIT)) + { + ix86_emit_save_regs_using_mov (frame.reg_save_offset); + int_registers_saved = true; + } } if (stack_realign_fp) @@ -8524,113 +10499,176 @@ int align_bytes = crtl->stack_alignment_needed / BITS_PER_UNIT; gcc_assert (align_bytes > MIN_STACK_BOUNDARY / BITS_PER_UNIT); + /* The computation of the size of the re-aligned stack frame means + that we must allocate the size of the register save area before + performing the actual alignment. Otherwise we cannot guarantee + that there's enough storage above the realignment point. */ + if (m->fs.sp_offset != frame.sse_reg_save_offset) + pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, + GEN_INT (m->fs.sp_offset + - frame.sse_reg_save_offset), + -1, false); + /* Align the stack. */ - insn = emit_insn ((*ix86_gen_andsp) (stack_pointer_rtx, - stack_pointer_rtx, - GEN_INT (-align_bytes))); - RTX_FRAME_RELATED_P (insn) = 1; - } - - allocate = frame.to_allocate + frame.nsseregs * 16 + frame.padding0; - - if (!frame.save_regs_using_mov) - ix86_emit_save_regs (); - else - allocate += frame.nregs * UNITS_PER_WORD; - - /* When using red zone we may start register saving before allocating - the stack frame saving one cycle of the prologue. However I will - avoid doing this if I am going to have to probe the stack since - at least on x86_64 the stack probe can turn into a call that clobbers - a red zone location */ - if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE && frame.save_regs_using_mov - && (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT)) - ix86_emit_save_regs_using_mov ((frame_pointer_needed - && !crtl->stack_realign_needed) - ? hard_frame_pointer_rtx - : stack_pointer_rtx, - -frame.nregs * UNITS_PER_WORD); + insn = emit_insn (ix86_gen_andsp (stack_pointer_rtx, + stack_pointer_rtx, + GEN_INT (-align_bytes))); + + /* For the purposes of register save area addressing, the stack + pointer is no longer valid. As for the value of sp_offset, + see ix86_compute_frame_layout, which we need to match in order + to pass verification of stack_pointer_offset at the end. */ + m->fs.sp_offset = (m->fs.sp_offset + align_bytes) & -align_bytes; + m->fs.sp_valid = false; + } + + allocate = frame.stack_pointer_offset - m->fs.sp_offset; + + if (flag_stack_usage) + { + /* We start to count from ARG_POINTER. */ + HOST_WIDE_INT stack_size = frame.stack_pointer_offset; + + /* If it was realigned, take into account the fake frame. */ + if (stack_realign_drap) + { + if (ix86_static_chain_on_stack) + stack_size += UNITS_PER_WORD; + + if (!call_used_regs[REGNO (crtl->drap_reg)]) + stack_size += UNITS_PER_WORD; + + /* This over-estimates by 1 minimal-stack-alignment-unit but + mitigates that by counting in the new return address slot. */ + current_function_dynamic_stack_size + += crtl->stack_alignment_needed / BITS_PER_UNIT; + } + + current_function_static_stack_size = stack_size; + } + + /* The stack has already been decremented by the instruction calling us + so we need to probe unconditionally to preserve the protection area. */ + if (flag_stack_check == STATIC_BUILTIN_STACK_CHECK) + { + /* We expect the registers to be saved when probes are used. */ + gcc_assert (int_registers_saved); + + if (STACK_CHECK_MOVING_SP) + { + ix86_adjust_stack_and_probe (allocate); + allocate = 0; + } + else + { + HOST_WIDE_INT size = allocate; + + if (TARGET_64BIT && size >= (HOST_WIDE_INT) 0x80000000) + size = 0x80000000 - STACK_CHECK_PROTECT - 1; + + if (TARGET_STACK_PROBE) + ix86_emit_probe_stack_range (0, size + STACK_CHECK_PROTECT); + else + ix86_emit_probe_stack_range (STACK_CHECK_PROTECT, size); + } + } if (allocate == 0) ; - else if (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT) - pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, - GEN_INT (-allocate), -1, - ix86_cfa_state->reg == stack_pointer_rtx); - else - { - /* Only valid for Win32. */ + else if (!ix86_target_stack_probe () + || frame.stack_pointer_offset < CHECK_STACK_LIMIT) + { + pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, + GEN_INT (-allocate), -1, + m->fs.cfa_reg == stack_pointer_rtx); + } + else + { rtx eax = gen_rtx_REG (Pmode, AX_REG); - bool eax_live; - rtx t; - - gcc_assert (!TARGET_64BIT || cfun->machine->call_abi == MS_ABI); - - if (cfun->machine->call_abi == MS_ABI) - eax_live = false; - else - eax_live = ix86_eax_live_at_start_p (); + rtx r10 = NULL; + rtx (*adjust_stack_insn)(rtx, rtx, rtx); + + bool eax_live = false; + bool r10_live = false; + + if (TARGET_64BIT) + r10_live = (DECL_STATIC_CHAIN (current_function_decl) != 0); + if (!TARGET_64BIT_MS_ABI) + eax_live = ix86_eax_live_at_start_p (); if (eax_live) { emit_insn (gen_push (eax)); allocate -= UNITS_PER_WORD; } + if (r10_live) + { + r10 = gen_rtx_REG (Pmode, R10_REG); + emit_insn (gen_push (r10)); + allocate -= UNITS_PER_WORD; + } emit_move_insn (eax, GEN_INT (allocate)); - - if (TARGET_64BIT) - insn = gen_allocate_stack_worker_64 (eax, eax); - else - insn = gen_allocate_stack_worker_32 (eax, eax); - insn = emit_insn (insn); - - if (ix86_cfa_state->reg == stack_pointer_rtx) - { - ix86_cfa_state->offset += allocate; - t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, GEN_INT (-allocate)); - t = gen_rtx_SET (VOIDmode, stack_pointer_rtx, t); - add_reg_note (insn, REG_CFA_ADJUST_CFA, t); + emit_insn (ix86_gen_allocate_stack_worker (eax, eax)); + + /* Use the fact that AX still contains ALLOCATE. */ + adjust_stack_insn = (TARGET_64BIT + ? gen_pro_epilogue_adjust_stack_di_sub + : gen_pro_epilogue_adjust_stack_si_sub); + + insn = emit_insn (adjust_stack_insn (stack_pointer_rtx, + stack_pointer_rtx, eax)); + + /* Note that SEH directives need to continue tracking the stack + pointer even after the frame pointer has been set up. */ + if (m->fs.cfa_reg == stack_pointer_rtx || TARGET_SEH) + { + if (m->fs.cfa_reg == stack_pointer_rtx) + m->fs.cfa_offset += allocate; + RTX_FRAME_RELATED_P (insn) = 1; - } - - if (eax_live) - { - if (frame_pointer_needed) - t = plus_constant (hard_frame_pointer_rtx, - allocate - - frame.to_allocate - - frame.nregs * UNITS_PER_WORD); - else - t = plus_constant (stack_pointer_rtx, allocate); - emit_move_insn (eax, gen_rtx_MEM (Pmode, t)); - } - } - - if (frame.save_regs_using_mov - && !(!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE - && (! TARGET_STACK_PROBE || allocate < CHECK_STACK_LIMIT))) - { - if (!frame_pointer_needed - || !(frame.to_allocate + frame.padding0) - || crtl->stack_realign_needed) - ix86_emit_save_regs_using_mov (stack_pointer_rtx, - frame.to_allocate - + frame.nsseregs * 16 + frame.padding0); - else - ix86_emit_save_regs_using_mov (hard_frame_pointer_rtx, - -frame.nregs * UNITS_PER_WORD); - } - if (!frame_pointer_needed - || !(frame.to_allocate + frame.padding0) - || crtl->stack_realign_needed) - ix86_emit_save_sse_regs_using_mov (stack_pointer_rtx, - frame.to_allocate); - else - ix86_emit_save_sse_regs_using_mov (hard_frame_pointer_rtx, - - frame.nregs * UNITS_PER_WORD - - frame.nsseregs * 16 - - frame.padding0); + add_reg_note (insn, REG_FRAME_RELATED_EXPR, + gen_rtx_SET (VOIDmode, stack_pointer_rtx, + plus_constant (stack_pointer_rtx, + -allocate))); + } + m->fs.sp_offset += allocate; + + if (r10_live && eax_live) + { + t = choose_baseaddr (m->fs.sp_offset - allocate); + emit_move_insn (r10, gen_frame_mem (Pmode, t)); + t = choose_baseaddr (m->fs.sp_offset - allocate - UNITS_PER_WORD); + emit_move_insn (eax, gen_frame_mem (Pmode, t)); + } + else if (eax_live || r10_live) + { + t = choose_baseaddr (m->fs.sp_offset - allocate); + emit_move_insn ((eax_live ? eax : r10), gen_frame_mem (Pmode, t)); + } + } + gcc_assert (m->fs.sp_offset == frame.stack_pointer_offset); + + /* If we havn't already set up the frame pointer, do so now. */ + if (frame_pointer_needed && !m->fs.fp_valid) + { + insn = ix86_gen_add3 (hard_frame_pointer_rtx, stack_pointer_rtx, + GEN_INT (frame.stack_pointer_offset + - frame.hard_frame_pointer_offset)); + insn = emit_insn (insn); + RTX_FRAME_RELATED_P (insn) = 1; + add_reg_note (insn, REG_CFA_ADJUST_CFA, NULL); + + if (m->fs.cfa_reg == stack_pointer_rtx) + m->fs.cfa_reg = hard_frame_pointer_rtx; + m->fs.fp_offset = frame.hard_frame_pointer_offset; + m->fs.fp_valid = true; + } + + if (!int_registers_saved) + ix86_emit_save_regs_using_mov (frame.reg_save_offset); + if (frame.nsseregs) + ix86_emit_save_sse_regs_using_mov (frame.sse_reg_save_offset); pic_reg_used = false; if (pic_offset_table_rtx @@ -8672,7 +10710,7 @@ when mcount needs it. Blockage to avoid call movement across mcount call is emitted in generic code after the NOTE_INSN_PROLOGUE_END note. */ - if (crtl->profile && pic_reg_used) + if (crtl->profile && !flag_fentry && pic_reg_used) emit_insn (gen_prologue_use (pic_offset_table_rtx)); if (crtl->drap_reg && !crtl->stack_realign_needed) @@ -8680,13 +10718,8 @@ /* vDRAP is setup but after reload it turns out stack realign isn't necessary, here we will emit prologue to setup DRAP without stack realign adjustment */ - rtx x; - int drap_bp_offset = UNITS_PER_WORD * 2; - - if (ix86_static_chain_on_stack) - drap_bp_offset += UNITS_PER_WORD; - x = plus_constant (hard_frame_pointer_rtx, drap_bp_offset); - insn = emit_insn (gen_rtx_SET (VOIDmode, crtl->drap_reg, x)); + t = choose_baseaddr (0); + emit_insn (gen_rtx_SET (VOIDmode, crtl->drap_reg, t)); } /* Prevent instructions from being scheduled into register save push @@ -8701,16 +10734,25 @@ /* Emit cld instruction if stringops are used in the function. */ if (TARGET_CLD && ix86_current_function_needs_cld) emit_insn (gen_cld ()); + + /* SEH requires that the prologue end within 256 bytes of the start of + the function. Prevent instruction schedules that would extend that. */ + if (TARGET_SEH) + emit_insn (gen_blockage ()); } /* Emit code to restore REG using a POP insn. */ static void -ix86_emit_restore_reg_using_pop (rtx reg, HOST_WIDE_INT red_offset) -{ - rtx insn = emit_insn (ix86_gen_pop1 (reg)); - - if (ix86_cfa_state->reg == crtl->drap_reg +ix86_emit_restore_reg_using_pop (rtx reg) +{ + struct machine_function *m = cfun->machine; + rtx insn = emit_insn (gen_pop (reg)); + + ix86_add_cfa_restore_note (insn, reg, m->fs.sp_offset); + m->fs.sp_offset -= UNITS_PER_WORD; + + if (m->fs.cfa_reg == crtl->drap_reg && REGNO (reg) == REGNO (crtl->drap_reg)) { /* Previously we'd represented the CFA as an expression @@ -8720,15 +10762,20 @@ the stack pointer. */ add_reg_note (insn, REG_CFA_DEF_CFA, reg); RTX_FRAME_RELATED_P (insn) = 1; + + /* This means that the DRAP register is valid for addressing too. */ + m->fs.drap_valid = true; return; } - if (ix86_cfa_state->reg == stack_pointer_rtx) - { - ix86_cfa_state->offset -= UNITS_PER_WORD; - add_reg_note (insn, REG_CFA_ADJUST_CFA, - copy_rtx (XVECEXP (PATTERN (insn), 0, 1))); + if (m->fs.cfa_reg == stack_pointer_rtx) + { + rtx x = plus_constant (stack_pointer_rtx, UNITS_PER_WORD); + x = gen_rtx_SET (VOIDmode, stack_pointer_rtx, x); + add_reg_note (insn, REG_CFA_ADJUST_CFA, x); RTX_FRAME_RELATED_P (insn) = 1; + + m->fs.cfa_offset -= UNITS_PER_WORD; } /* When the frame pointer is the CFA, and we pop it, we are @@ -8736,92 +10783,82 @@ for stack frames that don't allocate other data, so we assume the stack pointer is now pointing at the return address, i.e. the function entry state, which makes the offset be 1 word. */ - else if (ix86_cfa_state->reg == hard_frame_pointer_rtx - && reg == hard_frame_pointer_rtx) - { - ix86_cfa_state->reg = stack_pointer_rtx; - ix86_cfa_state->offset -= UNITS_PER_WORD; - - add_reg_note (insn, REG_CFA_DEF_CFA, - gen_rtx_PLUS (Pmode, stack_pointer_rtx, - GEN_INT (ix86_cfa_state->offset))); - RTX_FRAME_RELATED_P (insn) = 1; - } - - ix86_add_cfa_restore_note (insn, reg, red_offset); + if (reg == hard_frame_pointer_rtx) + { + m->fs.fp_valid = false; + if (m->fs.cfa_reg == hard_frame_pointer_rtx) + { + m->fs.cfa_reg = stack_pointer_rtx; + m->fs.cfa_offset -= UNITS_PER_WORD; + + add_reg_note (insn, REG_CFA_DEF_CFA, + gen_rtx_PLUS (Pmode, stack_pointer_rtx, + GEN_INT (m->fs.cfa_offset))); + RTX_FRAME_RELATED_P (insn) = 1; + } + } } /* Emit code to restore saved registers using POP insns. */ static void -ix86_emit_restore_regs_using_pop (HOST_WIDE_INT red_offset) -{ - int regno; +ix86_emit_restore_regs_using_pop (void) +{ + unsigned int regno; for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++) if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, false)) - { - ix86_emit_restore_reg_using_pop (gen_rtx_REG (Pmode, regno), - red_offset); - red_offset += UNITS_PER_WORD; - } + ix86_emit_restore_reg_using_pop (gen_rtx_REG (Pmode, regno)); } /* Emit code and notes for the LEAVE instruction. */ static void -ix86_emit_leave (HOST_WIDE_INT red_offset) -{ +ix86_emit_leave (void) +{ + struct machine_function *m = cfun->machine; rtx insn = emit_insn (ix86_gen_leave ()); ix86_add_queued_cfa_restore_notes (insn); - if (ix86_cfa_state->reg == hard_frame_pointer_rtx) - { - ix86_cfa_state->reg = stack_pointer_rtx; - ix86_cfa_state->offset -= UNITS_PER_WORD; - - add_reg_note (insn, REG_CFA_ADJUST_CFA, - copy_rtx (XVECEXP (PATTERN (insn), 0, 0))); + gcc_assert (m->fs.fp_valid); + m->fs.sp_valid = true; + m->fs.sp_offset = m->fs.fp_offset - UNITS_PER_WORD; + m->fs.fp_valid = false; + + if (m->fs.cfa_reg == hard_frame_pointer_rtx) + { + m->fs.cfa_reg = stack_pointer_rtx; + m->fs.cfa_offset = m->fs.sp_offset; + + add_reg_note (insn, REG_CFA_DEF_CFA, + plus_constant (stack_pointer_rtx, m->fs.sp_offset)); RTX_FRAME_RELATED_P (insn) = 1; - ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx, red_offset); - } -} - -/* Emit code to restore saved registers using MOV insns. First register - is restored from POINTER + OFFSET. */ -static void -ix86_emit_restore_regs_using_mov (rtx pointer, HOST_WIDE_INT offset, - HOST_WIDE_INT red_offset, + ix86_add_cfa_restore_note (insn, hard_frame_pointer_rtx, + m->fs.fp_offset); + } +} + +/* Emit code to restore saved registers using MOV insns. + First register is restored from CFA - CFA_OFFSET. */ +static void +ix86_emit_restore_regs_using_mov (HOST_WIDE_INT cfa_offset, int maybe_eh_return) { + struct machine_function *m = cfun->machine; unsigned int regno; - rtx base_address = gen_rtx_MEM (Pmode, pointer); - rtx insn; for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++) if (!SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return)) { rtx reg = gen_rtx_REG (Pmode, regno); - - /* Ensure that adjust_address won't be forced to produce pointer - out of range allowed by x86-64 instruction set. */ - if (TARGET_64BIT && offset != trunc_int_for_mode (offset, SImode)) - { - rtx r11; - - r11 = gen_rtx_REG (DImode, R11_REG); - emit_move_insn (r11, GEN_INT (offset)); - emit_insn (gen_adddi3 (r11, r11, pointer)); - base_address = gen_rtx_MEM (Pmode, r11); - offset = 0; - } - insn = emit_move_insn (reg, - adjust_address (base_address, Pmode, offset)); - offset += UNITS_PER_WORD; - - if (ix86_cfa_state->reg == crtl->drap_reg - && regno == REGNO (crtl->drap_reg)) + rtx insn, mem; + + mem = choose_baseaddr (cfa_offset); + mem = gen_frame_mem (Pmode, mem); + insn = emit_move_insn (reg, mem); + + if (m->fs.cfa_reg == crtl->drap_reg && regno == REGNO (crtl->drap_reg)) { /* Previously we'd represented the CFA as an expression like *(%ebp - 8). We've just popped that value from @@ -8830,50 +10867,39 @@ the stack pointer. */ add_reg_note (insn, REG_CFA_DEF_CFA, reg); RTX_FRAME_RELATED_P (insn) = 1; + + /* This means that the DRAP register is valid for addressing. */ + m->fs.drap_valid = true; } else - ix86_add_cfa_restore_note (NULL_RTX, reg, red_offset); - - red_offset += UNITS_PER_WORD; + ix86_add_cfa_restore_note (NULL_RTX, reg, cfa_offset); + + cfa_offset -= UNITS_PER_WORD; } } -/* Emit code to restore saved registers using MOV insns. First register - is restored from POINTER + OFFSET. */ -static void -ix86_emit_restore_sse_regs_using_mov (rtx pointer, HOST_WIDE_INT offset, - HOST_WIDE_INT red_offset, +/* Emit code to restore saved registers using MOV insns. + First register is restored from CFA - CFA_OFFSET. */ +static void +ix86_emit_restore_sse_regs_using_mov (HOST_WIDE_INT cfa_offset, int maybe_eh_return) { - int regno; - rtx base_address = gen_rtx_MEM (TImode, pointer); - rtx mem; + unsigned int regno; for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++) if (SSE_REGNO_P (regno) && ix86_save_reg (regno, maybe_eh_return)) { - rtx reg = gen_rtx_REG (TImode, regno); - - /* Ensure that adjust_address won't be forced to produce pointer - out of range allowed by x86-64 instruction set. */ - if (TARGET_64BIT && offset != trunc_int_for_mode (offset, SImode)) - { - rtx r11; - - r11 = gen_rtx_REG (DImode, R11_REG); - emit_move_insn (r11, GEN_INT (offset)); - emit_insn (gen_adddi3 (r11, r11, pointer)); - base_address = gen_rtx_MEM (TImode, r11); - offset = 0; - } - mem = adjust_address (base_address, TImode, offset); + rtx reg = gen_rtx_REG (V4SFmode, regno); + rtx mem; + + mem = choose_baseaddr (cfa_offset); + mem = gen_rtx_MEM (V4SFmode, mem); set_mem_align (mem, 128); emit_move_insn (reg, mem); - offset += 16; - - ix86_add_cfa_restore_note (NULL_RTX, reg, red_offset); - - red_offset += 16; + + ix86_add_cfa_restore_note (NULL_RTX, reg, cfa_offset); + + cfa_offset -= 16; } } @@ -8882,134 +10908,137 @@ void ix86_expand_epilogue (int style) { - int sp_valid; + struct machine_function *m = cfun->machine; + struct machine_frame_state frame_state_save = m->fs; struct ix86_frame frame; - HOST_WIDE_INT offset, red_offset; - struct machine_cfa_state cfa_state_save = *ix86_cfa_state; + bool restore_regs_via_mov; bool using_drap; ix86_finalize_stack_realign_flags (); - - /* When stack is realigned, SP must be valid. */ - sp_valid = (!frame_pointer_needed - || current_function_sp_is_unchanging - || stack_realign_fp); - ix86_compute_frame_layout (&frame); + m->fs.sp_valid = (!frame_pointer_needed + || (current_function_sp_is_unchanging + && !stack_realign_fp)); + gcc_assert (!m->fs.sp_valid + || m->fs.sp_offset == frame.stack_pointer_offset); + + /* The FP must be valid if the frame pointer is present. */ + gcc_assert (frame_pointer_needed == m->fs.fp_valid); + gcc_assert (!m->fs.fp_valid + || m->fs.fp_offset == frame.hard_frame_pointer_offset); + + /* We must have *some* valid pointer to the stack frame. */ + gcc_assert (m->fs.sp_valid || m->fs.fp_valid); + + /* The DRAP is never valid at this point. */ + gcc_assert (!m->fs.drap_valid); + /* See the comment about red zone and frame pointer usage in ix86_expand_prologue. */ if (frame_pointer_needed && frame.red_zone_size) - emit_insn (gen_memory_blockage ()); + emit_insn (gen_memory_blockage ()); using_drap = crtl->drap_reg && crtl->stack_realign_needed; - gcc_assert (!using_drap || ix86_cfa_state->reg == crtl->drap_reg); - - /* Calculate start of saved registers relative to ebp. Special care - must be taken for the normal return case of a function using - eh_return: the eax and edx registers are marked as saved, but not - restored along this path. */ - offset = frame.nregs; + gcc_assert (!using_drap || m->fs.cfa_reg == crtl->drap_reg); + + /* Determine the CFA offset of the end of the red-zone. */ + m->fs.red_zone_offset = 0; + if (ix86_using_red_zone () && crtl->args.pops_args < 65536) + { + /* The red-zone begins below the return address. */ + m->fs.red_zone_offset = RED_ZONE_SIZE + UNITS_PER_WORD; + + /* When the register save area is in the aligned portion of + the stack, determine the maximum runtime displacement that + matches up with the aligned frame. */ + if (stack_realign_drap) + m->fs.red_zone_offset -= (crtl->stack_alignment_needed / BITS_PER_UNIT + + UNITS_PER_WORD); + } + + /* Special care must be taken for the normal return case of a function + using eh_return: the eax and edx registers are marked as saved, but + not restored along this path. Adjust the save location to match. */ if (crtl->calls_eh_return && style != 2) - offset -= 2; - offset *= -UNITS_PER_WORD; - offset -= frame.nsseregs * 16 + frame.padding0; - - /* Calculate start of saved registers relative to esp on entry of the - function. When realigning stack, this needs to be the most negative - value possible at runtime. */ - red_offset = offset; - if (using_drap) - red_offset -= crtl->stack_alignment_needed / BITS_PER_UNIT - + UNITS_PER_WORD; - else if (stack_realign_fp) - red_offset -= crtl->stack_alignment_needed / BITS_PER_UNIT - - UNITS_PER_WORD; - if (ix86_static_chain_on_stack) - red_offset -= UNITS_PER_WORD; - if (frame_pointer_needed) - red_offset -= UNITS_PER_WORD; - + frame.reg_save_offset -= 2 * UNITS_PER_WORD; + + /* EH_RETURN requires the use of moves to function properly. */ + if (crtl->calls_eh_return) + restore_regs_via_mov = true; + /* SEH requires the use of pops to identify the epilogue. */ + else if (TARGET_SEH) + restore_regs_via_mov = false; /* If we're only restoring one register and sp is not valid then using a move instruction to restore the register since it's - less work than reloading sp and popping the register. - - The default code result in stack adjustment using add/lea instruction, - while this code results in LEAVE instruction (or discrete equivalent), - so it is profitable in some other cases as well. Especially when there - are no registers to restore. We also use this code when TARGET_USE_LEAVE - and there is exactly one register to pop. This heuristic may need some - tuning in future. */ - if ((!sp_valid && (frame.nregs + frame.nsseregs) <= 1) - || (TARGET_EPILOGUE_USING_MOVE - && cfun->machine->use_fast_prologue_epilogue - && ((frame.nregs + frame.nsseregs) > 1 - || (frame.to_allocate + frame.padding0) != 0)) - || (frame_pointer_needed && !(frame.nregs + frame.nsseregs) - && (frame.to_allocate + frame.padding0) != 0) - || (frame_pointer_needed && TARGET_USE_LEAVE - && cfun->machine->use_fast_prologue_epilogue - && (frame.nregs + frame.nsseregs) == 1) - || crtl->calls_eh_return) - { - /* Restore registers. We can use ebp or esp to address the memory - locations. If both are available, default to ebp, since offsets - are known to be small. Only exception is esp pointing directly - to the end of block of saved registers, where we may simplify - addressing mode. - - If we are realigning stack with bp and sp, regs restore can't - be addressed by bp. sp must be used instead. */ - - if (!frame_pointer_needed - || (sp_valid && !(frame.to_allocate + frame.padding0)) - || stack_realign_fp) - { - ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx, - frame.to_allocate, red_offset, - style == 2); - ix86_emit_restore_regs_using_mov (stack_pointer_rtx, - frame.to_allocate - + frame.nsseregs * 16 - + frame.padding0, - red_offset - + frame.nsseregs * 16 - + frame.padding0, style == 2); - } - else - { - ix86_emit_restore_sse_regs_using_mov (hard_frame_pointer_rtx, - offset, red_offset, - style == 2); - ix86_emit_restore_regs_using_mov (hard_frame_pointer_rtx, - offset - + frame.nsseregs * 16 - + frame.padding0, - red_offset - + frame.nsseregs * 16 - + frame.padding0, style == 2); - } - - red_offset -= offset; + less work than reloading sp and popping the register. */ + else if (!m->fs.sp_valid && frame.nregs <= 1) + restore_regs_via_mov = true; + else if (TARGET_EPILOGUE_USING_MOVE + && cfun->machine->use_fast_prologue_epilogue + && (frame.nregs > 1 + || m->fs.sp_offset != frame.reg_save_offset)) + restore_regs_via_mov = true; + else if (frame_pointer_needed + && !frame.nregs + && m->fs.sp_offset != frame.reg_save_offset) + restore_regs_via_mov = true; + else if (frame_pointer_needed + && TARGET_USE_LEAVE + && cfun->machine->use_fast_prologue_epilogue + && frame.nregs == 1) + restore_regs_via_mov = true; + else + restore_regs_via_mov = false; + + if (restore_regs_via_mov || frame.nsseregs) + { + /* Ensure that the entire register save area is addressable via + the stack pointer, if we will restore via sp. */ + if (TARGET_64BIT + && m->fs.sp_offset > 0x7fffffff + && !(m->fs.fp_valid || m->fs.drap_valid) + && (frame.nsseregs + frame.nregs) != 0) + { + pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, + GEN_INT (m->fs.sp_offset + - frame.sse_reg_save_offset), + style, + m->fs.cfa_reg == stack_pointer_rtx); + } + } + + /* If there are any SSE registers to restore, then we have to do it + via moves, since there's obviously no pop for SSE regs. */ + if (frame.nsseregs) + ix86_emit_restore_sse_regs_using_mov (frame.sse_reg_save_offset, + style == 2); + + if (restore_regs_via_mov) + { + rtx t; + + if (frame.nregs) + ix86_emit_restore_regs_using_mov (frame.reg_save_offset, style == 2); /* eh_return epilogues need %ecx added to the stack pointer. */ if (style == 2) { - rtx tmp, sa = EH_RETURN_STACKADJ_RTX; + rtx insn, sa = EH_RETURN_STACKADJ_RTX; /* Stack align doesn't work with eh_return. */ - gcc_assert (!crtl->stack_realign_needed); + gcc_assert (!stack_realign_drap); /* Neither does regparm nested functions. */ gcc_assert (!ix86_static_chain_on_stack); if (frame_pointer_needed) { - tmp = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa); - tmp = plus_constant (tmp, UNITS_PER_WORD); - tmp = emit_insn (gen_rtx_SET (VOIDmode, sa, tmp)); - - tmp = gen_rtx_MEM (Pmode, hard_frame_pointer_rtx); - tmp = emit_move_insn (hard_frame_pointer_rtx, tmp); + t = gen_rtx_PLUS (Pmode, hard_frame_pointer_rtx, sa); + t = plus_constant (t, m->fs.fp_offset - UNITS_PER_WORD); + emit_insn (gen_rtx_SET (VOIDmode, sa, t)); + + t = gen_frame_mem (Pmode, hard_frame_pointer_rtx); + insn = emit_move_insn (hard_frame_pointer_rtx, t); /* Note that we use SA as a temporary CFA, as the return address is at the proper place relative to it. We @@ -9019,117 +11048,100 @@ other reasonable register to use for the CFA. We don't bother resetting the CFA to the SP for the duration of the return insn. */ - add_reg_note (tmp, REG_CFA_DEF_CFA, + add_reg_note (insn, REG_CFA_DEF_CFA, plus_constant (sa, UNITS_PER_WORD)); - ix86_add_queued_cfa_restore_notes (tmp); - add_reg_note (tmp, REG_CFA_RESTORE, hard_frame_pointer_rtx); - RTX_FRAME_RELATED_P (tmp) = 1; - ix86_cfa_state->reg = sa; - ix86_cfa_state->offset = UNITS_PER_WORD; + ix86_add_queued_cfa_restore_notes (insn); + add_reg_note (insn, REG_CFA_RESTORE, hard_frame_pointer_rtx); + RTX_FRAME_RELATED_P (insn) = 1; + + m->fs.cfa_reg = sa; + m->fs.cfa_offset = UNITS_PER_WORD; + m->fs.fp_valid = false; pro_epilogue_adjust_stack (stack_pointer_rtx, sa, const0_rtx, style, false); } else { - tmp = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa); - tmp = plus_constant (tmp, (frame.to_allocate - + frame.nregs * UNITS_PER_WORD - + frame.nsseregs * 16 - + frame.padding0)); - tmp = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, tmp)); - ix86_add_queued_cfa_restore_notes (tmp); - - gcc_assert (ix86_cfa_state->reg == stack_pointer_rtx); - if (ix86_cfa_state->offset != UNITS_PER_WORD) - { - ix86_cfa_state->offset = UNITS_PER_WORD; - add_reg_note (tmp, REG_CFA_DEF_CFA, + t = gen_rtx_PLUS (Pmode, stack_pointer_rtx, sa); + t = plus_constant (t, m->fs.sp_offset - UNITS_PER_WORD); + insn = emit_insn (gen_rtx_SET (VOIDmode, stack_pointer_rtx, t)); + ix86_add_queued_cfa_restore_notes (insn); + + gcc_assert (m->fs.cfa_reg == stack_pointer_rtx); + if (m->fs.cfa_offset != UNITS_PER_WORD) + { + m->fs.cfa_offset = UNITS_PER_WORD; + add_reg_note (insn, REG_CFA_DEF_CFA, plus_constant (stack_pointer_rtx, UNITS_PER_WORD)); - RTX_FRAME_RELATED_P (tmp) = 1; - } - } - } - else if (!frame_pointer_needed) - pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, - GEN_INT (frame.to_allocate - + frame.nregs * UNITS_PER_WORD - + frame.nsseregs * 16 - + frame.padding0), - style, !using_drap); - /* If not an i386, mov & pop is faster than "leave". */ - else if (TARGET_USE_LEAVE || optimize_function_for_size_p (cfun) + RTX_FRAME_RELATED_P (insn) = 1; + } + } + m->fs.sp_offset = UNITS_PER_WORD; + m->fs.sp_valid = true; + } + } + else + { + /* SEH requires that the function end with (1) a stack adjustment + if necessary, (2) a sequence of pops, and (3) a return or + jump instruction. Prevent insns from the function body from + being scheduled into this sequence. */ + if (TARGET_SEH) + { + /* Prevent a catch region from being adjacent to the standard + epilogue sequence. Unfortuantely crtl->uses_eh_lsda nor + several other flags that would be interesting to test are + not yet set up. */ + if (flag_non_call_exceptions) + emit_insn (gen_nops (const1_rtx)); + else + emit_insn (gen_blockage ()); + } + + /* First step is to deallocate the stack frame so that we can + pop the registers. */ + if (!m->fs.sp_valid) + { + pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx, + GEN_INT (m->fs.fp_offset + - frame.reg_save_offset), + style, false); + } + else if (m->fs.sp_offset != frame.reg_save_offset) + { + pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, + GEN_INT (m->fs.sp_offset + - frame.reg_save_offset), + style, + m->fs.cfa_reg == stack_pointer_rtx); + } + + ix86_emit_restore_regs_using_pop (); + } + + /* If we used a stack pointer and haven't already got rid of it, + then do so now. */ + if (m->fs.fp_valid) + { + /* If the stack pointer is valid and pointing at the frame + pointer store address, then we only need a pop. */ + if (m->fs.sp_valid && m->fs.sp_offset == frame.hfp_save_offset) + ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx); + /* Leave results in shorter dependency chains on CPUs that are + able to grok it fast. */ + else if (TARGET_USE_LEAVE + || optimize_function_for_size_p (cfun) || !cfun->machine->use_fast_prologue_epilogue) - ix86_emit_leave (red_offset); - else - { + ix86_emit_leave (); + else + { pro_epilogue_adjust_stack (stack_pointer_rtx, hard_frame_pointer_rtx, const0_rtx, style, !using_drap); - - ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx, red_offset); - } - } - else - { - /* First step is to deallocate the stack frame so that we can - pop the registers. - - If we realign stack with frame pointer, then stack pointer - won't be able to recover via lea $offset(%bp), %sp, because - there is a padding area between bp and sp for realign. - "add $to_allocate, %sp" must be used instead. */ - if (!sp_valid) - { - gcc_assert (frame_pointer_needed); - gcc_assert (!stack_realign_fp); - pro_epilogue_adjust_stack (stack_pointer_rtx, - hard_frame_pointer_rtx, - GEN_INT (offset), style, false); - ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx, - 0, red_offset, - style == 2); - pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, - GEN_INT (frame.nsseregs * 16 - + frame.padding0), - style, false); - } - else if (frame.to_allocate || frame.padding0 || frame.nsseregs) - { - ix86_emit_restore_sse_regs_using_mov (stack_pointer_rtx, - frame.to_allocate, red_offset, - style == 2); - pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, - GEN_INT (frame.to_allocate - + frame.nsseregs * 16 - + frame.padding0), style, - !using_drap && !frame_pointer_needed); - } - - ix86_emit_restore_regs_using_pop (red_offset + frame.nsseregs * 16 - + frame.padding0); - red_offset -= offset; - - if (frame_pointer_needed) - { - /* Leave results in shorter dependency chains on CPUs that are - able to grok it fast. */ - if (TARGET_USE_LEAVE) - ix86_emit_leave (red_offset); - else - { - /* For stack realigned really happens, recover stack - pointer to hard frame pointer is a must, if not using - leave. */ - if (stack_realign_fp) - pro_epilogue_adjust_stack (stack_pointer_rtx, - hard_frame_pointer_rtx, - const0_rtx, style, !using_drap); - ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx, - red_offset); - } - } + ix86_emit_restore_reg_using_pop (hard_frame_pointer_rtx); + } } if (using_drap) @@ -9144,47 +11156,53 @@ if (!call_used_regs[REGNO (crtl->drap_reg)]) param_ptr_offset += UNITS_PER_WORD; - insn = emit_insn ((*ix86_gen_add3) (stack_pointer_rtx, - crtl->drap_reg, - GEN_INT (-param_ptr_offset))); - - ix86_cfa_state->reg = stack_pointer_rtx; - ix86_cfa_state->offset = param_ptr_offset; + insn = emit_insn (gen_rtx_SET + (VOIDmode, stack_pointer_rtx, + gen_rtx_PLUS (Pmode, + crtl->drap_reg, + GEN_INT (-param_ptr_offset)))); + m->fs.cfa_reg = stack_pointer_rtx; + m->fs.cfa_offset = param_ptr_offset; + m->fs.sp_offset = param_ptr_offset; + m->fs.realigned = false; add_reg_note (insn, REG_CFA_DEF_CFA, - gen_rtx_PLUS (Pmode, ix86_cfa_state->reg, - GEN_INT (ix86_cfa_state->offset))); + gen_rtx_PLUS (Pmode, stack_pointer_rtx, + GEN_INT (param_ptr_offset))); RTX_FRAME_RELATED_P (insn) = 1; if (!call_used_regs[REGNO (crtl->drap_reg)]) - ix86_emit_restore_reg_using_pop (crtl->drap_reg, -UNITS_PER_WORD); - } - - /* Remove the saved static chain from the stack. The use of ECX is - merely as a scratch register, not as the actual static chain. */ - if (ix86_static_chain_on_stack) - { - rtx r, insn; - - gcc_assert (ix86_cfa_state->reg == stack_pointer_rtx); - ix86_cfa_state->offset += UNITS_PER_WORD; - - r = gen_rtx_REG (Pmode, CX_REG); - insn = emit_insn (ix86_gen_pop1 (r)); - - r = plus_constant (stack_pointer_rtx, UNITS_PER_WORD); - r = gen_rtx_SET (VOIDmode, stack_pointer_rtx, r); - add_reg_note (insn, REG_CFA_ADJUST_CFA, r); - RTX_FRAME_RELATED_P (insn) = 1; + ix86_emit_restore_reg_using_pop (crtl->drap_reg); + } + + /* At this point the stack pointer must be valid, and we must have + restored all of the registers. We may not have deallocated the + entire stack frame. We've delayed this until now because it may + be possible to merge the local stack deallocation with the + deallocation forced by ix86_static_chain_on_stack. */ + gcc_assert (m->fs.sp_valid); + gcc_assert (!m->fs.fp_valid); + gcc_assert (!m->fs.realigned); + if (m->fs.sp_offset != UNITS_PER_WORD) + { + pro_epilogue_adjust_stack (stack_pointer_rtx, stack_pointer_rtx, + GEN_INT (m->fs.sp_offset - UNITS_PER_WORD), + style, true); } /* Sibcall epilogues don't want a return instruction. */ if (style == 0) { - *ix86_cfa_state = cfa_state_save; + m->fs = frame_state_save; return; } + /* Emit vzeroupper if needed. */ + if (TARGET_VZEROUPPER + && !TREE_THIS_VOLATILE (cfun->decl) + && !cfun->machine->caller_return_avx256_p) + emit_insn (gen_avx_vzeroupper (GEN_INT (call_no_avx256))); + if (crtl->args.pops_args && crtl->args.size) { rtx popc = GEN_INT (crtl->args.pops_args); @@ -9200,8 +11218,9 @@ /* There is no "pascal" calling convention in any 64bit ABI. */ gcc_assert (!TARGET_64BIT); - insn = emit_insn (gen_popsi1 (ecx)); - ix86_cfa_state->offset -= UNITS_PER_WORD; + insn = emit_insn (gen_pop (ecx)); + m->fs.cfa_offset -= UNITS_PER_WORD; + m->fs.sp_offset -= UNITS_PER_WORD; add_reg_note (insn, REG_CFA_ADJUST_CFA, copy_rtx (XVECEXP (PATTERN (insn), 0, 1))); @@ -9221,7 +11240,7 @@ /* Restore the state back to the state from the prologue, so that it's correct for the next epilogue. */ - *ix86_cfa_state = cfa_state_save; + m->fs = frame_state_save; } /* Reset from the function's potential modifications. */ @@ -9250,6 +11269,332 @@ #endif } + +/* Return a scratch register to use in the split stack prologue. The + split stack prologue is used for -fsplit-stack. It is the first + instructions in the function, even before the regular prologue. + The scratch register can be any caller-saved register which is not + used for parameters or for the static chain. */ + +static unsigned int +split_stack_prologue_scratch_regno (void) +{ + if (TARGET_64BIT) + return R11_REG; + else + { + bool is_fastcall; + int regparm; + + is_fastcall = (lookup_attribute ("fastcall", + TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl))) + != NULL); + regparm = ix86_function_regparm (TREE_TYPE (cfun->decl), cfun->decl); + + if (is_fastcall) + { + if (DECL_STATIC_CHAIN (cfun->decl)) + { + sorry ("-fsplit-stack does not support fastcall with " + "nested function"); + return INVALID_REGNUM; + } + return AX_REG; + } + else if (regparm < 3) + { + if (!DECL_STATIC_CHAIN (cfun->decl)) + return CX_REG; + else + { + if (regparm >= 2) + { + sorry ("-fsplit-stack does not support 2 register " + " parameters for a nested function"); + return INVALID_REGNUM; + } + return DX_REG; + } + } + else + { + /* FIXME: We could make this work by pushing a register + around the addition and comparison. */ + sorry ("-fsplit-stack does not support 3 register parameters"); + return INVALID_REGNUM; + } + } +} + +/* A SYMBOL_REF for the function which allocates new stackspace for + -fsplit-stack. */ + +static GTY(()) rtx split_stack_fn; + +/* A SYMBOL_REF for the more stack function when using the large + model. */ + +static GTY(()) rtx split_stack_fn_large; + +/* Handle -fsplit-stack. These are the first instructions in the + function, even before the regular prologue. */ + +void +ix86_expand_split_stack_prologue (void) +{ + struct ix86_frame frame; + HOST_WIDE_INT allocate; + unsigned HOST_WIDE_INT args_size; + rtx label, limit, current, jump_insn, allocate_rtx, call_insn, call_fusage; + rtx scratch_reg = NULL_RTX; + rtx varargs_label = NULL_RTX; + rtx fn; + + gcc_assert (flag_split_stack && reload_completed); + + ix86_finalize_stack_realign_flags (); + ix86_compute_frame_layout (&frame); + allocate = frame.stack_pointer_offset - INCOMING_FRAME_SP_OFFSET; + + /* This is the label we will branch to if we have enough stack + space. We expect the basic block reordering pass to reverse this + branch if optimizing, so that we branch in the unlikely case. */ + label = gen_label_rtx (); + + /* We need to compare the stack pointer minus the frame size with + the stack boundary in the TCB. The stack boundary always gives + us SPLIT_STACK_AVAILABLE bytes, so if we need less than that we + can compare directly. Otherwise we need to do an addition. */ + + limit = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx), + UNSPEC_STACK_CHECK); + limit = gen_rtx_CONST (Pmode, limit); + limit = gen_rtx_MEM (Pmode, limit); + if (allocate < SPLIT_STACK_AVAILABLE) + current = stack_pointer_rtx; + else + { + unsigned int scratch_regno; + rtx offset; + + /* We need a scratch register to hold the stack pointer minus + the required frame size. Since this is the very start of the + function, the scratch register can be any caller-saved + register which is not used for parameters. */ + offset = GEN_INT (- allocate); + scratch_regno = split_stack_prologue_scratch_regno (); + if (scratch_regno == INVALID_REGNUM) + return; + scratch_reg = gen_rtx_REG (Pmode, scratch_regno); + if (!TARGET_64BIT || x86_64_immediate_operand (offset, Pmode)) + { + /* We don't use ix86_gen_add3 in this case because it will + want to split to lea, but when not optimizing the insn + will not be split after this point. */ + emit_insn (gen_rtx_SET (VOIDmode, scratch_reg, + gen_rtx_PLUS (Pmode, stack_pointer_rtx, + offset))); + } + else + { + emit_move_insn (scratch_reg, offset); + emit_insn (gen_adddi3 (scratch_reg, scratch_reg, + stack_pointer_rtx)); + } + current = scratch_reg; + } + + ix86_expand_branch (GEU, current, limit, label); + jump_insn = get_last_insn (); + JUMP_LABEL (jump_insn) = label; + + /* Mark the jump as very likely to be taken. */ + add_reg_note (jump_insn, REG_BR_PROB, + GEN_INT (REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100)); + + if (split_stack_fn == NULL_RTX) + split_stack_fn = gen_rtx_SYMBOL_REF (Pmode, "__morestack"); + fn = split_stack_fn; + + /* Get more stack space. We pass in the desired stack space and the + size of the arguments to copy to the new stack. In 32-bit mode + we push the parameters; __morestack will return on a new stack + anyhow. In 64-bit mode we pass the parameters in r10 and + r11. */ + allocate_rtx = GEN_INT (allocate); + args_size = crtl->args.size >= 0 ? crtl->args.size : 0; + call_fusage = NULL_RTX; + if (TARGET_64BIT) + { + rtx reg10, reg11; + + reg10 = gen_rtx_REG (Pmode, R10_REG); + reg11 = gen_rtx_REG (Pmode, R11_REG); + + /* If this function uses a static chain, it will be in %r10. + Preserve it across the call to __morestack. */ + if (DECL_STATIC_CHAIN (cfun->decl)) + { + rtx rax; + + rax = gen_rtx_REG (Pmode, AX_REG); + emit_move_insn (rax, reg10); + use_reg (&call_fusage, rax); + } + + if (ix86_cmodel == CM_LARGE || ix86_cmodel == CM_LARGE_PIC) + { + HOST_WIDE_INT argval; + + /* When using the large model we need to load the address + into a register, and we've run out of registers. So we + switch to a different calling convention, and we call a + different function: __morestack_large. We pass the + argument size in the upper 32 bits of r10 and pass the + frame size in the lower 32 bits. */ + gcc_assert ((allocate & (HOST_WIDE_INT) 0xffffffff) == allocate); + gcc_assert ((args_size & 0xffffffff) == args_size); + + if (split_stack_fn_large == NULL_RTX) + split_stack_fn_large = + gen_rtx_SYMBOL_REF (Pmode, "__morestack_large_model"); + + if (ix86_cmodel == CM_LARGE_PIC) + { + rtx label, x; + + label = gen_label_rtx (); + emit_label (label); + LABEL_PRESERVE_P (label) = 1; + emit_insn (gen_set_rip_rex64 (reg10, label)); + emit_insn (gen_set_got_offset_rex64 (reg11, label)); + emit_insn (gen_adddi3 (reg10, reg10, reg11)); + x = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, split_stack_fn_large), + UNSPEC_GOT); + x = gen_rtx_CONST (Pmode, x); + emit_move_insn (reg11, x); + x = gen_rtx_PLUS (Pmode, reg10, reg11); + x = gen_const_mem (Pmode, x); + emit_move_insn (reg11, x); + } + else + emit_move_insn (reg11, split_stack_fn_large); + + fn = reg11; + + argval = ((args_size << 16) << 16) + allocate; + emit_move_insn (reg10, GEN_INT (argval)); + } + else + { + emit_move_insn (reg10, allocate_rtx); + emit_move_insn (reg11, GEN_INT (args_size)); + use_reg (&call_fusage, reg11); + } + + use_reg (&call_fusage, reg10); + } + else + { + emit_insn (gen_push (GEN_INT (args_size))); + emit_insn (gen_push (allocate_rtx)); + } + call_insn = ix86_expand_call (NULL_RTX, gen_rtx_MEM (QImode, fn), + GEN_INT (UNITS_PER_WORD), constm1_rtx, + NULL_RTX, 0); + add_function_usage_to (call_insn, call_fusage); + + /* In order to make call/return prediction work right, we now need + to execute a return instruction. See + libgcc/config/i386/morestack.S for the details on how this works. + + For flow purposes gcc must not see this as a return + instruction--we need control flow to continue at the subsequent + label. Therefore, we use an unspec. */ + gcc_assert (crtl->args.pops_args < 65536); + emit_insn (gen_split_stack_return (GEN_INT (crtl->args.pops_args))); + + /* If we are in 64-bit mode and this function uses a static chain, + we saved %r10 in %rax before calling _morestack. */ + if (TARGET_64BIT && DECL_STATIC_CHAIN (cfun->decl)) + emit_move_insn (gen_rtx_REG (Pmode, R10_REG), + gen_rtx_REG (Pmode, AX_REG)); + + /* If this function calls va_start, we need to store a pointer to + the arguments on the old stack, because they may not have been + all copied to the new stack. At this point the old stack can be + found at the frame pointer value used by __morestack, because + __morestack has set that up before calling back to us. Here we + store that pointer in a scratch register, and in + ix86_expand_prologue we store the scratch register in a stack + slot. */ + if (cfun->machine->split_stack_varargs_pointer != NULL_RTX) + { + unsigned int scratch_regno; + rtx frame_reg; + int words; + + scratch_regno = split_stack_prologue_scratch_regno (); + scratch_reg = gen_rtx_REG (Pmode, scratch_regno); + frame_reg = gen_rtx_REG (Pmode, BP_REG); + + /* 64-bit: + fp -> old fp value + return address within this function + return address of caller of this function + stack arguments + So we add three words to get to the stack arguments. + + 32-bit: + fp -> old fp value + return address within this function + first argument to __morestack + second argument to __morestack + return address of caller of this function + stack arguments + So we add five words to get to the stack arguments. + */ + words = TARGET_64BIT ? 3 : 5; + emit_insn (gen_rtx_SET (VOIDmode, scratch_reg, + gen_rtx_PLUS (Pmode, frame_reg, + GEN_INT (words * UNITS_PER_WORD)))); + + varargs_label = gen_label_rtx (); + emit_jump_insn (gen_jump (varargs_label)); + JUMP_LABEL (get_last_insn ()) = varargs_label; + + emit_barrier (); + } + + emit_label (label); + LABEL_NUSES (label) = 1; + + /* If this function calls va_start, we now have to set the scratch + register for the case where we do not call __morestack. In this + case we need to set it based on the stack pointer. */ + if (cfun->machine->split_stack_varargs_pointer != NULL_RTX) + { + emit_insn (gen_rtx_SET (VOIDmode, scratch_reg, + gen_rtx_PLUS (Pmode, stack_pointer_rtx, + GEN_INT (UNITS_PER_WORD)))); + + emit_label (varargs_label); + LABEL_NUSES (varargs_label) = 1; + } +} + +/* We may have to tell the dataflow pass that the split stack prologue + is initializing a scratch register. */ + +static void +ix86_live_on_entry (bitmap regs) +{ + if (cfun->machine->split_stack_varargs_pointer != NULL_RTX) + { + gcc_assert (flag_split_stack); + bitmap_set_bit (regs, split_stack_prologue_scratch_regno ()); + } +} /* Extract the parts of an RTL expression that is a valid memory address for an instruction. Return 0 if the structure of the address is @@ -9263,6 +11608,7 @@ rtx base_reg, index_reg; HOST_WIDE_INT scale = 1; rtx scale_rtx = NULL_RTX; + rtx tmp; int retval = 1; enum ix86_address_seg seg = SEG_DEFAULT; @@ -9298,6 +11644,19 @@ scale_rtx = XEXP (op, 1); break; + case ASHIFT: + if (index) + return 0; + index = XEXP (op, 0); + tmp = XEXP (op, 1); + if (!CONST_INT_P (tmp)) + return 0; + scale = INTVAL (tmp); + if ((unsigned HOST_WIDE_INT) scale > 3) + return 0; + scale = 1 << scale; + break; + case UNSPEC: if (XINT (op, 1) == UNSPEC_TP && TARGET_TLS_DIRECT_SEG_REFS @@ -9338,8 +11697,6 @@ } else if (GET_CODE (addr) == ASHIFT) { - rtx tmp; - /* We're called for lea too, which implements ashift on occasion. */ index = XEXP (addr, 0); tmp = XEXP (addr, 1); @@ -9398,8 +11755,7 @@ to test cfun for being non-NULL. */ if (TARGET_K6 && cfun && optimize_function_for_speed_p (cfun) && base_reg && !index_reg && !disp - && REG_P (base_reg) - && REGNO_REG_CLASS (REGNO (base_reg)) == SIREG) + && REG_P (base_reg) && REGNO (base_reg) == SI_REG) disp = const0_rtx; /* Special case: encode reg+reg instead of reg*2. */ @@ -9546,6 +11902,12 @@ if (TARGET_DLLIMPORT_DECL_ATTRIBUTES && SYMBOL_REF_DLLIMPORT_P (x)) return false; + +#if TARGET_MACHO + /* mdynamic-no-pic */ + if (MACHO_DYNAMIC_NO_PIC_P) + return machopic_symbol_defined_p (x); +#endif break; case CONST_DOUBLE: @@ -9637,7 +11999,7 @@ /* Determine if a given CONST RTX is a valid memory displacement in PIC mode. */ -int +bool legitimate_pic_address_disp_p (rtx disp) { bool saw_plus; @@ -9682,7 +12044,7 @@ } } if (GET_CODE (disp) != CONST) - return 0; + return false; disp = XEXP (disp, 0); if (TARGET_64BIT) @@ -9692,29 +12054,30 @@ if (GET_CODE (disp) != UNSPEC || (XINT (disp, 1) != UNSPEC_GOTPCREL && XINT (disp, 1) != UNSPEC_GOTOFF + && XINT (disp, 1) != UNSPEC_PCREL && XINT (disp, 1) != UNSPEC_PLTOFF)) - return 0; + return false; if (GET_CODE (XVECEXP (disp, 0, 0)) != SYMBOL_REF && GET_CODE (XVECEXP (disp, 0, 0)) != LABEL_REF) - return 0; - return 1; + return false; + return true; } saw_plus = false; if (GET_CODE (disp) == PLUS) { if (!CONST_INT_P (XEXP (disp, 1))) - return 0; + return false; disp = XEXP (disp, 0); saw_plus = true; } if (TARGET_MACHO && darwin_local_data_pic (disp)) - return 1; + return true; if (GET_CODE (disp) != UNSPEC) - return 0; + return false; switch (XINT (disp, 1)) { @@ -9753,7 +12116,7 @@ && SYMBOL_REF_TLS_MODEL (disp) == TLS_MODEL_LOCAL_DYNAMIC); } - return 0; + return false; } /* Recognizes RTL expressions that are valid memory addresses for an @@ -9874,6 +12237,7 @@ return false; case UNSPEC_GOTPCREL: + case UNSPEC_PCREL: gcc_assert (flag_pic); goto is_legitimate_pic; @@ -9884,6 +12248,10 @@ case UNSPEC_DTPOFF: break; + case UNSPEC_STACK_CHECK: + gcc_assert (flag_split_stack); + break; + default: /* Invalid address unspec. */ return false; @@ -9912,9 +12280,15 @@ /* Non-constant pic memory reference. */ return false; } - else if (! legitimate_pic_address_disp_p (disp)) + else if ((!TARGET_MACHO || flag_pic) + && ! legitimate_pic_address_disp_p (disp)) /* Displacement is an invalid pic construct. */ return false; +#if TARGET_MACHO + else if (MACHO_DYNAMIC_NO_PIC_P && !legitimate_constant_p (disp)) + /* displacment must be referenced via non_lazy_pointer */ + return false; +#endif /* This code used to verify that a symbolic pic displacement includes the pic_offset_table_rtx register. @@ -10091,7 +12465,19 @@ } } - if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC) + /* For x64 PE-COFF there is no GOT table. So we use address + directly. */ + if (TARGET_64BIT && DEFAULT_ABI == MS_ABI) + { + new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_PCREL); + new_rtx = gen_rtx_CONST (Pmode, new_rtx); + + if (reg == 0) + reg = gen_reg_rtx (Pmode); + emit_move_insn (reg, new_rtx); + new_rtx = reg; + } + else if (TARGET_64BIT && ix86_cmodel != CM_LARGE_PIC) { new_rtx = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, addr), UNSPEC_GOTPCREL); new_rtx = gen_rtx_CONST (Pmode, new_rtx); @@ -10319,6 +12705,17 @@ case TLS_MODEL_INITIAL_EXEC: if (TARGET_64BIT) { + if (TARGET_SUN_TLS) + { + /* The Sun linker took the AMD64 TLS spec literally + and can only handle %rax as destination of the + initial executable code sequence. */ + + dest = gen_reg_rtx (Pmode); + emit_insn (gen_tls_initial_exec_64_sun (dest, x)); + return dest; + } + pic = NULL; type = UNSPEC_GOTNTPOFF; } @@ -10416,7 +12813,7 @@ if (h) return h->to; - *loc = h = GGC_NEW (struct tree_map); + *loc = h = ggc_alloc_tree_map (); h->hash = in.hash; h->base.from = decl; h->to = to = build_decl (DECL_SOURCE_LOCATION (decl), @@ -10523,6 +12920,11 @@ if (flag_pic && SYMBOLIC_CONST (x)) return legitimize_pic_address (x, 0); +#if TARGET_MACHO + if (MACHO_DYNAMIC_NO_PIC_P && SYMBOLIC_CONST (x)) + return machopic_indirect_data_reference (x, 0); +#endif + /* Canonicalize shifts by 0, 1, 2, 3 into multiply */ if (GET_CODE (x) == ASHIFT && CONST_INT_P (XEXP (x, 1)) @@ -10616,7 +13018,7 @@ } } - if (changed && ix86_legitimate_address_p (mode, x, FALSE)) + if (changed && ix86_legitimate_address_p (mode, x, false)) return x; if (GET_CODE (XEXP (x, 0)) == MULT) @@ -10642,7 +13044,7 @@ x = legitimize_pic_address (x, 0); } - if (changed && ix86_legitimate_address_p (mode, x, FALSE)) + if (changed && ix86_legitimate_address_p (mode, x, false)) return x; if (REG_P (XEXP (x, 0))) @@ -10689,7 +13091,7 @@ break; case SYMBOL_REF: - if (! TARGET_MACHO || TARGET_64BIT) + if (TARGET_64BIT || ! TARGET_MACHO_BRANCH_ISLANDS) output_addr_const (file, x); else { @@ -10743,7 +13145,7 @@ } else /* We can't handle floating point constants; - PRINT_OPERAND must handle them. */ + TARGET_PRINT_OPERAND must handle them. */ output_operand_lossage ("floating constant misused"); break; @@ -10775,6 +13177,13 @@ break; case UNSPEC: + if (XINT (x, 1) == UNSPEC_STACK_CHECK) + { + bool f = i386_asm_output_addr_const_extra (file, x); + gcc_assert (f); + break; + } + gcc_assert (XVECLEN (x, 0) == 1); output_pic_addr_const (file, XVECEXP (x, 0, 0), code); switch (XINT (x, 1)) @@ -10788,35 +13197,39 @@ case UNSPEC_PLTOFF: fputs ("@PLTOFF", file); break; + case UNSPEC_PCREL: + fputs (ASSEMBLER_DIALECT == ASM_ATT ? + "(%rip)" : "[rip]", file); + break; case UNSPEC_GOTPCREL: fputs (ASSEMBLER_DIALECT == ASM_ATT ? "@GOTPCREL(%rip)" : "@GOTPCREL[rip]", file); break; case UNSPEC_GOTTPOFF: /* FIXME: This might be @TPOFF in Sun ld too. */ - fputs ("@GOTTPOFF", file); + fputs ("@gottpoff", file); break; case UNSPEC_TPOFF: - fputs ("@TPOFF", file); + fputs ("@tpoff", file); break; case UNSPEC_NTPOFF: if (TARGET_64BIT) - fputs ("@TPOFF", file); - else - fputs ("@NTPOFF", file); + fputs ("@tpoff", file); + else + fputs ("@ntpoff", file); break; case UNSPEC_DTPOFF: - fputs ("@DTPOFF", file); + fputs ("@dtpoff", file); break; case UNSPEC_GOTNTPOFF: if (TARGET_64BIT) fputs (ASSEMBLER_DIALECT == ASM_ATT ? - "@GOTTPOFF(%rip)": "@GOTTPOFF[rip]", file); - else - fputs ("@GOTNTPOFF", file); + "@gottpoff(%rip)": "@gottpoff[rip]", file); + else + fputs ("@gotntpoff", file); break; case UNSPEC_INDNTPOFF: - fputs ("@INDNTPOFF", file); + fputs ("@indntpoff", file); break; #if TARGET_MACHO case UNSPEC_MACHOPIC_OFFSET: @@ -10843,7 +13256,7 @@ { fputs (ASM_LONG, file); output_addr_const (file, x); - fputs ("@DTPOFF", file); + fputs ("@dtpoff", file); switch (size) { case 4: @@ -10870,6 +13283,49 @@ return REG_P (x) && REGNO (x) == PIC_OFFSET_TABLE_REGNUM; } +/* Helper function for ix86_delegitimize_address. + Attempt to delegitimize TLS local-exec accesses. */ + +static rtx +ix86_delegitimize_tls_address (rtx orig_x) +{ + rtx x = orig_x, unspec; + struct ix86_address addr; + + if (!TARGET_TLS_DIRECT_SEG_REFS) + return orig_x; + if (MEM_P (x)) + x = XEXP (x, 0); + if (GET_CODE (x) != PLUS || GET_MODE (x) != Pmode) + return orig_x; + if (ix86_decompose_address (x, &addr) == 0 + || addr.seg != (TARGET_64BIT ? SEG_FS : SEG_GS) + || addr.disp == NULL_RTX + || GET_CODE (addr.disp) != CONST) + return orig_x; + unspec = XEXP (addr.disp, 0); + if (GET_CODE (unspec) == PLUS && CONST_INT_P (XEXP (unspec, 1))) + unspec = XEXP (unspec, 0); + if (GET_CODE (unspec) != UNSPEC || XINT (unspec, 1) != UNSPEC_NTPOFF) + return orig_x; + x = XVECEXP (unspec, 0, 0); + gcc_assert (GET_CODE (x) == SYMBOL_REF); + if (unspec != XEXP (addr.disp, 0)) + x = gen_rtx_PLUS (Pmode, x, XEXP (XEXP (addr.disp, 0), 1)); + if (addr.index) + { + rtx idx = addr.index; + if (addr.scale != 1) + idx = gen_rtx_MULT (Pmode, idx, GEN_INT (addr.scale)); + x = gen_rtx_PLUS (Pmode, idx, x); + } + if (addr.base) + x = gen_rtx_PLUS (Pmode, addr.base, x); + if (MEM_P (orig_x)) + x = replace_equiv_address_nv (orig_x, x); + return x; +} + /* In the name of slightly smaller debug output, and to cater to general assembler lossage, recognize PIC+GOTOFF and turn it back into a direct symbol reference. @@ -10884,6 +13340,9 @@ ix86_delegitimize_address (rtx x) { rtx orig_x = delegitimize_mem_from_attrs (x); + /* addend is NULL or some rtx if x is something+GOTOFF where + something doesn't include the PIC register. */ + rtx addend = NULL_RTX; /* reg_addend is NULL or a multiple of some register. */ rtx reg_addend = NULL_RTX; /* const_addend is NULL or a const_int. */ @@ -10900,15 +13359,23 @@ { if (GET_CODE (x) != CONST || GET_CODE (XEXP (x, 0)) != UNSPEC - || XINT (XEXP (x, 0), 1) != UNSPEC_GOTPCREL + || (XINT (XEXP (x, 0), 1) != UNSPEC_GOTPCREL + && XINT (XEXP (x, 0), 1) != UNSPEC_PCREL) || !MEM_P (orig_x)) - return orig_x; - return XVECEXP (XEXP (x, 0), 0, 0); + return ix86_delegitimize_tls_address (orig_x); + x = XVECEXP (XEXP (x, 0), 0, 0); + if (GET_MODE (orig_x) != Pmode) + { + x = simplify_gen_subreg (GET_MODE (orig_x), x, Pmode, 0); + if (x == NULL_RTX) + return orig_x; + } + return x; } if (GET_CODE (x) != PLUS || GET_CODE (XEXP (x, 1)) != CONST) - return orig_x; + return ix86_delegitimize_tls_address (orig_x); if (ix86_pic_register_p (XEXP (x, 0))) /* %ebx + GOT/GOTOFF */ @@ -10922,14 +13389,13 @@ else if (ix86_pic_register_p (XEXP (reg_addend, 1))) reg_addend = XEXP (reg_addend, 0); else - return orig_x; - if (!REG_P (reg_addend) - && GET_CODE (reg_addend) != MULT - && GET_CODE (reg_addend) != ASHIFT) - return orig_x; - } - else - return orig_x; + { + reg_addend = NULL_RTX; + addend = XEXP (x, 0); + } + } + else + addend = XEXP (x, 0); x = XEXP (XEXP (x, 1), 0); if (GET_CODE (x) == PLUS @@ -10940,7 +13406,7 @@ } if (GET_CODE (x) == UNSPEC - && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x)) + && ((XINT (x, 1) == UNSPEC_GOT && MEM_P (orig_x) && !addend) || (XINT (x, 1) == UNSPEC_GOTOFF && !MEM_P (orig_x)))) result = XVECEXP (x, 0, 0); @@ -10949,12 +13415,34 @@ result = XVECEXP (x, 0, 0); if (! result) - return orig_x; + return ix86_delegitimize_tls_address (orig_x); if (const_addend) result = gen_rtx_CONST (Pmode, gen_rtx_PLUS (Pmode, result, const_addend)); if (reg_addend) result = gen_rtx_PLUS (Pmode, reg_addend, result); + if (addend) + { + /* If the rest of original X doesn't involve the PIC register, add + addend and subtract pic_offset_table_rtx. This can happen e.g. + for code like: + leal (%ebx, %ecx, 4), %ecx + ... + movl foo@GOTOFF(%ecx), %edx + in which case we return (%ecx - %ebx) + foo. */ + if (pic_offset_table_rtx) + result = gen_rtx_PLUS (Pmode, gen_rtx_MINUS (Pmode, copy_rtx (addend), + pic_offset_table_rtx), + result); + else + return orig_x; + } + if (GET_MODE (orig_x) != Pmode && MEM_P (orig_x)) + { + result = simplify_gen_subreg (GET_MODE (orig_x), result, Pmode, 0); + if (result == NULL_RTX) + return orig_x; + } return result; } @@ -10977,7 +13465,8 @@ || GET_CODE (XEXP (term, 1)) == CONST_DOUBLE)) term = XEXP (term, 0); if (GET_CODE (term) != UNSPEC - || XINT (term, 1) != UNSPEC_GOTPCREL) + || (XINT (term, 1) != UNSPEC_GOTPCREL + && XINT (term, 1) != UNSPEC_PCREL)) return x; return XVECEXP (term, 0, 0); @@ -11295,7 +13784,7 @@ return cfun->machine->some_ld_name; for (insn = get_insns (); insn ; insn = NEXT_INSN (insn)) - if (INSN_P (insn) + if (NONDEBUG_INSN_P (insn) && for_each_rtx (&PATTERN (insn), get_some_local_dynamic_name_1, 0)) return cfun->machine->some_ld_name; @@ -11306,7 +13795,6 @@ L,W,B,Q,S,T -- print the opcode suffix for specified size of operand. C -- print opcode suffix for set/cmov insn. c -- like C, but print reversed condition - E,e -- likewise, but for compare-and-branch fused insn. F,f -- likewise, but for floating-point. O -- if HAVE_AS_IX86_CMOV_SUN_SYNTAX, expand to "w.", "l." or "q.", otherwise nothing @@ -11336,10 +13824,11 @@ Y -- print condition for XOP pcom* instruction. + -- print a branch hint as 'cs' or 'ds' prefix ; -- print a semicolon (after prefixes due to bug in older gas). + @ -- print a segment register of thread base pointer load */ void -print_operand (FILE *file, rtx x, int code) +ix86_print_operand (FILE *file, rtx x, int code) { if (code) { @@ -11374,7 +13863,7 @@ if (!REG_P (x)) { putc ('[', file); - PRINT_OPERAND (file, x, 0); + ix86_print_operand (file, x, 0); putc (']', file); return; } @@ -11384,7 +13873,7 @@ gcc_unreachable (); } - PRINT_OPERAND (file, x, 0); + ix86_print_operand (file, x, 0); return; @@ -11522,7 +14011,7 @@ output_operand_lossage ("invalid operand size for operand code '%c'", code); return; - + case 'd': case 'b': case 'w': @@ -11539,7 +14028,7 @@ case 's': if (CONST_INT_P (x) || ! SHIFT_DOUBLE_OMITS_COUNT) { - PRINT_OPERAND (file, x, 0); + ix86_print_operand (file, x, 0); fputs (", ", file); } return; @@ -11711,14 +14200,6 @@ put_condition_code (GET_CODE (x), GET_MODE (XEXP (x, 0)), 1, 1, file); return; - case 'E': - put_condition_code (GET_CODE (x), CCmode, 0, 0, file); - return; - - case 'e': - put_condition_code (GET_CODE (x), CCmode, 1, 0, file); - return; - case 'H': /* It doesn't actually matter what mode we use here, as we're only going to use this for printing. */ @@ -11817,11 +14298,22 @@ return; case ';': -#if TARGET_MACHO - fputs (" ; ", file); -#else - putc (' ', file); -#endif +#ifndef HAVE_AS_IX86_REP_LOCK_PREFIX + putc (';', file); +#endif + return; + + case '@': + if (ASSEMBLER_DIALECT == ASM_ATT) + putc ('%', file); + + /* The kernel uses a different segment register for performance + reasons; a system call would not have to trash the userspace + segment register, which would be expensive. */ + if (TARGET_64BIT && ix86_cmodel != CM_KERNEL) + fputs ("fs", file); + else + fputs ("gs", file); return; default: @@ -11890,7 +14382,11 @@ if (ASSEMBLER_DIALECT == ASM_ATT) putc ('$', file); - fprintf (file, "0x%08lx", (long unsigned int) l); + /* Sign extend 32bit SFmode immediate to 8 bytes. */ + if (code == 'q') + fprintf (file, "0x%08llx", (unsigned long long) (int) l); + else + fprintf (file, "0x%08x", (unsigned int) l); } /* These float cases don't actually occur as immediate operands. */ @@ -11940,17 +14436,24 @@ } if (CONST_INT_P (x)) fprintf (file, HOST_WIDE_INT_PRINT_DEC, INTVAL (x)); - else if (flag_pic) + else if (flag_pic || MACHOPIC_INDIRECT) output_pic_addr_const (file, x, code); else output_addr_const (file, x); } } + +static bool +ix86_print_operand_punct_valid_p (unsigned char code) +{ + return (code == '@' || code == '*' || code == '+' + || code == '&' || code == ';'); +} /* Print a memory operand whose address is ADDR. */ -void -print_operand_address (FILE *file, rtx addr) +static void +ix86_print_operand_address (FILE *file, rtx addr) { struct ix86_address parts; rtx base, index, disp; @@ -12088,8 +14591,10 @@ } } -bool -output_addr_const_extra (FILE *file, rtx x) +/* Implementation of TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA. */ + +static bool +i386_asm_output_addr_const_extra (FILE *file, rtx x) { rtx op; @@ -12102,34 +14607,34 @@ case UNSPEC_GOTTPOFF: output_addr_const (file, op); /* FIXME: This might be @TPOFF in Sun ld. */ - fputs ("@GOTTPOFF", file); + fputs ("@gottpoff", file); break; case UNSPEC_TPOFF: output_addr_const (file, op); - fputs ("@TPOFF", file); + fputs ("@tpoff", file); break; case UNSPEC_NTPOFF: output_addr_const (file, op); if (TARGET_64BIT) - fputs ("@TPOFF", file); - else - fputs ("@NTPOFF", file); + fputs ("@tpoff", file); + else + fputs ("@ntpoff", file); break; case UNSPEC_DTPOFF: output_addr_const (file, op); - fputs ("@DTPOFF", file); + fputs ("@dtpoff", file); break; case UNSPEC_GOTNTPOFF: output_addr_const (file, op); if (TARGET_64BIT) fputs (ASSEMBLER_DIALECT == ASM_ATT ? - "@GOTTPOFF(%rip)" : "@GOTTPOFF[rip]", file); - else - fputs ("@GOTNTPOFF", file); + "@gottpoff(%rip)" : "@gottpoff[rip]", file); + else + fputs ("@gotntpoff", file); break; case UNSPEC_INDNTPOFF: output_addr_const (file, op); - fputs ("@INDNTPOFF", file); + fputs ("@indntpoff", file); break; #if TARGET_MACHO case UNSPEC_MACHOPIC_OFFSET: @@ -12139,6 +14644,22 @@ break; #endif + case UNSPEC_STACK_CHECK: + { + int offset; + + gcc_assert (flag_split_stack); + +#ifdef TARGET_THREAD_SPLIT_STACK_OFFSET + offset = TARGET_THREAD_SPLIT_STACK_OFFSET; +#else + gcc_unreachable (); +#endif + + fprintf (file, "%s:%d", TARGET_64BIT ? "%fs" : "%gs", offset); + } + break; + default: return false; } @@ -12146,15 +14667,33 @@ return true; } -/* Split one or more DImode RTL references into pairs of SImode +/* Split one or more double-mode RTL references into pairs of half-mode references. The RTL can be REG, offsettable MEM, integer constant, or - CONST_DOUBLE. "operands" is a pointer to an array of DImode RTL to + CONST_DOUBLE. "operands" is a pointer to an array of double-mode RTLs to split and "num" is its length. lo_half and hi_half are output arrays that parallel "operands". */ void -split_di (rtx operands[], int num, rtx lo_half[], rtx hi_half[]) -{ +split_double_mode (enum machine_mode mode, rtx operands[], + int num, rtx lo_half[], rtx hi_half[]) +{ + enum machine_mode half_mode; + unsigned int byte; + + switch (mode) + { + case TImode: + half_mode = DImode; + break; + case DImode: + half_mode = SImode; + break; + default: + gcc_unreachable (); + } + + byte = GET_MODE_SIZE (half_mode); + while (num--) { rtx op = operands[num]; @@ -12163,44 +14702,17 @@ but we still have to handle it. */ if (MEM_P (op)) { - lo_half[num] = adjust_address (op, SImode, 0); - hi_half[num] = adjust_address (op, SImode, 4); - } - else - { - lo_half[num] = simplify_gen_subreg (SImode, op, - GET_MODE (op) == VOIDmode - ? DImode : GET_MODE (op), 0); - hi_half[num] = simplify_gen_subreg (SImode, op, + lo_half[num] = adjust_address (op, half_mode, 0); + hi_half[num] = adjust_address (op, half_mode, byte); + } + else + { + lo_half[num] = simplify_gen_subreg (half_mode, op, GET_MODE (op) == VOIDmode - ? DImode : GET_MODE (op), 4); - } - } -} -/* Split one or more TImode RTL references into pairs of DImode - references. The RTL can be REG, offsettable MEM, integer constant, or - CONST_DOUBLE. "operands" is a pointer to an array of DImode RTL to - split and "num" is its length. lo_half and hi_half are output arrays - that parallel "operands". */ - -void -split_ti (rtx operands[], int num, rtx lo_half[], rtx hi_half[]) -{ - while (num--) - { - rtx op = operands[num]; - - /* simplify_subreg refuse to split volatile memory addresses, but we - still have to handle it. */ - if (MEM_P (op)) - { - lo_half[num] = adjust_address (op, DImode, 0); - hi_half[num] = adjust_address (op, DImode, 8); - } - else - { - lo_half[num] = simplify_gen_subreg (DImode, op, TImode, 0); - hi_half[num] = simplify_gen_subreg (DImode, op, TImode, 8); + ? mode : GET_MODE (op), 0); + hi_half[num] = simplify_gen_subreg (half_mode, op, + GET_MODE (op) == VOIDmode + ? mode : GET_MODE (op), byte); } } } @@ -12784,7 +15296,7 @@ gcc_assert (!TARGET_64BIT); #endif - fprintf (file, "%s" LPREFIX "%d\n", directive, value); + fprintf (file, "%s%s%d\n", directive, LPREFIX, value); } void @@ -12800,21 +15312,21 @@ #endif /* We can't use @GOTOFF for text labels on VxWorks; see gotoff_operand. */ if (TARGET_64BIT || TARGET_VXWORKS_RTP) - fprintf (file, "%s" LPREFIX "%d-" LPREFIX "%d\n", - directive, value, rel); + fprintf (file, "%s%s%d-%s%d\n", + directive, LPREFIX, value, LPREFIX, rel); else if (HAVE_AS_GOTOFF_IN_DATA) - fprintf (file, ASM_LONG LPREFIX "%d@GOTOFF\n", value); + fprintf (file, ASM_LONG "%s%d@GOTOFF\n", LPREFIX, value); #if TARGET_MACHO else if (TARGET_MACHO) { - fprintf (file, ASM_LONG LPREFIX "%d-", value); + fprintf (file, ASM_LONG "%s%d-", LPREFIX, value); machopic_output_function_base_name (file); putc ('\n', file); } #endif else - asm_fprintf (file, ASM_LONG "%U%s+[.-" LPREFIX "%d]\n", - GOT_SYMBOL_NAME, value); + asm_fprintf (file, ASM_LONG "%U%s+[.-%s%d]\n", + GOT_SYMBOL_NAME, LPREFIX, value); } /* Generate either "mov $0, reg" or "xor reg, reg", as appropriate @@ -12905,25 +15417,43 @@ } } - if (flag_pic && mode == Pmode && symbolic_operand (op1, Pmode)) + if ((flag_pic || MACHOPIC_INDIRECT) + && mode == Pmode && symbolic_operand (op1, Pmode)) { if (TARGET_MACHO && !TARGET_64BIT) { #if TARGET_MACHO - if (MACHOPIC_PURE) + /* dynamic-no-pic */ + if (MACHOPIC_INDIRECT) { rtx temp = ((reload_in_progress || ((op0 && REG_P (op0)) && mode == Pmode)) ? op0 : gen_reg_rtx (Pmode)); op1 = machopic_indirect_data_reference (op1, temp); - op1 = machopic_legitimize_pic_address (op1, mode, - temp == op1 ? 0 : temp); - } - else if (MACHOPIC_INDIRECT) - op1 = machopic_indirect_data_reference (op1, 0); - if (op0 == op1) + if (MACHOPIC_PURE) + op1 = machopic_legitimize_pic_address (op1, mode, + temp == op1 ? 0 : temp); + } + if (op0 != op1 && GET_CODE (op0) != MEM) + { + rtx insn = gen_rtx_SET (VOIDmode, op0, op1); + emit_insn (insn); + return; + } + if (GET_CODE (op0) == MEM) + op1 = force_reg (Pmode, op1); + else + { + rtx temp = op0; + if (GET_CODE (temp) != REG) + temp = gen_reg_rtx (Pmode); + temp = legitimize_pic_address (op1, temp); + if (temp == op0) return; + op1 = temp; + } + /* dynamic-no-pic */ #endif } else @@ -13107,6 +15637,14 @@ switch (GET_MODE_SIZE (mode)) { case 16: + /* If we're optimizing for size, movups is the smallest. */ + if (TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL) + { + op0 = gen_lowpart (V4SFmode, op0); + op1 = gen_lowpart (V4SFmode, op1); + emit_insn (gen_avx_movups (op0, op1)); + return; + } op0 = gen_lowpart (V16QImode, op0); op1 = gen_lowpart (V16QImode, op1); emit_insn (gen_avx_movdqu (op0, op1)); @@ -13125,7 +15663,7 @@ op1 = gen_lowpart (mode, op1); switch (mode) - { + { case V4SFmode: emit_insn (gen_avx_movups (op0, op1)); break; @@ -13133,6 +15671,13 @@ emit_insn (gen_avx_movups256 (op0, op1)); break; case V2DFmode: + if (TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL) + { + op0 = gen_lowpart (V4SFmode, op0); + op1 = gen_lowpart (V4SFmode, op1); + emit_insn (gen_avx_movups (op0, op1)); + return; + } emit_insn (gen_avx_movupd (op0, op1)); break; case V4DFmode: @@ -13153,7 +15698,8 @@ if (MEM_P (op1)) { /* If we're optimizing for size, movups is the smallest. */ - if (optimize_insn_for_size_p ()) + if (optimize_insn_for_size_p () + || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL) { op0 = gen_lowpart (V4SFmode, op0); op1 = gen_lowpart (V4SFmode, op1); @@ -13176,13 +15722,13 @@ { rtx zero; - if (TARGET_SSE_UNALIGNED_MOVE_OPTIMAL) - { - op0 = gen_lowpart (V2DFmode, op0); - op1 = gen_lowpart (V2DFmode, op1); - emit_insn (gen_sse2_movupd (op0, op1)); - return; - } + if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL) + { + op0 = gen_lowpart (V2DFmode, op0); + op1 = gen_lowpart (V2DFmode, op1); + emit_insn (gen_sse2_movupd (op0, op1)); + return; + } /* When SSE registers are split into halves, we can avoid writing to the top half twice. */ @@ -13211,12 +15757,12 @@ } else { - if (TARGET_SSE_UNALIGNED_MOVE_OPTIMAL) - { - op0 = gen_lowpart (V4SFmode, op0); - op1 = gen_lowpart (V4SFmode, op1); - emit_insn (gen_sse_movups (op0, op1)); - return; + if (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL) + { + op0 = gen_lowpart (V4SFmode, op0); + op1 = gen_lowpart (V4SFmode, op1); + emit_insn (gen_sse_movups (op0, op1)); + return; } if (TARGET_SSE_PARTIAL_REG_DEPENDENCY) @@ -13235,7 +15781,8 @@ else if (MEM_P (op0)) { /* If we're optimizing for size, movups is the smallest. */ - if (optimize_insn_for_size_p ()) + if (optimize_insn_for_size_p () + || TARGET_SSE_PACKED_SINGLE_INSN_OPTIMAL) { op0 = gen_lowpart (V4SFmode, op0); op1 = gen_lowpart (V4SFmode, op1); @@ -13256,19 +15803,37 @@ if (TARGET_SSE2 && mode == V2DFmode) { - m = adjust_address (op0, DFmode, 0); - emit_insn (gen_sse2_storelpd (m, op1)); - m = adjust_address (op0, DFmode, 8); - emit_insn (gen_sse2_storehpd (m, op1)); + if (TARGET_SSE_UNALIGNED_STORE_OPTIMAL) + { + op0 = gen_lowpart (V2DFmode, op0); + op1 = gen_lowpart (V2DFmode, op1); + emit_insn (gen_sse2_movupd (op0, op1)); + } + else + { + m = adjust_address (op0, DFmode, 0); + emit_insn (gen_sse2_storelpd (m, op1)); + m = adjust_address (op0, DFmode, 8); + emit_insn (gen_sse2_storehpd (m, op1)); + } } else { if (mode != V4SFmode) op1 = gen_lowpart (V4SFmode, op1); - m = adjust_address (op0, V2SFmode, 0); - emit_insn (gen_sse_storelps (m, op1)); - m = adjust_address (op0, V2SFmode, 8); - emit_insn (gen_sse_storehps (m, op1)); + + if (TARGET_SSE_UNALIGNED_STORE_OPTIMAL) + { + op0 = gen_lowpart (V4SFmode, op0); + emit_insn (gen_sse_movups (op0, op1)); + } + else + { + m = adjust_address (op0, V2SFmode, 0); + emit_insn (gen_sse_storelps (m, op1)); + m = adjust_address (op0, V2SFmode, 8); + emit_insn (gen_sse_storehps (m, op1)); + } } } else @@ -13387,16 +15952,6 @@ if (MEM_P (src1) && !rtx_equal_p (dst, src1)) src1 = force_reg (mode, src1); - /* In order for the multiply-add patterns to get matched, we need - to aid combine by forcing all operands into registers to start. */ - if (optimize && TARGET_FMA4) - { - if (MEM_P (src2)) - src2 = force_reg (GET_MODE (src2), src2); - else if (MEM_P (src1)) - src1 = force_reg (GET_MODE (src1), src1); - } - operands[1] = src1; operands[2] = src2; return dst; @@ -13437,6 +15992,13 @@ gcc_assert (code == PLUS); emit_insn (op); } + else if (reload_completed + && code == PLUS + && !rtx_equal_p (dst, src1)) + { + /* This is going to be an LEA; avoid splitting it later. */ + emit_insn (op); + } else { clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG)); @@ -13451,7 +16013,7 @@ /* Return TRUE or FALSE depending on whether the binary operator meets the appropriate constraints. */ -int +bool ix86_binary_operator_ok (enum rtx_code code, enum machine_mode mode, rtx operands[3]) { @@ -13461,7 +16023,7 @@ /* Both source operands cannot be in memory. */ if (MEM_P (src1) && MEM_P (src2)) - return 0; + return false; /* Canonicalize operand order for commutative operators. */ if (ix86_swap_binary_operands_p (code, mode, operands)) @@ -13473,17 +16035,26 @@ /* If the destination is memory, we must have a matching source operand. */ if (MEM_P (dst) && !rtx_equal_p (dst, src1)) - return 0; + return false; /* Source 1 cannot be a constant. */ if (CONSTANT_P (src1)) - return 0; + return false; /* Source 1 cannot be a non-matching memory. */ if (MEM_P (src1) && !rtx_equal_p (dst, src1)) - return 0; - - return 1; + { + /* Support "andhi/andsi/anddi" as a zero-extending move. */ + return (code == AND + && (mode == HImode + || mode == SImode + || (TARGET_64BIT && mode == DImode)) + && CONST_INT_P (src2) + && (INTVAL (src2) == 0xff + || INTVAL (src2) == 0xffff)); + } + + return true; } /* Attempt to expand a unary operator. Make the expansion closer to the @@ -13536,10 +16107,111 @@ emit_move_insn (operands[0], dst); } +/* Split 32bit/64bit divmod with 8bit unsigned divmod if dividend and + divisor are within the the range [0-255]. */ + +void +ix86_split_idivmod (enum machine_mode mode, rtx operands[], + bool signed_p) +{ + rtx end_label, qimode_label; + rtx insn, div, mod; + rtx scratch, tmp0, tmp1, tmp2; + rtx (*gen_divmod4_1) (rtx, rtx, rtx, rtx); + rtx (*gen_zero_extend) (rtx, rtx); + rtx (*gen_test_ccno_1) (rtx, rtx); + + switch (mode) + { + case SImode: + gen_divmod4_1 = signed_p ? gen_divmodsi4_1 : gen_udivmodsi4_1; + gen_test_ccno_1 = gen_testsi_ccno_1; + gen_zero_extend = gen_zero_extendqisi2; + break; + case DImode: + gen_divmod4_1 = signed_p ? gen_divmoddi4_1 : gen_udivmoddi4_1; + gen_test_ccno_1 = gen_testdi_ccno_1; + gen_zero_extend = gen_zero_extendqidi2; + break; + default: + gcc_unreachable (); + } + + end_label = gen_label_rtx (); + qimode_label = gen_label_rtx (); + + scratch = gen_reg_rtx (mode); + + /* Use 8bit unsigned divimod if dividend and divisor are within the + the range [0-255]. */ + emit_move_insn (scratch, operands[2]); + scratch = expand_simple_binop (mode, IOR, scratch, operands[3], + scratch, 1, OPTAB_DIRECT); + emit_insn (gen_test_ccno_1 (scratch, GEN_INT (-0x100))); + tmp0 = gen_rtx_REG (CCNOmode, FLAGS_REG); + tmp0 = gen_rtx_EQ (VOIDmode, tmp0, const0_rtx); + tmp0 = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp0, + gen_rtx_LABEL_REF (VOIDmode, qimode_label), + pc_rtx); + insn = emit_jump_insn (gen_rtx_SET (VOIDmode, pc_rtx, tmp0)); + predict_jump (REG_BR_PROB_BASE * 50 / 100); + JUMP_LABEL (insn) = qimode_label; + + /* Generate original signed/unsigned divimod. */ + div = gen_divmod4_1 (operands[0], operands[1], + operands[2], operands[3]); + emit_insn (div); + + /* Branch to the end. */ + emit_jump_insn (gen_jump (end_label)); + emit_barrier (); + + /* Generate 8bit unsigned divide. */ + emit_label (qimode_label); + /* Don't use operands[0] for result of 8bit divide since not all + registers support QImode ZERO_EXTRACT. */ + tmp0 = simplify_gen_subreg (HImode, scratch, mode, 0); + tmp1 = simplify_gen_subreg (HImode, operands[2], mode, 0); + tmp2 = simplify_gen_subreg (QImode, operands[3], mode, 0); + emit_insn (gen_udivmodhiqi3 (tmp0, tmp1, tmp2)); + + if (signed_p) + { + div = gen_rtx_DIV (SImode, operands[2], operands[3]); + mod = gen_rtx_MOD (SImode, operands[2], operands[3]); + } + else + { + div = gen_rtx_UDIV (SImode, operands[2], operands[3]); + mod = gen_rtx_UMOD (SImode, operands[2], operands[3]); + } + + /* Extract remainder from AH. */ + tmp1 = gen_rtx_ZERO_EXTRACT (mode, tmp0, GEN_INT (8), GEN_INT (8)); + if (REG_P (operands[1])) + insn = emit_move_insn (operands[1], tmp1); + else + { + /* Need a new scratch register since the old one has result + of 8bit divide. */ + scratch = gen_reg_rtx (mode); + emit_move_insn (scratch, tmp1); + insn = emit_move_insn (operands[1], scratch); + } + set_unique_reg_note (insn, REG_EQUAL, mod); + + /* Zero extend quotient from AL. */ + tmp1 = gen_lowpart (QImode, tmp0); + insn = emit_insn (gen_zero_extend (operands[0], tmp1)); + set_unique_reg_note (insn, REG_EQUAL, div); + + emit_label (end_label); +} + #define LEA_SEARCH_THRESHOLD 12 /* Search backward for non-agu definition of register number REGNO1 - or register number REGNO2 in INSN's basic block until + or register number REGNO2 in INSN's basic block until 1. Pass LEA_SEARCH_THRESHOLD instructions, or 2. Reach BB boundary, or 3. Reach agu definition. @@ -13560,7 +16232,7 @@ rtx prev = PREV_INSN (insn); while (prev && distance < LEA_SEARCH_THRESHOLD) { - if (INSN_P (prev)) + if (NONDEBUG_INSN_P (prev)) { distance++; for (def_rec = DF_INSN_DEFS (prev); *def_rec; def_rec++) @@ -13579,20 +16251,20 @@ prev = PREV_INSN (prev); } } - + if (distance < LEA_SEARCH_THRESHOLD) { edge e; edge_iterator ei; bool simple_loop = false; - + FOR_EACH_EDGE (e, ei, bb->preds) if (e->src == bb) { simple_loop = true; break; } - + if (simple_loop) { rtx prev = BB_END (bb); @@ -13600,7 +16272,7 @@ && prev != insn && distance < LEA_SEARCH_THRESHOLD) { - if (INSN_P (prev)) + if (NONDEBUG_INSN_P (prev)) { distance++; for (def_rec = DF_INSN_DEFS (prev); *def_rec; def_rec++) @@ -13629,7 +16301,7 @@ return distance; } -/* Return the distance between INSN and the next insn that uses +/* Return the distance between INSN and the next insn that uses register number REGNO0 in memory address. Return -1 if no such a use is found within LEA_SEARCH_THRESHOLD or REGNO0 is set. */ @@ -13646,7 +16318,7 @@ rtx next = NEXT_INSN (insn); while (next && distance < LEA_SEARCH_THRESHOLD) { - if (INSN_P (next)) + if (NONDEBUG_INSN_P (next)) { distance++; @@ -13680,14 +16352,14 @@ edge e; edge_iterator ei; bool simple_loop = false; - + FOR_EACH_EDGE (e, ei, bb->succs) if (e->dest == bb) { simple_loop = true; break; } - + if (simple_loop) { rtx next = BB_HEAD (bb); @@ -13695,7 +16367,7 @@ && next != insn && distance < LEA_SEARCH_THRESHOLD) { - if (INSN_P (next)) + if (NONDEBUG_INSN_P (next)) { distance++; @@ -13722,7 +16394,7 @@ next = NEXT_INSN (next); } } - } + } return -1; } @@ -13735,39 +16407,40 @@ #define IX86_LEA_PRIORITY 2 /* Return true if it is ok to optimize an ADD operation to LEA - operation to avoid flag register consumation. For the processors - like ATOM, if the destination register of LEA holds an actual - address which will be used soon, LEA is better and otherwise ADD - is better. */ + operation to avoid flag register consumation. For most processors, + ADD is faster than LEA. For the processors like ATOM, if the + destination register of LEA holds an actual address which will be + used soon, LEA is better and otherwise ADD is better. */ bool -ix86_lea_for_add_ok (enum rtx_code code ATTRIBUTE_UNUSED, - rtx insn, rtx operands[]) +ix86_lea_for_add_ok (rtx insn, rtx operands[]) { unsigned int regno0 = true_regnum (operands[0]); unsigned int regno1 = true_regnum (operands[1]); - unsigned int regno2; - - if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun)) - return regno0 != regno1; - - regno2 = true_regnum (operands[2]); + unsigned int regno2 = true_regnum (operands[2]); /* If a = b + c, (a!=b && a!=c), must use lea form. */ if (regno0 != regno1 && regno0 != regno2) return true; - else + + if (!TARGET_OPT_AGU || optimize_function_for_size_p (cfun)) + return false; + else { int dist_define, dist_use; + + /* Return false if REGNO0 isn't used in memory address. */ + dist_use = distance_agu_use (regno0, insn); + if (dist_use <= 0) + return false; + dist_define = distance_non_agu_define (regno1, regno2, insn); if (dist_define <= 0) return true; /* If this insn has both backward non-agu dependence and forward agu dependence, the one with short distance take effect. */ - dist_use = distance_agu_use (regno0, insn); - if (dist_use <= 0 - || (dist_define + IX86_LEA_PRIORITY) < dist_use) + if ((dist_define + IX86_LEA_PRIORITY) < dist_use) return false; return true; @@ -13818,7 +16491,7 @@ break; } - if (shift_rtx + if (shift_rtx && (GET_CODE (shift_rtx) == ASHIFT || GET_CODE (shift_rtx) == LSHIFTRT || GET_CODE (shift_rtx) == ASHIFTRT @@ -13849,7 +16522,7 @@ /* Return TRUE or FALSE depending on whether the unary operator meets the appropriate constraints. */ -int +bool ix86_unary_operator_ok (enum rtx_code code ATTRIBUTE_UNUSED, enum machine_mode mode ATTRIBUTE_UNUSED, rtx operands[2] ATTRIBUTE_UNUSED) @@ -13858,8 +16531,8 @@ if ((MEM_P (operands[0]) || MEM_P (operands[1])) && ! rtx_equal_p (operands[0], operands[1])) - return FALSE; - return TRUE; + return false; + return true; } /* Return TRUE if the operands to a vec_interleave_{high,low}v2df @@ -14096,17 +16769,28 @@ rtvec v; switch (mode) { - case SImode: + case V4SImode: gcc_assert (vect); v = gen_rtvec (4, value, value, value, value); return gen_rtx_CONST_VECTOR (V4SImode, v); - case DImode: + case V2DImode: gcc_assert (vect); v = gen_rtvec (2, value, value); return gen_rtx_CONST_VECTOR (V2DImode, v); - case SFmode: + case V8SFmode: + if (vect) + v = gen_rtvec (8, value, value, value, value, + value, value, value, value); + else + v = gen_rtvec (8, value, CONST0_RTX (SFmode), + CONST0_RTX (SFmode), CONST0_RTX (SFmode), + CONST0_RTX (SFmode), CONST0_RTX (SFmode), + CONST0_RTX (SFmode), CONST0_RTX (SFmode)); + return gen_rtx_CONST_VECTOR (V8SFmode, v); + + case V4SFmode: if (vect) v = gen_rtvec (4, value, value, value, value); else @@ -14114,7 +16798,15 @@ CONST0_RTX (SFmode), CONST0_RTX (SFmode)); return gen_rtx_CONST_VECTOR (V4SFmode, v); - case DFmode: + case V4DFmode: + if (vect) + v = gen_rtvec (4, value, value, value, value); + else + v = gen_rtvec (4, value, CONST0_RTX (DFmode), + CONST0_RTX (DFmode), CONST0_RTX (DFmode)); + return gen_rtx_CONST_VECTOR (V4DFmode, v); + + case V2DFmode: if (vect) v = gen_rtvec (2, value, value); else @@ -14144,17 +16836,21 @@ /* Find the sign bit, sign extended to 2*HWI. */ switch (mode) { - case SImode: - case SFmode: + case V4SImode: + case V8SFmode: + case V4SFmode: + vec_mode = mode; + mode = GET_MODE_INNER (mode); imode = SImode; - vec_mode = (mode == SImode) ? V4SImode : V4SFmode; lo = 0x80000000, hi = lo < 0; break; - case DImode: - case DFmode: + case V2DImode: + case V4DFmode: + case V2DFmode: + vec_mode = mode; + mode = GET_MODE_INNER (mode); imode = DImode; - vec_mode = (mode == DImode) ? V2DImode : V2DFmode; if (HOST_BITS_PER_WIDE_INT >= 64) lo = (HOST_WIDE_INT)1 << shift, hi = -1; else @@ -14208,7 +16904,7 @@ if (vec_mode == VOIDmode) return force_reg (mode, mask); - v = ix86_build_const_vector (mode, vect, mask); + v = ix86_build_const_vector (vec_mode, vect, mask); return force_reg (vec_mode, v); } @@ -14218,51 +16914,54 @@ ix86_expand_fp_absneg_operator (enum rtx_code code, enum machine_mode mode, rtx operands[]) { - rtx mask, set, use, clob, dst, src; + rtx mask, set, dst, src; bool use_sse = false; bool vector_mode = VECTOR_MODE_P (mode); - enum machine_mode elt_mode = mode; + enum machine_mode vmode = mode; if (vector_mode) - { - elt_mode = GET_MODE_INNER (mode); - use_sse = true; - } + use_sse = true; else if (mode == TFmode) use_sse = true; else if (TARGET_SSE_MATH) - use_sse = SSE_FLOAT_MODE_P (mode); + { + use_sse = SSE_FLOAT_MODE_P (mode); + if (mode == SFmode) + vmode = V4SFmode; + else if (mode == DFmode) + vmode = V2DFmode; + } /* NEG and ABS performed with SSE use bitwise mask operations. Create the appropriate mask now. */ if (use_sse) - mask = ix86_build_signbit_mask (elt_mode, vector_mode, code == ABS); + mask = ix86_build_signbit_mask (vmode, vector_mode, code == ABS); else mask = NULL_RTX; dst = operands[0]; src = operands[1]; - if (vector_mode) - { - set = gen_rtx_fmt_ee (code == NEG ? XOR : AND, mode, src, mask); - set = gen_rtx_SET (VOIDmode, dst, set); - emit_insn (set); - } - else - { - set = gen_rtx_fmt_e (code, mode, src); - set = gen_rtx_SET (VOIDmode, dst, set); - if (mask) - { - use = gen_rtx_USE (VOIDmode, mask); + set = gen_rtx_fmt_e (code, mode, src); + set = gen_rtx_SET (VOIDmode, dst, set); + + if (mask) + { + rtx use, clob; + rtvec par; + + use = gen_rtx_USE (VOIDmode, mask); + if (vector_mode) + par = gen_rtvec (2, set, use); + else + { clob = gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (CCmode, FLAGS_REG)); - emit_insn (gen_rtx_PARALLEL (VOIDmode, - gen_rtvec (3, set, use, clob))); + par = gen_rtvec (3, set, use, clob); } - else - emit_insn (set); - } + emit_insn (gen_rtx_PARALLEL (VOIDmode, par)); + } + else + emit_insn (set); } /* Expand a copysign operation. Special case operand 0 being a constant. */ @@ -14270,7 +16969,7 @@ void ix86_expand_copysign (rtx operands[]) { - enum machine_mode mode; + enum machine_mode mode, vmode; rtx dest, op0, op1, mask, nmask; dest = operands[0]; @@ -14279,6 +16978,13 @@ mode = GET_MODE (dest); + if (mode == SFmode) + vmode = V4SFmode; + else if (mode == DFmode) + vmode = V2DFmode; + else + vmode = mode; + if (GET_CODE (op0) == CONST_DOUBLE) { rtx (*copysign_insn)(rtx, rtx, rtx, rtx); @@ -14288,15 +16994,11 @@ if (mode == SFmode || mode == DFmode) { - enum machine_mode vmode; - - vmode = mode == SFmode ? V4SFmode : V2DFmode; - if (op0 == CONST0_RTX (mode)) op0 = CONST0_RTX (vmode); else { - rtx v = ix86_build_const_vector (mode, false, op0); + rtx v = ix86_build_const_vector (vmode, false, op0); op0 = force_reg (vmode, v); } @@ -14304,7 +17006,7 @@ else if (op0 != CONST0_RTX (mode)) op0 = force_reg (mode, op0); - mask = ix86_build_signbit_mask (mode, 0, 0); + mask = ix86_build_signbit_mask (vmode, 0, 0); if (mode == SFmode) copysign_insn = gen_copysignsf3_const; @@ -14319,8 +17021,8 @@ { rtx (*copysign_insn)(rtx, rtx, rtx, rtx, rtx, rtx); - nmask = ix86_build_signbit_mask (mode, 0, 1); - mask = ix86_build_signbit_mask (mode, 0, 0); + nmask = ix86_build_signbit_mask (vmode, 0, 1); + mask = ix86_build_signbit_mask (vmode, 0, 0); if (mode == SFmode) copysign_insn = gen_copysignsf3_var; @@ -14437,7 +17139,7 @@ has source and destination with matching CC modes, and that the CC mode is at least as constrained as REQ_MODE. */ -int +bool ix86_match_ccmode (rtx insn, enum machine_mode req_mode) { rtx set; @@ -14456,19 +17158,19 @@ if (req_mode != CCNOmode && (req_mode != CCmode || XEXP (SET_SRC (set), 1) != const0_rtx)) - return 0; + return false; break; case CCmode: if (req_mode == CCGCmode) - return 0; + return false; /* FALLTHRU */ case CCGCmode: if (req_mode == CCGOCmode || req_mode == CCNOmode) - return 0; + return false; /* FALLTHRU */ case CCGOCmode: if (req_mode == CCZmode) - return 0; + return false; /* FALLTHRU */ case CCAmode: case CCCmode: @@ -14481,7 +17183,7 @@ gcc_unreachable (); } - return (GET_MODE (SET_SRC (set)) == set_mode); + return GET_MODE (SET_SRC (set)) == set_mode; } /* Generate insn patterns to do an integer compare of OPERANDS. */ @@ -14648,7 +17350,7 @@ } -/* Return a comparison we can do and that it is equivalent to +/* Return a comparison we can do and that it is equivalent to swap_condition (code) apart possibly from orderedness. But, never change orderedness if TARGET_IEEE_FP, returning UNKNOWN in that case if necessary. */ @@ -15022,15 +17724,13 @@ const0_rtx); } -rtx -ix86_expand_compare (enum rtx_code code) -{ - rtx op0, op1, ret; - op0 = ix86_compare_op0; - op1 = ix86_compare_op1; - - if (GET_MODE_CLASS (GET_MODE (ix86_compare_op0)) == MODE_CC) - ret = gen_rtx_fmt_ee (code, VOIDmode, ix86_compare_op0, ix86_compare_op1); +static rtx +ix86_expand_compare (enum rtx_code code, rtx op0, rtx op1) +{ + rtx ret; + + if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC) + ret = gen_rtx_fmt_ee (code, VOIDmode, op0, op1); else if (SCALAR_FLOAT_MODE_P (GET_MODE (op0))) { @@ -15044,11 +17744,12 @@ } void -ix86_expand_branch (enum rtx_code code, rtx label) -{ +ix86_expand_branch (enum rtx_code code, rtx op0, rtx op1, rtx label) +{ + enum machine_mode mode = GET_MODE (op0); rtx tmp; - switch (GET_MODE (ix86_compare_op0)) + switch (mode) { case SFmode: case DFmode: @@ -15057,7 +17758,7 @@ case HImode: case SImode: simple: - tmp = ix86_expand_compare (code); + tmp = ix86_expand_compare (code, op0, op1); tmp = gen_rtx_IF_THEN_ELSE (VOIDmode, tmp, gen_rtx_LABEL_REF (VOIDmode, label), pc_rtx); @@ -15074,25 +17775,16 @@ enum rtx_code code1, code2, code3; enum machine_mode submode; - if (CONSTANT_P (ix86_compare_op0) && ! CONSTANT_P (ix86_compare_op1)) + if (CONSTANT_P (op0) && !CONSTANT_P (op1)) { - tmp = ix86_compare_op0; - ix86_compare_op0 = ix86_compare_op1; - ix86_compare_op1 = tmp; + tmp = op0, op0 = op1, op1 = tmp; code = swap_condition (code); } - if (GET_MODE (ix86_compare_op0) == DImode) - { - split_di (&ix86_compare_op0, 1, lo+0, hi+0); - split_di (&ix86_compare_op1, 1, lo+1, hi+1); - submode = SImode; - } - else - { - split_ti (&ix86_compare_op0, 1, lo+0, hi+0); - split_ti (&ix86_compare_op1, 1, lo+1, hi+1); - submode = DImode; - } + + split_double_mode (mode, &op0, 1, lo+0, hi+0); + split_double_mode (mode, &op1, 1, lo+1, hi+1); + + submode = mode == DImode ? SImode : DImode; /* When comparing for equality, we can use (hi0^hi1)|(lo0^lo1) to avoid two branches. This costs one extra insn, so disable when @@ -15117,9 +17809,7 @@ tmp = expand_binop (submode, ior_optab, xor1, xor0, NULL_RTX, 0, OPTAB_WIDEN); - ix86_compare_op0 = tmp; - ix86_compare_op1 = const0_rtx; - ix86_expand_branch (code, label); + ix86_expand_branch (code, tmp, const0_rtx, label); return; } @@ -15134,18 +17824,14 @@ case LT: case LTU: case GE: case GEU: if (lo[1] == const0_rtx) { - ix86_compare_op0 = hi[0]; - ix86_compare_op1 = hi[1]; - ix86_expand_branch (code, label); + ix86_expand_branch (code, hi[0], hi[1], label); return; } break; case LE: case LEU: case GT: case GTU: if (lo[1] == constm1_rtx) { - ix86_compare_op0 = hi[0]; - ix86_compare_op1 = hi[1]; - ix86_expand_branch (code, label); + ix86_expand_branch (code, hi[0], hi[1], label); return; } break; @@ -15186,17 +17872,12 @@ * false: */ - ix86_compare_op0 = hi[0]; - ix86_compare_op1 = hi[1]; - if (code1 != UNKNOWN) - ix86_expand_branch (code1, label); + ix86_expand_branch (code1, hi[0], hi[1], label); if (code2 != UNKNOWN) - ix86_expand_branch (code2, label2); - - ix86_compare_op0 = lo[0]; - ix86_compare_op1 = lo[1]; - ix86_expand_branch (code3, label); + ix86_expand_branch (code2, hi[0], hi[1], label2); + + ix86_expand_branch (code3, lo[0], lo[1], label); if (code2 != UNKNOWN) emit_label (label2); @@ -15204,10 +17885,7 @@ } default: - /* If we have already emitted a compare insn, go straight to simple. - ix86_expand_compare won't emit anything if ix86_compare_emitted - is non NULL. */ - gcc_assert (GET_MODE_CLASS (GET_MODE (ix86_compare_op0)) == MODE_CC); + gcc_assert (GET_MODE_CLASS (GET_MODE (op0)) == MODE_CC); goto simple; } } @@ -15244,13 +17922,13 @@ } void -ix86_expand_setcc (enum rtx_code code, rtx dest) +ix86_expand_setcc (rtx dest, enum rtx_code code, rtx op0, rtx op1) { rtx ret; gcc_assert (GET_MODE (dest) == QImode); - ret = ix86_expand_compare (code); + ret = ix86_expand_compare (code, op0, op1); PUT_MODE (ret, QImode); emit_insn (gen_rtx_SET (VOIDmode, dest, ret)); } @@ -15263,7 +17941,7 @@ enum machine_mode mode = GET_MODE (op0) != VOIDmode ? GET_MODE (op0) : GET_MODE (op1); - /* Do not handle DImode compares that go through special path. */ + /* Do not handle double-mode compares that go through special path. */ if (mode == (TARGET_64BIT ? TImode : DImode)) return false; @@ -15378,32 +18056,30 @@ return false; op0 = force_reg (mode, op0); } - ix86_compare_op0 = op0; - ix86_compare_op1 = op1; - *pop = ix86_expand_compare (code); + *pop = ix86_expand_compare (code, op0, op1); gcc_assert (GET_CODE (*pop) == LTU || GET_CODE (*pop) == GEU); return true; } -int +bool ix86_expand_int_movcc (rtx operands[]) { enum rtx_code code = GET_CODE (operands[1]), compare_code; rtx compare_seq, compare_op; enum machine_mode mode = GET_MODE (operands[0]); - bool sign_bit_compare_p = false;; + bool sign_bit_compare_p = false; + rtx op0 = XEXP (operands[1], 0); + rtx op1 = XEXP (operands[1], 1); start_sequence (); - ix86_compare_op0 = XEXP (operands[1], 0); - ix86_compare_op1 = XEXP (operands[1], 1); - compare_op = ix86_expand_compare (code); + compare_op = ix86_expand_compare (code, op0, op1); compare_seq = get_insns (); end_sequence (); compare_code = GET_CODE (compare_op); - if ((ix86_compare_op1 == const0_rtx && (code == GE || code == LT)) - || (ix86_compare_op1 == constm1_rtx && (code == GT || code == LE))) + if ((op1 == const0_rtx && (code == GE || code == LT)) + || (op1 == constm1_rtx && (code == GT || code == LE))) sign_bit_compare_p = true; /* Don't attempt mode expansion here -- if we had to expand 5 or 6 @@ -15423,8 +18099,7 @@ /* Sign bit compares are better done using shifts than we do by using sbb. */ if (sign_bit_compare_p - || ix86_expand_carry_flag_compare (code, ix86_compare_op0, - ix86_compare_op1, &compare_op)) + || ix86_expand_carry_flag_compare (code, op0, op1, &compare_op)) { /* Detect overlap between destination and compare sources. */ rtx tmp = out; @@ -15432,7 +18107,6 @@ if (!sign_bit_compare_p) { rtx flags; - rtx (*insn)(rtx, rtx, rtx); bool fpcmp = false; compare_code = GET_CODE (compare_op); @@ -15468,16 +18142,15 @@ } diff = ct - cf; - if (reg_overlap_mentioned_p (out, ix86_compare_op0) - || reg_overlap_mentioned_p (out, ix86_compare_op1)) + if (reg_overlap_mentioned_p (out, op0) + || reg_overlap_mentioned_p (out, op1)) tmp = gen_reg_rtx (mode); if (mode == DImode) - insn = gen_x86_movdicc_0_m1; + emit_insn (gen_x86_movdicc_0_m1 (tmp, flags, compare_op)); else - insn = gen_x86_movsicc_0_m1; - - emit_insn (insn (tmp, flags, compare_op)); + emit_insn (gen_x86_movsicc_0_m1 (gen_lowpart (SImode, tmp), + flags, compare_op)); } else { @@ -15490,8 +18163,7 @@ cf = tmp; diff = ct - cf; } - tmp = emit_store_flag (tmp, code, ix86_compare_op0, - ix86_compare_op1, VOIDmode, 0, -1); + tmp = emit_store_flag (tmp, code, op0, op1, VOIDmode, 0, -1); } if (diff == 1) @@ -15569,12 +18241,12 @@ if (!rtx_equal_p (tmp, out)) emit_move_insn (copy_rtx (out), copy_rtx (tmp)); - return 1; /* DONE */ + return true; } if (diff < 0) { - enum machine_mode cmp_mode = GET_MODE (ix86_compare_op0); + enum machine_mode cmp_mode = GET_MODE (op0); HOST_WIDE_INT tmp; tmp = ct, ct = cf, cf = tmp; @@ -15599,13 +18271,13 @@ } compare_code = UNKNOWN; - if (GET_MODE_CLASS (GET_MODE (ix86_compare_op0)) == MODE_INT - && CONST_INT_P (ix86_compare_op1)) - { - if (ix86_compare_op1 == const0_rtx + if (GET_MODE_CLASS (GET_MODE (op0)) == MODE_INT + && CONST_INT_P (op1)) + { + if (op1 == const0_rtx && (code == LT || code == GE)) compare_code = code; - else if (ix86_compare_op1 == constm1_rtx) + else if (op1 == constm1_rtx) { if (code == LE) compare_code = LT; @@ -15616,7 +18288,7 @@ /* Optimize dest = (op0 < 0) ? -1 : cf. */ if (compare_code != UNKNOWN - && GET_MODE (ix86_compare_op0) == GET_MODE (out) + && GET_MODE (op0) == GET_MODE (out) && (cf == -1 || ct == -1)) { /* If lea code below could be used, only optimize @@ -15639,8 +18311,7 @@ code = reverse_condition (code); } - out = emit_store_flag (out, code, ix86_compare_op0, - ix86_compare_op1, VOIDmode, 0, -1); + out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1); out = expand_simple_binop (mode, IOR, out, GEN_INT (cf), @@ -15648,7 +18319,7 @@ if (out != operands[0]) emit_move_insn (operands[0], out); - return 1; /* DONE */ + return true; } } @@ -15673,8 +18344,7 @@ rtx tmp; int nops; - out = emit_store_flag (out, code, ix86_compare_op0, - ix86_compare_op1, VOIDmode, 0, 1); + out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1); nops = 0; /* On x86_64 the lea instruction operates on Pmode, so we need @@ -15708,7 +18378,7 @@ if (!rtx_equal_p (out, operands[0])) emit_move_insn (operands[0], copy_rtx (out)); - return 1; /* DONE */ + return true; } /* @@ -15733,7 +18403,7 @@ { if (cf == 0) { - enum machine_mode cmp_mode = GET_MODE (ix86_compare_op0); + enum machine_mode cmp_mode = GET_MODE (op0); cf = ct; ct = 0; @@ -15782,15 +18452,14 @@ ct = tmp; } - out = emit_store_flag (out, code, ix86_compare_op0, - ix86_compare_op1, VOIDmode, 0, -1); - } - else - { - out = emit_store_flag (out, code, ix86_compare_op0, - ix86_compare_op1, VOIDmode, 0, 1); - - out = expand_simple_binop (mode, PLUS, copy_rtx (out), constm1_rtx, + out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, -1); + } + else + { + out = emit_store_flag (out, code, op0, op1, VOIDmode, 0, 1); + + out = expand_simple_binop (mode, PLUS, copy_rtx (out), + constm1_rtx, copy_rtx (out), 1, OPTAB_DIRECT); } @@ -15803,7 +18472,7 @@ if (!rtx_equal_p (out, operands[0])) emit_move_insn (operands[0], copy_rtx (out)); - return 1; /* DONE */ + return true; } } @@ -15815,7 +18484,7 @@ rtx var, orig_out, out, tmp; if (BRANCH_COST (optimize_insn_for_speed_p (), false) <= 2) - return 0; /* FAIL */ + return false; /* If one of the two operands is an interesting constant, load a constant with the above and mask it in with a logical operation. */ @@ -15828,7 +18497,7 @@ else if (INTVAL (operands[2]) == -1 && operands[3] != const0_rtx) operands[3] = const0_rtx, op = ior_optab; else - return 0; /* FAIL */ + return false; } else if (CONST_INT_P (operands[3])) { @@ -15838,10 +18507,10 @@ else if (INTVAL (operands[3]) == -1 && operands[3] != const0_rtx) operands[2] = const0_rtx, op = ior_optab; else - return 0; /* FAIL */ - } - else - return 0; /* FAIL */ + return false; + } + else + return false; orig_out = operands[0]; tmp = gen_reg_rtx (mode); @@ -15849,7 +18518,7 @@ /* Recurse to get the constant loaded. */ if (ix86_expand_int_movcc (operands) == 0) - return 0; /* FAIL */ + return false; /* Mask in the interesting variable. */ out = expand_binop (mode, op, var, tmp, orig_out, 0, @@ -15857,7 +18526,7 @@ if (!rtx_equal_p (out, orig_out)) emit_move_insn (copy_rtx (orig_out), copy_rtx (out)); - return 1; /* DONE */ + return true; } /* @@ -15890,8 +18559,7 @@ gen_rtx_IF_THEN_ELSE (mode, compare_op, operands[2], operands[3]))); - - return 1; /* DONE */ + return true; } /* Swap, force into registers, or otherwise massage the two operands @@ -16089,15 +18757,15 @@ /* Expand a floating-point conditional move. Return true if successful. */ -int +bool ix86_expand_fp_movcc (rtx operands[]) { enum machine_mode mode = GET_MODE (operands[0]); enum rtx_code code = GET_CODE (operands[1]); rtx tmp, compare_op; - - ix86_compare_op0 = XEXP (operands[1], 0); - ix86_compare_op1 = XEXP (operands[1], 1); + rtx op0 = XEXP (operands[1], 0); + rtx op1 = XEXP (operands[1], 1); + if (TARGET_SSE_MATH && SSE_FLOAT_MODE_P (mode)) { enum machine_mode cmode; @@ -16105,48 +18773,43 @@ /* Since we've no cmove for sse registers, don't force bad register allocation just to gain access to it. Deny movcc when the comparison mode doesn't match the move mode. */ - cmode = GET_MODE (ix86_compare_op0); + cmode = GET_MODE (op0); if (cmode == VOIDmode) - cmode = GET_MODE (ix86_compare_op1); + cmode = GET_MODE (op1); if (cmode != mode) - return 0; - - code = ix86_prepare_sse_fp_compare_args (operands[0], code, - &ix86_compare_op0, - &ix86_compare_op1); + return false; + + code = ix86_prepare_sse_fp_compare_args (operands[0], code, &op0, &op1); if (code == UNKNOWN) - return 0; - - if (ix86_expand_sse_fp_minmax (operands[0], code, ix86_compare_op0, - ix86_compare_op1, operands[2], - operands[3])) - return 1; - - tmp = ix86_expand_sse_cmp (operands[0], code, ix86_compare_op0, - ix86_compare_op1, operands[2], operands[3]); + return false; + + if (ix86_expand_sse_fp_minmax (operands[0], code, op0, op1, + operands[2], operands[3])) + return true; + + tmp = ix86_expand_sse_cmp (operands[0], code, op0, op1, + operands[2], operands[3]); ix86_expand_sse_movcc (operands[0], tmp, operands[2], operands[3]); - return 1; + return true; } /* The floating point conditional move instructions don't directly support conditions resulting from a signed integer comparison. */ - compare_op = ix86_expand_compare (code); + compare_op = ix86_expand_compare (code, op0, op1); if (!fcmov_comparison_operator (compare_op, VOIDmode)) { tmp = gen_reg_rtx (QImode); - ix86_expand_setcc (code, tmp); - code = NE; - ix86_compare_op0 = tmp; - ix86_compare_op1 = const0_rtx; - compare_op = ix86_expand_compare (code); + ix86_expand_setcc (tmp, code, op0, op1); + + compare_op = ix86_expand_compare (NE, tmp, const0_rtx); } emit_insn (gen_rtx_SET (VOIDmode, operands[0], gen_rtx_IF_THEN_ELSE (mode, compare_op, operands[2], operands[3]))); - return 1; + return true; } /* Expand a floating-point vector conditional move; a vcond operation @@ -16256,29 +18919,21 @@ case V2DImode: { rtx t1, t2, mask; - - /* Perform a parallel modulo subtraction. */ + rtx (*gen_sub3) (rtx, rtx, rtx); + + /* Subtract (-(INT MAX) - 1) from both operands to make + them signed. */ + mask = ix86_build_signbit_mask (mode, true, false); + gen_sub3 = (mode == V4SImode + ? gen_subv4si3 : gen_subv2di3); t1 = gen_reg_rtx (mode); - emit_insn ((mode == V4SImode - ? gen_subv4si3 - : gen_subv2di3) (t1, cop0, cop1)); - - /* Extract the original sign bit of op0. */ - mask = ix86_build_signbit_mask (GET_MODE_INNER (mode), - true, false); + emit_insn (gen_sub3 (t1, cop0, mask)); + t2 = gen_reg_rtx (mode); - emit_insn ((mode == V4SImode - ? gen_andv4si3 - : gen_andv2di3) (t2, cop0, mask)); - - /* XOR it back into the result of the subtraction. - This results in the sign bit set iff we saw - unsigned underflow. */ - x = gen_reg_rtx (mode); - emit_insn ((mode == V4SImode - ? gen_xorv4si3 - : gen_xorv2di3) (x, t1, t2)); - + emit_insn (gen_sub3 (t2, cop1, mask)); + + cop0 = t1; + cop1 = t2; code = GT; } break; @@ -16290,6 +18945,8 @@ emit_insn (gen_rtx_SET (VOIDmode, x, gen_rtx_US_MINUS (mode, cop0, cop1))); + cop0 = x; + cop1 = CONST0_RTX (mode); code = EQ; negate = !negate; break; @@ -16297,9 +18954,6 @@ default: gcc_unreachable (); } - - cop0 = x; - cop1 = CONST0_RTX (mode); } } @@ -16373,19 +19027,19 @@ if (unsigned_p) unpack = gen_sse4_1_zero_extendv8qiv8hi2; else - unpack = gen_sse4_1_extendv8qiv8hi2; + unpack = gen_sse4_1_sign_extendv8qiv8hi2; break; case V8HImode: if (unsigned_p) unpack = gen_sse4_1_zero_extendv4hiv4si2; else - unpack = gen_sse4_1_extendv4hiv4si2; + unpack = gen_sse4_1_sign_extendv4hiv4si2; break; case V4SImode: if (unsigned_p) unpack = gen_sse4_1_zero_extendv2siv2di2; else - unpack = gen_sse4_1_extendv2siv2di2; + unpack = gen_sse4_1_sign_extendv2siv2di2; break; default: gcc_unreachable (); @@ -16409,7 +19063,7 @@ /* Expand conditional increment or decrement using adb/sbb instructions. The default case using setcc followed by the conditional move can be done by generic code. */ -int +bool ix86_expand_int_addcc (rtx operands[]) { enum rtx_code code = GET_CODE (operands[1]); @@ -16419,15 +19073,14 @@ rtx val = const0_rtx; bool fpcmp = false; enum machine_mode mode; - - ix86_compare_op0 = XEXP (operands[1], 0); - ix86_compare_op1 = XEXP (operands[1], 1); + rtx op0 = XEXP (operands[1], 0); + rtx op1 = XEXP (operands[1], 1); + if (operands[3] != const1_rtx && operands[3] != constm1_rtx) - return 0; - if (!ix86_expand_carry_flag_compare (code, ix86_compare_op0, - ix86_compare_op1, &compare_op)) - return 0; + return false; + if (!ix86_expand_carry_flag_compare (code, op0, op1, &compare_op)) + return false; code = GET_CODE (compare_op); flags = XEXP (compare_op, 0); @@ -16495,12 +19148,12 @@ } emit_insn (insn (operands[0], operands[2], val, flags, compare_op)); - return 1; /* DONE */ -} - - -/* Split operands 0 and 1 into SImode parts. Similar to split_di, but - works for floating pointer parameters and nonoffsetable memories. + return true; +} + + +/* Split operands 0 and 1 into half-mode parts. Similar to split_double_mode, + but works for floating pointer parameters and nonoffsetable memories. For pushes, it returns just stack offsets; the values will be saved in the right order. Maximally three parts are generated. */ @@ -16553,7 +19206,7 @@ if (!TARGET_64BIT) { if (mode == DImode) - split_di (&operand, 1, &parts[0], &parts[1]); + split_double_mode (mode, &operand, 1, &parts[0], &parts[1]); else { int i; @@ -16604,7 +19257,7 @@ else { if (mode == TImode) - split_ti (&operand, 1, &parts[0], &parts[1]); + split_double_mode (mode, &operand, 1, &parts[0], &parts[1]); if (mode == XFmode || mode == TFmode) { enum machine_mode upper_mode = mode==XFmode ? SImode : DImode; @@ -16675,7 +19328,7 @@ /* The DFmode expanders may ask us to move double. For 64bit target this is single move. By hiding the fact here we simplify i386.md splitters. */ - if (GET_MODE_SIZE (GET_MODE (operands[0])) == 8 && TARGET_64BIT) + if (TARGET_64BIT && GET_MODE_SIZE (GET_MODE (operands[0])) == 8) { /* Optimize constant pool reference to immediates. This is used by fp moves, that force all constants to memory to allow combining. */ @@ -16885,62 +19538,62 @@ static void ix86_expand_ashl_const (rtx operand, int count, enum machine_mode mode) { - if (count == 1) - { - emit_insn ((mode == DImode - ? gen_addsi3 - : gen_adddi3) (operand, operand, operand)); - } - else if (!optimize_insn_for_size_p () - && count * ix86_cost->add <= ix86_cost->shift_const) - { - int i; - for (i=0; i<count; i++) - { - emit_insn ((mode == DImode - ? gen_addsi3 - : gen_adddi3) (operand, operand, operand)); - } - } - else - emit_insn ((mode == DImode - ? gen_ashlsi3 - : gen_ashldi3) (operand, operand, GEN_INT (count))); + rtx (*insn)(rtx, rtx, rtx); + + if (count == 1 + || (count * ix86_cost->add <= ix86_cost->shift_const + && !optimize_insn_for_size_p ())) + { + insn = mode == DImode ? gen_addsi3 : gen_adddi3; + while (count-- > 0) + emit_insn (insn (operand, operand, operand)); + } + else + { + insn = mode == DImode ? gen_ashlsi3 : gen_ashldi3; + emit_insn (insn (operand, operand, GEN_INT (count))); + } } void ix86_split_ashl (rtx *operands, rtx scratch, enum machine_mode mode) { + rtx (*gen_ashl3)(rtx, rtx, rtx); + rtx (*gen_shld)(rtx, rtx, rtx); + int half_width = GET_MODE_BITSIZE (mode) >> 1; + rtx low[2], high[2]; int count; - const int single_width = mode == DImode ? 32 : 64; if (CONST_INT_P (operands[2])) { - (mode == DImode ? split_di : split_ti) (operands, 2, low, high); - count = INTVAL (operands[2]) & (single_width * 2 - 1); - - if (count >= single_width) + split_double_mode (mode, operands, 2, low, high); + count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1); + + if (count >= half_width) { emit_move_insn (high[0], low[1]); emit_move_insn (low[0], const0_rtx); - if (count > single_width) - ix86_expand_ashl_const (high[0], count - single_width, mode); - } - else - { + if (count > half_width) + ix86_expand_ashl_const (high[0], count - half_width, mode); + } + else + { + gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld; + if (!rtx_equal_p (operands[0], operands[1])) emit_move_insn (operands[0], operands[1]); - emit_insn ((mode == DImode - ? gen_x86_shld - : gen_x86_64_shld) (high[0], low[0], GEN_INT (count))); + + emit_insn (gen_shld (high[0], low[0], GEN_INT (count))); ix86_expand_ashl_const (low[0], count, mode); } return; } - (mode == DImode ? split_di : split_ti) (operands, 1, low, high); + split_double_mode (mode, operands, 1, low, high); + + gen_ashl3 = mode == DImode ? gen_ashlsi3 : gen_ashldi3; if (operands[1] == const1_rtx) { @@ -16952,7 +19605,7 @@ ix86_expand_clear (low[0]); ix86_expand_clear (high[0]); - emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (single_width))); + emit_insn (gen_testqi_ccz_1 (operands[2], GEN_INT (half_width))); d = gen_lowpart (QImode, low[0]); d = gen_rtx_STRICT_LOW_PART (VOIDmode, d); @@ -16972,33 +19625,44 @@ pentium4 a bit; no one else seems to care much either way. */ else { + enum machine_mode half_mode; + rtx (*gen_lshr3)(rtx, rtx, rtx); + rtx (*gen_and3)(rtx, rtx, rtx); + rtx (*gen_xor3)(rtx, rtx, rtx); + HOST_WIDE_INT bits; rtx x; + if (mode == DImode) + { + half_mode = SImode; + gen_lshr3 = gen_lshrsi3; + gen_and3 = gen_andsi3; + gen_xor3 = gen_xorsi3; + bits = 5; + } + else + { + half_mode = DImode; + gen_lshr3 = gen_lshrdi3; + gen_and3 = gen_anddi3; + gen_xor3 = gen_xordi3; + bits = 6; + } + if (TARGET_PARTIAL_REG_STALL && !optimize_insn_for_size_p ()) - x = gen_rtx_ZERO_EXTEND (mode == DImode ? SImode : DImode, operands[2]); - else - x = gen_lowpart (mode == DImode ? SImode : DImode, operands[2]); + x = gen_rtx_ZERO_EXTEND (half_mode, operands[2]); + else + x = gen_lowpart (half_mode, operands[2]); emit_insn (gen_rtx_SET (VOIDmode, high[0], x)); - emit_insn ((mode == DImode - ? gen_lshrsi3 - : gen_lshrdi3) (high[0], high[0], - GEN_INT (mode == DImode ? 5 : 6))); - emit_insn ((mode == DImode - ? gen_andsi3 - : gen_anddi3) (high[0], high[0], const1_rtx)); + emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (bits))); + emit_insn (gen_and3 (high[0], high[0], const1_rtx)); emit_move_insn (low[0], high[0]); - emit_insn ((mode == DImode - ? gen_xorsi3 - : gen_xordi3) (low[0], low[0], const1_rtx)); - } - - emit_insn ((mode == DImode - ? gen_ashlsi3 - : gen_ashldi3) (low[0], low[0], operands[2])); - emit_insn ((mode == DImode - ? gen_ashlsi3 - : gen_ashldi3) (high[0], high[0], operands[2])); + emit_insn (gen_xor3 (low[0], low[0], const1_rtx)); + } + + emit_insn (gen_ashl3 (low[0], low[0], operands[2])); + emit_insn (gen_ashl3 (high[0], high[0], operands[2])); return; } @@ -17014,174 +19678,177 @@ } else { + gen_shld = mode == DImode ? gen_x86_shld : gen_x86_64_shld; + if (!rtx_equal_p (operands[0], operands[1])) emit_move_insn (operands[0], operands[1]); - (mode == DImode ? split_di : split_ti) (operands, 1, low, high); - emit_insn ((mode == DImode - ? gen_x86_shld - : gen_x86_64_shld) (high[0], low[0], operands[2])); - } - - emit_insn ((mode == DImode ? gen_ashlsi3 : gen_ashldi3) (low[0], low[0], operands[2])); + split_double_mode (mode, operands, 1, low, high); + emit_insn (gen_shld (high[0], low[0], operands[2])); + } + + emit_insn (gen_ashl3 (low[0], low[0], operands[2])); if (TARGET_CMOVE && scratch) { + rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx) + = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1; + ix86_expand_clear (scratch); - emit_insn ((mode == DImode - ? gen_x86_shift_adj_1 - : gen_x86_64_shift_adj_1) (high[0], low[0], operands[2], - scratch)); - } - else - emit_insn ((mode == DImode - ? gen_x86_shift_adj_2 - : gen_x86_64_shift_adj_2) (high[0], low[0], operands[2])); + emit_insn (gen_x86_shift_adj_1 (high[0], low[0], operands[2], scratch)); + } + else + { + rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx) + = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2; + + emit_insn (gen_x86_shift_adj_2 (high[0], low[0], operands[2])); + } } void ix86_split_ashr (rtx *operands, rtx scratch, enum machine_mode mode) { + rtx (*gen_ashr3)(rtx, rtx, rtx) + = mode == DImode ? gen_ashrsi3 : gen_ashrdi3; + rtx (*gen_shrd)(rtx, rtx, rtx); + int half_width = GET_MODE_BITSIZE (mode) >> 1; + rtx low[2], high[2]; int count; - const int single_width = mode == DImode ? 32 : 64; if (CONST_INT_P (operands[2])) { - (mode == DImode ? split_di : split_ti) (operands, 2, low, high); - count = INTVAL (operands[2]) & (single_width * 2 - 1); - - if (count == single_width * 2 - 1) + split_double_mode (mode, operands, 2, low, high); + count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1); + + if (count == GET_MODE_BITSIZE (mode) - 1) { emit_move_insn (high[0], high[1]); - emit_insn ((mode == DImode - ? gen_ashrsi3 - : gen_ashrdi3) (high[0], high[0], - GEN_INT (single_width - 1))); + emit_insn (gen_ashr3 (high[0], high[0], + GEN_INT (half_width - 1))); emit_move_insn (low[0], high[0]); } - else if (count >= single_width) + else if (count >= half_width) { emit_move_insn (low[0], high[1]); emit_move_insn (high[0], low[0]); - emit_insn ((mode == DImode - ? gen_ashrsi3 - : gen_ashrdi3) (high[0], high[0], - GEN_INT (single_width - 1))); - if (count > single_width) - emit_insn ((mode == DImode - ? gen_ashrsi3 - : gen_ashrdi3) (low[0], low[0], - GEN_INT (count - single_width))); - } - else - { + emit_insn (gen_ashr3 (high[0], high[0], + GEN_INT (half_width - 1))); + + if (count > half_width) + emit_insn (gen_ashr3 (low[0], low[0], + GEN_INT (count - half_width))); + } + else + { + gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd; + if (!rtx_equal_p (operands[0], operands[1])) emit_move_insn (operands[0], operands[1]); - emit_insn ((mode == DImode - ? gen_x86_shrd - : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count))); - emit_insn ((mode == DImode - ? gen_ashrsi3 - : gen_ashrdi3) (high[0], high[0], GEN_INT (count))); - } - } - else - { - if (!rtx_equal_p (operands[0], operands[1])) + + emit_insn (gen_shrd (low[0], high[0], GEN_INT (count))); + emit_insn (gen_ashr3 (high[0], high[0], GEN_INT (count))); + } + } + else + { + gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd; + + if (!rtx_equal_p (operands[0], operands[1])) emit_move_insn (operands[0], operands[1]); - (mode == DImode ? split_di : split_ti) (operands, 1, low, high); - - emit_insn ((mode == DImode - ? gen_x86_shrd - : gen_x86_64_shrd) (low[0], high[0], operands[2])); - emit_insn ((mode == DImode - ? gen_ashrsi3 - : gen_ashrdi3) (high[0], high[0], operands[2])); + split_double_mode (mode, operands, 1, low, high); + + emit_insn (gen_shrd (low[0], high[0], operands[2])); + emit_insn (gen_ashr3 (high[0], high[0], operands[2])); if (TARGET_CMOVE && scratch) { + rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx) + = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1; + emit_move_insn (scratch, high[0]); - emit_insn ((mode == DImode - ? gen_ashrsi3 - : gen_ashrdi3) (scratch, scratch, - GEN_INT (single_width - 1))); - emit_insn ((mode == DImode - ? gen_x86_shift_adj_1 - : gen_x86_64_shift_adj_1) (low[0], high[0], operands[2], - scratch)); - } - else - emit_insn ((mode == DImode - ? gen_x86_shift_adj_3 - : gen_x86_64_shift_adj_3) (low[0], high[0], operands[2])); + emit_insn (gen_ashr3 (scratch, scratch, + GEN_INT (half_width - 1))); + emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2], + scratch)); + } + else + { + rtx (*gen_x86_shift_adj_3)(rtx, rtx, rtx) + = mode == DImode ? gen_x86_shiftsi_adj_3 : gen_x86_shiftdi_adj_3; + + emit_insn (gen_x86_shift_adj_3 (low[0], high[0], operands[2])); + } } } void ix86_split_lshr (rtx *operands, rtx scratch, enum machine_mode mode) { + rtx (*gen_lshr3)(rtx, rtx, rtx) + = mode == DImode ? gen_lshrsi3 : gen_lshrdi3; + rtx (*gen_shrd)(rtx, rtx, rtx); + int half_width = GET_MODE_BITSIZE (mode) >> 1; + rtx low[2], high[2]; int count; - const int single_width = mode == DImode ? 32 : 64; if (CONST_INT_P (operands[2])) { - (mode == DImode ? split_di : split_ti) (operands, 2, low, high); - count = INTVAL (operands[2]) & (single_width * 2 - 1); - - if (count >= single_width) + split_double_mode (mode, operands, 2, low, high); + count = INTVAL (operands[2]) & (GET_MODE_BITSIZE (mode) - 1); + + if (count >= half_width) { emit_move_insn (low[0], high[1]); ix86_expand_clear (high[0]); - if (count > single_width) - emit_insn ((mode == DImode - ? gen_lshrsi3 - : gen_lshrdi3) (low[0], low[0], - GEN_INT (count - single_width))); - } - else - { + if (count > half_width) + emit_insn (gen_lshr3 (low[0], low[0], + GEN_INT (count - half_width))); + } + else + { + gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd; + if (!rtx_equal_p (operands[0], operands[1])) emit_move_insn (operands[0], operands[1]); - emit_insn ((mode == DImode - ? gen_x86_shrd - : gen_x86_64_shrd) (low[0], high[0], GEN_INT (count))); - emit_insn ((mode == DImode - ? gen_lshrsi3 - : gen_lshrdi3) (high[0], high[0], GEN_INT (count))); - } - } - else - { + + emit_insn (gen_shrd (low[0], high[0], GEN_INT (count))); + emit_insn (gen_lshr3 (high[0], high[0], GEN_INT (count))); + } + } + else + { + gen_shrd = mode == DImode ? gen_x86_shrd : gen_x86_64_shrd; + if (!rtx_equal_p (operands[0], operands[1])) emit_move_insn (operands[0], operands[1]); - (mode == DImode ? split_di : split_ti) (operands, 1, low, high); - - emit_insn ((mode == DImode - ? gen_x86_shrd - : gen_x86_64_shrd) (low[0], high[0], operands[2])); - emit_insn ((mode == DImode - ? gen_lshrsi3 - : gen_lshrdi3) (high[0], high[0], operands[2])); - - /* Heh. By reversing the arguments, we can reuse this pattern. */ + split_double_mode (mode, operands, 1, low, high); + + emit_insn (gen_shrd (low[0], high[0], operands[2])); + emit_insn (gen_lshr3 (high[0], high[0], operands[2])); + if (TARGET_CMOVE && scratch) { + rtx (*gen_x86_shift_adj_1)(rtx, rtx, rtx, rtx) + = mode == DImode ? gen_x86_shiftsi_adj_1 : gen_x86_shiftdi_adj_1; + ix86_expand_clear (scratch); - emit_insn ((mode == DImode - ? gen_x86_shift_adj_1 - : gen_x86_64_shift_adj_1) (low[0], high[0], operands[2], - scratch)); - } - else - emit_insn ((mode == DImode - ? gen_x86_shift_adj_2 - : gen_x86_64_shift_adj_2) (low[0], high[0], operands[2])); + emit_insn (gen_x86_shift_adj_1 (low[0], high[0], operands[2], + scratch)); + } + else + { + rtx (*gen_x86_shift_adj_2)(rtx, rtx, rtx) + = mode == DImode ? gen_x86_shiftsi_adj_2 : gen_x86_shiftdi_adj_2; + + emit_insn (gen_x86_shift_adj_2 (low[0], high[0], operands[2])); + } } } @@ -17218,10 +19885,10 @@ static void ix86_adjust_counter (rtx countreg, HOST_WIDE_INT value) { - if (GET_MODE (countreg) == DImode) - emit_insn (gen_adddi3 (countreg, countreg, GEN_INT (-value))); - else - emit_insn (gen_addsi3 (countreg, countreg, GEN_INT (-value))); + rtx (*gen_add)(rtx, rtx, rtx) + = GET_MODE (countreg) == DImode ? gen_adddi3 : gen_addsi3; + + emit_insn (gen_add (countreg, countreg, GEN_INT (-value))); } /* Zero extend possibly SImode EXP to Pmode register. */ @@ -18002,7 +20669,7 @@ && alg != rep_prefix_4_byte \ && alg != rep_prefix_8_byte)) const struct processor_costs *cost; - + /* Even if the string operation call is cold, we still might spend a lot of time processing large blocks. */ if (optimize_function_for_size_p (cfun) @@ -18037,7 +20704,7 @@ { unsigned int i; enum stringop_alg alg = libcall; - for (i = 0; i < NAX_STRINGOP_ALGS; i++) + for (i = 0; i < MAX_STRINGOP_ALGS; i++) { /* We get here if the algorithms that were not libcall-based were rep-prefix based and we are unable to use rep prefixes @@ -18083,7 +20750,7 @@ int i; bool any_alg_usable_p = true; - for (i = 0; i < NAX_STRINGOP_ALGS; i++) + for (i = 0; i < MAX_STRINGOP_ALGS; i++) { enum stringop_alg candidate = algs->size[i].alg; any_alg_usable_p = any_alg_usable_p && ALG_USABLE_P (candidate); @@ -18200,7 +20867,7 @@ 4) Epilogue: code copying tail of the block that is too small to be handled by main body (or up to size guarded by prologue guard). */ -int +bool ix86_expand_movmem (rtx dst, rtx src, rtx count_exp, rtx align_exp, rtx expected_align_exp, rtx expected_size_exp) { @@ -18236,7 +20903,7 @@ /* Make sure we don't need to care about overflow later on. */ if (count > ((unsigned HOST_WIDE_INT) 1 << 30)) - return 0; + return false; /* Step 0: Decide on preferred algorithm, desired alignment and size of chunks to be copied by main loop. */ @@ -18248,7 +20915,7 @@ align = desired_align; if (alg == libcall) - return 0; + return false; gcc_assert (alg != no_stringop); if (!count) count_exp = copy_to_mode_reg (GET_MODE (count_exp), count_exp); @@ -18494,7 +21161,7 @@ epilogue_size_needed); if (jump_around_label) emit_label (jump_around_label); - return 1; + return true; } /* Helper function for memcpy. For QImode value 0xXY produce @@ -18549,7 +21216,7 @@ if (mode == SImode) emit_insn (gen_movsi_insv_1 (reg, reg)); else - emit_insn (gen_movdi_insv_1_rex64 (reg, reg)); + emit_insn (gen_movdi_insv_1 (reg, reg)); else { tmp = expand_simple_binop (mode, ASHIFT, reg, GEN_INT (8), @@ -18593,7 +21260,7 @@ /* Expand string clear operation (bzero). Use i386 string operations when profitable. See expand_movmem comment for explanation of individual steps performed. */ -int +bool ix86_expand_setmem (rtx dst, rtx count_exp, rtx val_exp, rtx align_exp, rtx expected_align_exp, rtx expected_size_exp) { @@ -18625,7 +21292,7 @@ /* Make sure we don't need to care about overflow later on. */ if (count > ((unsigned HOST_WIDE_INT) 1 << 30)) - return 0; + return false; /* Step 0: Decide on preferred algorithm, desired alignment and size of chunks to be copied by main loop. */ @@ -18637,7 +21304,7 @@ align = desired_align; if (alg == libcall) - return 0; + return false; gcc_assert (alg != no_stringop); if (!count) count_exp = copy_to_mode_reg (counter_mode (count_exp), count_exp); @@ -18883,7 +21550,7 @@ } if (jump_around_label) emit_label (jump_around_label); - return 1; + return true; } /* Expand the appropriate insns for doing strlen if not just doing @@ -18960,7 +21627,7 @@ QImode, 1, end_0_label); /* Increment the address. */ - emit_insn ((*ix86_gen_add3) (out, out, const1_rtx)); + emit_insn (ix86_gen_add3 (out, out, const1_rtx)); /* Not needed with an alignment of 2 */ if (align != 2) @@ -18970,7 +21637,7 @@ emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1, end_0_label); - emit_insn ((*ix86_gen_add3) (out, out, const1_rtx)); + emit_insn (ix86_gen_add3 (out, out, const1_rtx)); emit_label (align_3_label); } @@ -18978,7 +21645,7 @@ emit_cmp_and_jump_insns (mem, const0_rtx, EQ, NULL, QImode, 1, end_0_label); - emit_insn ((*ix86_gen_add3) (out, out, const1_rtx)); + emit_insn (ix86_gen_add3 (out, out, const1_rtx)); } /* Generate loop to check 4 bytes at a time. It is not a good idea to @@ -18988,7 +21655,7 @@ mem = change_address (src, SImode, out); emit_move_insn (scratch, mem); - emit_insn ((*ix86_gen_add3) (out, out, GEN_INT (4))); + emit_insn (ix86_gen_add3 (out, out, GEN_INT (4))); /* This formula yields a nonzero result iff one of the bytes is zero. This saves three branches inside loop and many cycles. */ @@ -19043,7 +21710,7 @@ /* Not in the first two. Move two bytes forward. */ emit_insn (gen_lshrsi3 (tmpreg, tmpreg, GEN_INT (16))); - emit_insn ((*ix86_gen_add3) (out, out, const2_rtx)); + emit_insn (ix86_gen_add3 (out, out, const2_rtx)); emit_label (end_2_label); @@ -19054,14 +21721,14 @@ emit_insn (gen_addqi3_cc (tmpreg, tmpreg, tmpreg)); tmp = gen_rtx_REG (CCmode, FLAGS_REG); cmp = gen_rtx_LTU (VOIDmode, tmp, const0_rtx); - emit_insn ((*ix86_gen_sub3_carry) (out, out, GEN_INT (3), tmp, cmp)); + emit_insn (ix86_gen_sub3_carry (out, out, GEN_INT (3), tmp, cmp)); emit_label (end_0_label); } /* Expand strlen. */ -int +bool ix86_expand_strlen (rtx out, rtx src, rtx eoschar, rtx align) { rtx addr, scratch1, scratch2, scratch3, scratch4; @@ -19073,7 +21740,7 @@ && !TARGET_INLINE_ALL_STRINGOPS && !optimize_insn_for_size_p () && (!CONST_INT_P (align) || INTVAL (align) < 4)) - return 0; + return false; addr = force_reg (Pmode, XEXP (src, 0)); scratch1 = gen_reg_rtx (Pmode); @@ -19096,7 +21763,7 @@ /* strlensi_unroll_1 returns the address of the zero at the end of the string, like memchr(), so compute the length by subtracting the start address. */ - emit_insn ((*ix86_gen_sub3) (out, out, addr)); + emit_insn (ix86_gen_sub3 (out, out, addr)); } else { @@ -19119,10 +21786,10 @@ unspec = gen_rtx_UNSPEC (Pmode, gen_rtvec (4, src, eoschar, align, scratch4), UNSPEC_SCAS); emit_insn (gen_strlenqi_1 (scratch1, scratch3, unspec)); - emit_insn ((*ix86_gen_one_cmpl2) (scratch2, scratch1)); - emit_insn ((*ix86_gen_add3) (out, scratch2, constm1_rtx)); - } - return 1; + emit_insn (ix86_gen_one_cmpl2 (scratch2, scratch1)); + emit_insn (ix86_gen_add3 (out, scratch2, constm1_rtx)); + } + return true; } /* For given symbol (function) construct code to compute address of it's PLT @@ -19141,7 +21808,7 @@ return tmp; } -void +rtx ix86_expand_call (rtx retval, rtx fnaddr, rtx callarg1, rtx callarg2, rtx pop, int sibcall) @@ -19176,7 +21843,7 @@ } if (ix86_cmodel == CM_LARGE_PIC - && MEM_P (fnaddr) + && MEM_P (fnaddr) && GET_CODE (XEXP (fnaddr, 0)) == SYMBOL_REF && !local_symbolic_operand (XEXP (fnaddr, 0), VOIDmode)) fnaddr = gen_rtx_MEM (QImode, construct_plt_address (XEXP (fnaddr, 0))); @@ -19241,11 +21908,118 @@ + 2, vec)); } + /* Add UNSPEC_CALL_NEEDS_VZEROUPPER decoration. */ + if (TARGET_VZEROUPPER) + { + rtx unspec; + int avx256; + + if (cfun->machine->callee_pass_avx256_p) + { + if (cfun->machine->callee_return_avx256_p) + avx256 = callee_return_pass_avx256; + else + avx256 = callee_pass_avx256; + } + else if (cfun->machine->callee_return_avx256_p) + avx256 = callee_return_avx256; + else + avx256 = call_no_avx256; + + if (reload_completed) + emit_insn (gen_avx_vzeroupper (GEN_INT (avx256))); + else + { + unspec = gen_rtx_UNSPEC (VOIDmode, + gen_rtvec (1, GEN_INT (avx256)), + UNSPEC_CALL_NEEDS_VZEROUPPER); + call = gen_rtx_PARALLEL (VOIDmode, + gen_rtvec (2, call, unspec)); + } + } + call = emit_call_insn (call); if (use) CALL_INSN_FUNCTION_USAGE (call) = use; -} - + + return call; +} + +void +ix86_split_call_vzeroupper (rtx insn, rtx vzeroupper) +{ + rtx call = XVECEXP (PATTERN (insn), 0, 0); + emit_insn (gen_avx_vzeroupper (vzeroupper)); + emit_call_insn (call); +} + +/* Output the assembly for a call instruction. */ + +const char * +ix86_output_call_insn (rtx insn, rtx call_op, int addr_op) +{ + bool direct_p = constant_call_address_operand (call_op, Pmode); + bool seh_nop_p = false; + + gcc_assert (addr_op == 0 || addr_op == 1); + + if (SIBLING_CALL_P (insn)) + { + if (direct_p) + return addr_op ? "jmp\t%P1" : "jmp\t%P0"; + /* SEH epilogue detection requires the indirect branch case + to include REX.W. */ + else if (TARGET_SEH) + return addr_op ? "rex.W jmp %A1" : "rex.W jmp %A0"; + else + return addr_op ? "jmp\t%A1" : "jmp\t%A0"; + } + + /* SEH unwinding can require an extra nop to be emitted in several + circumstances. Determine if we have one of those. */ + if (TARGET_SEH) + { + rtx i; + + for (i = NEXT_INSN (insn); i ; i = NEXT_INSN (i)) + { + /* If we get to another real insn, we don't need the nop. */ + if (INSN_P (i)) + break; + + /* If we get to the epilogue note, prevent a catch region from + being adjacent to the standard epilogue sequence. If non- + call-exceptions, we'll have done this during epilogue emission. */ + if (NOTE_P (i) && NOTE_KIND (i) == NOTE_INSN_EPILOGUE_BEG + && !flag_non_call_exceptions + && !can_throw_internal (insn)) + { + seh_nop_p = true; + break; + } + } + + /* If we didn't find a real insn following the call, prevent the + unwinder from looking into the next function. */ + if (i == NULL) + seh_nop_p = true; + } + + if (direct_p) + { + if (seh_nop_p) + return addr_op ? "call\t%P1\n\tnop" : "call\t%P0\n\tnop"; + else + return addr_op ? "call\t%P1" : "call\t%P0"; + } + else + { + if (seh_nop_p) + return addr_op ? "call\t%A1\n\tnop" : "call\t%A0\n\tnop"; + else + return addr_op ? "call\t%A1" : "call\t%A0"; + } +} /* Clear stack slot assignments remembered from previous functions. This is called from INIT_EXPANDERS once before RTL is emitted for each @@ -19256,7 +22030,7 @@ { struct machine_function *f; - f = GGC_CNEW (struct machine_function); + f = ggc_alloc_cleared_machine_function (); f->use_fast_prologue_epilogue_nregs = -1; f->tls_descriptor_call_expanded_p = 0; f->call_abi = ix86_abi; @@ -19284,8 +22058,7 @@ if (s->mode == mode && s->n == n) return copy_rtx (s->rtl); - s = (struct stack_local_entry *) - ggc_alloc (sizeof (struct stack_local_entry)); + s = ggc_alloc_stack_local_entry (); s->n = n; s->mode = mode; s->rtl = assign_stack_local (mode, GET_MODE_SIZE (mode), 0); @@ -19407,6 +22180,7 @@ || SYMBOL_REF_TLS_MODEL (symbol) != 0) && (GET_CODE (symbol) != UNSPEC || (XINT (symbol, 1) != UNSPEC_GOTPCREL + && XINT (symbol, 1) != UNSPEC_PCREL && XINT (symbol, 1) != UNSPEC_GOTNTPOFF))) len += 1; } @@ -19612,17 +22386,20 @@ case PROCESSOR_PENTIUMPRO: case PROCESSOR_PENTIUM4: + case PROCESSOR_CORE2_32: + case PROCESSOR_CORE2_64: + case PROCESSOR_COREI7_32: + case PROCESSOR_COREI7_64: case PROCESSOR_ATHLON: case PROCESSOR_K8: case PROCESSOR_AMDFAM10: case PROCESSOR_NOCONA: case PROCESSOR_GENERIC32: case PROCESSOR_GENERIC64: + case PROCESSOR_BDVER1: + case PROCESSOR_BTVER1: return 3; - case PROCESSOR_CORE2: - return 4; - default: return 1; } @@ -19807,6 +22584,8 @@ case PROCESSOR_ATHLON: case PROCESSOR_K8: case PROCESSOR_AMDFAM10: + case PROCESSOR_BDVER1: + case PROCESSOR_BTVER1: case PROCESSOR_ATOM: case PROCESSOR_GENERIC32: case PROCESSOR_GENERIC64: @@ -19860,12 +22639,267 @@ case PROCESSOR_K6: return 1; + case PROCESSOR_CORE2_32: + case PROCESSOR_CORE2_64: + case PROCESSOR_COREI7_32: + case PROCESSOR_COREI7_64: + /* Generally, we want haifa-sched:max_issue() to look ahead as far + as many instructions can be executed on a cycle, i.e., + issue_rate. I wonder why tuning for many CPUs does not do this. */ + return ix86_issue_rate (); + default: return 0; } } + +/* Model decoder of Core 2/i7. + Below hooks for multipass scheduling (see haifa-sched.c:max_issue) + track the instruction fetch block boundaries and make sure that long + (9+ bytes) instructions are assigned to D0. */ + +/* Maximum length of an insn that can be handled by + a secondary decoder unit. '8' for Core 2/i7. */ +static int core2i7_secondary_decoder_max_insn_size; + +/* Ifetch block size, i.e., number of bytes decoder reads per cycle. + '16' for Core 2/i7. */ +static int core2i7_ifetch_block_size; + +/* Maximum number of instructions decoder can handle per cycle. + '6' for Core 2/i7. */ +static int core2i7_ifetch_block_max_insns; + +typedef struct ix86_first_cycle_multipass_data_ * + ix86_first_cycle_multipass_data_t; +typedef const struct ix86_first_cycle_multipass_data_ * + const_ix86_first_cycle_multipass_data_t; + +/* A variable to store target state across calls to max_issue within + one cycle. */ +static struct ix86_first_cycle_multipass_data_ _ix86_first_cycle_multipass_data, + *ix86_first_cycle_multipass_data = &_ix86_first_cycle_multipass_data; + +/* Initialize DATA. */ +static void +core2i7_first_cycle_multipass_init (void *_data) +{ + ix86_first_cycle_multipass_data_t data + = (ix86_first_cycle_multipass_data_t) _data; + + data->ifetch_block_len = 0; + data->ifetch_block_n_insns = 0; + data->ready_try_change = NULL; + data->ready_try_change_size = 0; +} + +/* Advancing the cycle; reset ifetch block counts. */ +static void +core2i7_dfa_post_advance_cycle (void) +{ + ix86_first_cycle_multipass_data_t data = ix86_first_cycle_multipass_data; + + gcc_assert (data->ifetch_block_n_insns <= core2i7_ifetch_block_max_insns); + + data->ifetch_block_len = 0; + data->ifetch_block_n_insns = 0; +} + +static int min_insn_size (rtx); + +/* Filter out insns from ready_try that the core will not be able to issue + on current cycle due to decoder. */ +static void +core2i7_first_cycle_multipass_filter_ready_try +(const_ix86_first_cycle_multipass_data_t data, + char *ready_try, int n_ready, bool first_cycle_insn_p) +{ + while (n_ready--) + { + rtx insn; + int insn_size; + + if (ready_try[n_ready]) + continue; + + insn = get_ready_element (n_ready); + insn_size = min_insn_size (insn); + + if (/* If this is a too long an insn for a secondary decoder ... */ + (!first_cycle_insn_p + && insn_size > core2i7_secondary_decoder_max_insn_size) + /* ... or it would not fit into the ifetch block ... */ + || data->ifetch_block_len + insn_size > core2i7_ifetch_block_size + /* ... or the decoder is full already ... */ + || data->ifetch_block_n_insns + 1 > core2i7_ifetch_block_max_insns) + /* ... mask the insn out. */ + { + ready_try[n_ready] = 1; + + if (data->ready_try_change) + SET_BIT (data->ready_try_change, n_ready); + } + } +} + +/* Prepare for a new round of multipass lookahead scheduling. */ +static void +core2i7_first_cycle_multipass_begin (void *_data, char *ready_try, int n_ready, + bool first_cycle_insn_p) +{ + ix86_first_cycle_multipass_data_t data + = (ix86_first_cycle_multipass_data_t) _data; + const_ix86_first_cycle_multipass_data_t prev_data + = ix86_first_cycle_multipass_data; + + /* Restore the state from the end of the previous round. */ + data->ifetch_block_len = prev_data->ifetch_block_len; + data->ifetch_block_n_insns = prev_data->ifetch_block_n_insns; + + /* Filter instructions that cannot be issued on current cycle due to + decoder restrictions. */ + core2i7_first_cycle_multipass_filter_ready_try (data, ready_try, n_ready, + first_cycle_insn_p); +} + +/* INSN is being issued in current solution. Account for its impact on + the decoder model. */ +static void +core2i7_first_cycle_multipass_issue (void *_data, char *ready_try, int n_ready, + rtx insn, const void *_prev_data) +{ + ix86_first_cycle_multipass_data_t data + = (ix86_first_cycle_multipass_data_t) _data; + const_ix86_first_cycle_multipass_data_t prev_data + = (const_ix86_first_cycle_multipass_data_t) _prev_data; + + int insn_size = min_insn_size (insn); + + data->ifetch_block_len = prev_data->ifetch_block_len + insn_size; + data->ifetch_block_n_insns = prev_data->ifetch_block_n_insns + 1; + gcc_assert (data->ifetch_block_len <= core2i7_ifetch_block_size + && data->ifetch_block_n_insns <= core2i7_ifetch_block_max_insns); + + /* Allocate or resize the bitmap for storing INSN's effect on ready_try. */ + if (!data->ready_try_change) + { + data->ready_try_change = sbitmap_alloc (n_ready); + data->ready_try_change_size = n_ready; + } + else if (data->ready_try_change_size < n_ready) + { + data->ready_try_change = sbitmap_resize (data->ready_try_change, + n_ready, 0); + data->ready_try_change_size = n_ready; + } + sbitmap_zero (data->ready_try_change); + + /* Filter out insns from ready_try that the core will not be able to issue + on current cycle due to decoder. */ + core2i7_first_cycle_multipass_filter_ready_try (data, ready_try, n_ready, + false); +} + +/* Revert the effect on ready_try. */ +static void +core2i7_first_cycle_multipass_backtrack (const void *_data, + char *ready_try, + int n_ready ATTRIBUTE_UNUSED) +{ + const_ix86_first_cycle_multipass_data_t data + = (const_ix86_first_cycle_multipass_data_t) _data; + unsigned int i = 0; + sbitmap_iterator sbi; + + gcc_assert (sbitmap_last_set_bit (data->ready_try_change) < n_ready); + EXECUTE_IF_SET_IN_SBITMAP (data->ready_try_change, 0, i, sbi) + { + ready_try[i] = 0; + } +} + +/* Save the result of multipass lookahead scheduling for the next round. */ +static void +core2i7_first_cycle_multipass_end (const void *_data) +{ + const_ix86_first_cycle_multipass_data_t data + = (const_ix86_first_cycle_multipass_data_t) _data; + ix86_first_cycle_multipass_data_t next_data + = ix86_first_cycle_multipass_data; + + if (data != NULL) + { + next_data->ifetch_block_len = data->ifetch_block_len; + next_data->ifetch_block_n_insns = data->ifetch_block_n_insns; + } +} + +/* Deallocate target data. */ +static void +core2i7_first_cycle_multipass_fini (void *_data) +{ + ix86_first_cycle_multipass_data_t data + = (ix86_first_cycle_multipass_data_t) _data; + + if (data->ready_try_change) + { + sbitmap_free (data->ready_try_change); + data->ready_try_change = NULL; + data->ready_try_change_size = 0; + } +} + +/* Prepare for scheduling pass. */ +static void +ix86_sched_init_global (FILE *dump ATTRIBUTE_UNUSED, + int verbose ATTRIBUTE_UNUSED, + int max_uid ATTRIBUTE_UNUSED) +{ + /* Install scheduling hooks for current CPU. Some of these hooks are used + in time-critical parts of the scheduler, so we only set them up when + they are actually used. */ + switch (ix86_tune) + { + case PROCESSOR_CORE2_32: + case PROCESSOR_CORE2_64: + case PROCESSOR_COREI7_32: + case PROCESSOR_COREI7_64: + targetm.sched.dfa_post_advance_cycle + = core2i7_dfa_post_advance_cycle; + targetm.sched.first_cycle_multipass_init + = core2i7_first_cycle_multipass_init; + targetm.sched.first_cycle_multipass_begin + = core2i7_first_cycle_multipass_begin; + targetm.sched.first_cycle_multipass_issue + = core2i7_first_cycle_multipass_issue; + targetm.sched.first_cycle_multipass_backtrack + = core2i7_first_cycle_multipass_backtrack; + targetm.sched.first_cycle_multipass_end + = core2i7_first_cycle_multipass_end; + targetm.sched.first_cycle_multipass_fini + = core2i7_first_cycle_multipass_fini; + + /* Set decoder parameters. */ + core2i7_secondary_decoder_max_insn_size = 8; + core2i7_ifetch_block_size = 16; + core2i7_ifetch_block_max_insns = 6; + break; + + default: + targetm.sched.dfa_post_advance_cycle = NULL; + targetm.sched.first_cycle_multipass_init = NULL; + targetm.sched.first_cycle_multipass_begin = NULL; + targetm.sched.first_cycle_multipass_issue = NULL; + targetm.sched.first_cycle_multipass_backtrack = NULL; + targetm.sched.first_cycle_multipass_end = NULL; + targetm.sched.first_cycle_multipass_fini = NULL; + break; + } +} + + /* Compute the alignment given to a constant that is being placed in memory. EXP is the constant and ALIGN is the alignment that the object would ordinarily have. @@ -20002,10 +23036,27 @@ } /* x86-64 ABI requires arrays greater than 16 bytes to be aligned - to 16byte boundary. */ - if (TARGET_64BIT) + to 16byte boundary. Exact wording is: + + An array uses the same alignment as its elements, except that a local or + global array variable of length at least 16 bytes or + a C99 variable-length array variable always has alignment of at least 16 bytes. + + This was added to allow use of aligned SSE instructions at arrays. This + rule is meant for static storage (where compiler can not do the analysis + by itself). We follow it for automatic variables only when convenient. + We fully control everything in the function compiled and functions from + other unit can not rely on the alignment. + + Exclude va_list type. It is the common case of local array where + we can not benefit from the alignment. */ + if (TARGET_64BIT && optimize_function_for_speed_p (cfun) + && TARGET_SSE) { if (AGGREGATE_TYPE_P (type) + && (va_list_type_node == NULL_TREE + || (TYPE_MAIN_VARIANT (type) + != TYPE_MAIN_VARIANT (va_list_type_node))) && TYPE_SIZE (type) && TREE_CODE (TYPE_SIZE (type)) == INTEGER_CST && (TREE_INT_CST_LOW (TYPE_SIZE (type)) >= 16 @@ -20060,9 +23111,6 @@ { tree type, decl; - if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64) - return align; - if (exp && DECL_P (exp)) { type = TREE_TYPE (exp); @@ -20074,6 +23122,9 @@ decl = NULL; } + if (TARGET_64BIT || align != 64 || ix86_preferred_stack_boundary >= 64) + return align; + /* Don't do dynamic stack realignment for long long objects with -mpreferred-stack-boundary=2. */ if ((mode == DImode || (type && TYPE_MODE (type) == DImode)) @@ -20113,6 +23164,12 @@ us with EAX for the static chain. */ regno = AX_REG; } + else if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (fntype))) + { + /* Thiscall functions use ecx for arguments, which leaves + us with EAX for the static chain. */ + regno = AX_REG; + } else if (ix86_function_regparm (fntype, fndecl) == 3) { /* For regparm 3, we have no free call-clobbered registers in @@ -20138,7 +23195,7 @@ } /* Emit RTL insns to initialize the variable parts of a trampoline. - FNDECL is the decl of the target address; M_TRAMP is a MEM for + FNDECL is the decl of the target address; M_TRAMP is a MEM for the trampoline, and CHAIN_VALUE is an RTX for the static chain to be passed to the target function. */ @@ -20990,6 +24047,10 @@ IX86_BUILTIN_VPERMILPS, IX86_BUILTIN_VPERMILPD256, IX86_BUILTIN_VPERMILPS256, + IX86_BUILTIN_VPERMIL2PD, + IX86_BUILTIN_VPERMIL2PS, + IX86_BUILTIN_VPERMIL2PD256, + IX86_BUILTIN_VPERMIL2PS256, IX86_BUILTIN_VPERM2F128PD256, IX86_BUILTIN_VPERM2F128PS256, IX86_BUILTIN_VPERM2F128SI256, @@ -21074,6 +24135,8 @@ /* Vectorizer support builtins. */ IX86_BUILTIN_CPYSGNPS, IX86_BUILTIN_CPYSGNPD, + IX86_BUILTIN_CPYSGNPS256, + IX86_BUILTIN_CPYSGNPD256, IX86_BUILTIN_CVTUDQ2PS, @@ -21095,34 +24158,12 @@ IX86_BUILTIN_VFMADDSD, IX86_BUILTIN_VFMADDPS, IX86_BUILTIN_VFMADDPD, - IX86_BUILTIN_VFMSUBSS, - IX86_BUILTIN_VFMSUBSD, - IX86_BUILTIN_VFMSUBPS, - IX86_BUILTIN_VFMSUBPD, + IX86_BUILTIN_VFMADDPS256, + IX86_BUILTIN_VFMADDPD256, IX86_BUILTIN_VFMADDSUBPS, IX86_BUILTIN_VFMADDSUBPD, - IX86_BUILTIN_VFMSUBADDPS, - IX86_BUILTIN_VFMSUBADDPD, - IX86_BUILTIN_VFNMADDSS, - IX86_BUILTIN_VFNMADDSD, - IX86_BUILTIN_VFNMADDPS, - IX86_BUILTIN_VFNMADDPD, - IX86_BUILTIN_VFNMSUBSS, - IX86_BUILTIN_VFNMSUBSD, - IX86_BUILTIN_VFNMSUBPS, - IX86_BUILTIN_VFNMSUBPD, - IX86_BUILTIN_VFMADDPS256, - IX86_BUILTIN_VFMADDPD256, - IX86_BUILTIN_VFMSUBPS256, - IX86_BUILTIN_VFMSUBPD256, IX86_BUILTIN_VFMADDSUBPS256, IX86_BUILTIN_VFMADDSUBPD256, - IX86_BUILTIN_VFMSUBADDPS256, - IX86_BUILTIN_VFMSUBADDPD256, - IX86_BUILTIN_VFNMADDPS256, - IX86_BUILTIN_VFNMADDPD256, - IX86_BUILTIN_VFNMSUBPS256, - IX86_BUILTIN_VFNMSUBPD256, IX86_BUILTIN_VPCMOV, IX86_BUILTIN_VPCMOV_V2DI, @@ -21277,6 +24318,40 @@ IX86_BUILTIN_CLZS, + /* BMI instructions. */ + IX86_BUILTIN_BEXTR32, + IX86_BUILTIN_BEXTR64, + IX86_BUILTIN_CTZS, + + /* TBM instructions. */ + IX86_BUILTIN_BEXTRI32, + IX86_BUILTIN_BEXTRI64, + + + /* FSGSBASE instructions. */ + IX86_BUILTIN_RDFSBASE32, + IX86_BUILTIN_RDFSBASE64, + IX86_BUILTIN_RDGSBASE32, + IX86_BUILTIN_RDGSBASE64, + IX86_BUILTIN_WRFSBASE32, + IX86_BUILTIN_WRFSBASE64, + IX86_BUILTIN_WRGSBASE32, + IX86_BUILTIN_WRGSBASE64, + + /* RDRND instructions. */ + IX86_BUILTIN_RDRAND16_STEP, + IX86_BUILTIN_RDRAND32_STEP, + IX86_BUILTIN_RDRAND64_STEP, + + /* F16C instructions. */ + IX86_BUILTIN_CVTPH2PS, + IX86_BUILTIN_CVTPH2PS256, + IX86_BUILTIN_CVTPS2PH, + IX86_BUILTIN_CVTPS2PH256, + + /* CFString built-in for darwin */ + IX86_BUILTIN_CFSTRING, + IX86_BUILTIN_MAX }; @@ -21323,6 +24398,7 @@ { ix86_builtins_isa[(int) code].isa = mask; + mask &= ~OPTION_MASK_ISA_64BIT; if (mask == 0 || (mask & ix86_isa_flags) != 0 || (lang_hooks.builtin_function @@ -21534,14 +24610,14 @@ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv4df, "__builtin_ia32_movntpd256", IX86_BUILTIN_MOVNTPD256, UNKNOWN, (int) VOID_FTYPE_PDOUBLE_V4DF }, { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movntv8sf, "__builtin_ia32_movntps256", IX86_BUILTIN_MOVNTPS256, UNKNOWN, (int) VOID_FTYPE_PFLOAT_V8SF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DF_V2DF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SF_V4SF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DF_V4DF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SF_V8SF }, + { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd, "__builtin_ia32_maskloadpd", IX86_BUILTIN_MASKLOADPD, UNKNOWN, (int) V2DF_FTYPE_PCV2DF_V2DI }, + { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps, "__builtin_ia32_maskloadps", IX86_BUILTIN_MASKLOADPS, UNKNOWN, (int) V4SF_FTYPE_PCV4SF_V4SI }, + { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadpd256, "__builtin_ia32_maskloadpd256", IX86_BUILTIN_MASKLOADPD256, UNKNOWN, (int) V4DF_FTYPE_PCV4DF_V4DI }, + { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskloadps256, "__builtin_ia32_maskloadps256", IX86_BUILTIN_MASKLOADPS256, UNKNOWN, (int) V8SF_FTYPE_PCV8SF_V8SI }, + { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd, "__builtin_ia32_maskstorepd", IX86_BUILTIN_MASKSTOREPD, UNKNOWN, (int) VOID_FTYPE_PV2DF_V2DI_V2DF }, + { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps, "__builtin_ia32_maskstoreps", IX86_BUILTIN_MASKSTOREPS, UNKNOWN, (int) VOID_FTYPE_PV4SF_V4SI_V4SF }, + { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstorepd256, "__builtin_ia32_maskstorepd256", IX86_BUILTIN_MASKSTOREPD256, UNKNOWN, (int) VOID_FTYPE_PV4DF_V4DI_V4DF }, + { OPTION_MASK_ISA_AVX, CODE_FOR_avx_maskstoreps256, "__builtin_ia32_maskstoreps256", IX86_BUILTIN_MASKSTOREPS256, UNKNOWN, (int) VOID_FTYPE_PV8SF_V8SI_V8SF }, { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_llwpcb, "__builtin_ia32_llwpcb", IX86_BUILTIN_LLWPCB, UNKNOWN, (int) VOID_FTYPE_PVOID }, { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_slwpcb, "__builtin_ia32_slwpcb", IX86_BUILTIN_SLWPCB, UNKNOWN, (int) PVOID_FTYPE_VOID }, @@ -21550,6 +24626,15 @@ { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinssi3, "__builtin_ia32_lwpins32", IX86_BUILTIN_LWPINS32, UNKNOWN, (int) UCHAR_FTYPE_UINT_UINT_UINT }, { OPTION_MASK_ISA_LWP, CODE_FOR_lwp_lwpinsdi3, "__builtin_ia32_lwpins64", IX86_BUILTIN_LWPINS64, UNKNOWN, (int) UCHAR_FTYPE_UINT64_UINT_UINT }, + /* FSGSBASE */ + { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasesi, "__builtin_ia32_rdfsbase32", IX86_BUILTIN_RDFSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID }, + { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdfsbasedi, "__builtin_ia32_rdfsbase64", IX86_BUILTIN_RDFSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID }, + { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasesi, "__builtin_ia32_rdgsbase32", IX86_BUILTIN_RDGSBASE32, UNKNOWN, (int) UNSIGNED_FTYPE_VOID }, + { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_rdgsbasedi, "__builtin_ia32_rdgsbase64", IX86_BUILTIN_RDGSBASE64, UNKNOWN, (int) UINT64_FTYPE_VOID }, + { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasesi, "__builtin_ia32_wrfsbase32", IX86_BUILTIN_WRFSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED }, + { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrfsbasedi, "__builtin_ia32_wrfsbase64", IX86_BUILTIN_WRFSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 }, + { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasesi, "__builtin_ia32_wrgsbase32", IX86_BUILTIN_WRGSBASE32, UNKNOWN, (int) VOID_FTYPE_UNSIGNED }, + { OPTION_MASK_ISA_FSGSBASE | OPTION_MASK_ISA_64BIT, CODE_FOR_wrgsbasedi, "__builtin_ia32_wrgsbase64", IX86_BUILTIN_WRGSBASE64, UNKNOWN, (int) VOID_FTYPE_UINT64 }, }; /* Builtins with variable number of arguments. */ @@ -21999,12 +25084,12 @@ { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendvb, "__builtin_ia32_pblendvb128", IX86_BUILTIN_PBLENDVB128, UNKNOWN, (int) V16QI_FTYPE_V16QI_V16QI_V16QI }, { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_pblendw, "__builtin_ia32_pblendw128", IX86_BUILTIN_PBLENDW128, UNKNOWN, (int) V8HI_FTYPE_V8HI_V8HI_INT }, - { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI }, - { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI }, - { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI }, - { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI }, - { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI }, - { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI }, + { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv8qiv8hi2, "__builtin_ia32_pmovsxbw128", IX86_BUILTIN_PMOVSXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI }, + { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4qiv4si2, "__builtin_ia32_pmovsxbd128", IX86_BUILTIN_PMOVSXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI }, + { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2qiv2di2, "__builtin_ia32_pmovsxbq128", IX86_BUILTIN_PMOVSXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI }, + { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv4hiv4si2, "__builtin_ia32_pmovsxwd128", IX86_BUILTIN_PMOVSXWD128, UNKNOWN, (int) V4SI_FTYPE_V8HI }, + { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2hiv2di2, "__builtin_ia32_pmovsxwq128", IX86_BUILTIN_PMOVSXWQ128, UNKNOWN, (int) V2DI_FTYPE_V8HI }, + { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_sign_extendv2siv2di2, "__builtin_ia32_pmovsxdq128", IX86_BUILTIN_PMOVSXDQ128, UNKNOWN, (int) V2DI_FTYPE_V4SI }, { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv8qiv8hi2, "__builtin_ia32_pmovzxbw128", IX86_BUILTIN_PMOVZXBW128, UNKNOWN, (int) V8HI_FTYPE_V16QI }, { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv4qiv4si2, "__builtin_ia32_pmovzxbd128", IX86_BUILTIN_PMOVZXBD128, UNKNOWN, (int) V4SI_FTYPE_V16QI }, { OPTION_MASK_ISA_SSE4_1, CODE_FOR_sse4_1_zero_extendv2qiv2di2, "__builtin_ia32_pmovzxbq128", IX86_BUILTIN_PMOVZXBQ128, UNKNOWN, (int) V2DI_FTYPE_V16QI }, @@ -22152,9 +25237,9 @@ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si256_si, "__builtin_ia32_si256_si", IX86_BUILTIN_SI256_SI, UNKNOWN, (int) V8SI_FTYPE_V4SI }, { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps256_ps, "__builtin_ia32_ps256_ps", IX86_BUILTIN_PS256_PS, UNKNOWN, (int) V8SF_FTYPE_V4SF }, { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd256_pd, "__builtin_ia32_pd256_pd", IX86_BUILTIN_PD256_PD, UNKNOWN, (int) V4DF_FTYPE_V2DF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_si_si256, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_ps_ps256, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF }, - { OPTION_MASK_ISA_AVX, CODE_FOR_avx_pd_pd256, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF }, + { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8si, "__builtin_ia32_si_si256", IX86_BUILTIN_SI_SI256, UNKNOWN, (int) V4SI_FTYPE_V8SI }, + { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v8sf, "__builtin_ia32_ps_ps256", IX86_BUILTIN_PS_PS256, UNKNOWN, (int) V4SF_FTYPE_V8SF }, + { OPTION_MASK_ISA_AVX, CODE_FOR_vec_extract_lo_v4df, "__builtin_ia32_pd_pd256", IX86_BUILTIN_PD_PD256, UNKNOWN, (int) V2DF_FTYPE_V4DF }, { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestzpd", IX86_BUILTIN_VTESTZPD, EQ, (int) INT_FTYPE_V2DF_V2DF_PTEST }, { OPTION_MASK_ISA_AVX, CODE_FOR_avx_vtestpd, "__builtin_ia32_vtestcpd", IX86_BUILTIN_VTESTCPD, LTU, (int) INT_FTYPE_V2DF_V2DF_PTEST }, @@ -22175,10 +25260,32 @@ { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskpd256, "__builtin_ia32_movmskpd256", IX86_BUILTIN_MOVMSKPD256, UNKNOWN, (int) INT_FTYPE_V4DF }, { OPTION_MASK_ISA_AVX, CODE_FOR_avx_movmskps256, "__builtin_ia32_movmskps256", IX86_BUILTIN_MOVMSKPS256, UNKNOWN, (int) INT_FTYPE_V8SF }, + { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv8sf3, "__builtin_ia32_copysignps256", IX86_BUILTIN_CPYSGNPS256, UNKNOWN, (int) V8SF_FTYPE_V8SF_V8SF }, + { OPTION_MASK_ISA_AVX, CODE_FOR_copysignv4df3, "__builtin_ia32_copysignpd256", IX86_BUILTIN_CPYSGNPD256, UNKNOWN, (int) V4DF_FTYPE_V4DF_V4DF }, + { OPTION_MASK_ISA_ABM, CODE_FOR_clzhi2_abm, "__builtin_clzs", IX86_BUILTIN_CLZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 }, + + /* BMI */ + { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_si, "__builtin_ia32_bextr_u32", IX86_BUILTIN_BEXTR32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT }, + { OPTION_MASK_ISA_BMI, CODE_FOR_bmi_bextr_di, "__builtin_ia32_bextr_u64", IX86_BUILTIN_BEXTR64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 }, + { OPTION_MASK_ISA_BMI, CODE_FOR_ctzhi2, "__builtin_ctzs", IX86_BUILTIN_CTZS, UNKNOWN, (int) UINT16_FTYPE_UINT16 }, + + /* TBM */ + { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_si, "__builtin_ia32_bextri_u32", IX86_BUILTIN_BEXTRI32, UNKNOWN, (int) UINT_FTYPE_UINT_UINT }, + { OPTION_MASK_ISA_TBM, CODE_FOR_tbm_bextri_di, "__builtin_ia32_bextri_u64", IX86_BUILTIN_BEXTRI64, UNKNOWN, (int) UINT64_FTYPE_UINT64_UINT64 }, + + /* F16C */ + { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps, "__builtin_ia32_vcvtph2ps", IX86_BUILTIN_CVTPH2PS, UNKNOWN, (int) V4SF_FTYPE_V8HI }, + { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtph2ps256, "__builtin_ia32_vcvtph2ps256", IX86_BUILTIN_CVTPH2PS256, UNKNOWN, (int) V8SF_FTYPE_V8HI }, + { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph, "__builtin_ia32_vcvtps2ph", IX86_BUILTIN_CVTPS2PH, UNKNOWN, (int) V8HI_FTYPE_V4SF_INT }, + { OPTION_MASK_ISA_F16C, CODE_FOR_vcvtps2ph256, "__builtin_ia32_vcvtps2ph256", IX86_BUILTIN_CVTPS2PH256, UNKNOWN, (int) V8HI_FTYPE_V8SF_INT }, }; /* FMA4 and XOP. */ +#define MULTI_ARG_4_DF2_DI_I V2DF_FTYPE_V2DF_V2DF_V2DI_INT +#define MULTI_ARG_4_DF2_DI_I1 V4DF_FTYPE_V4DF_V4DF_V4DI_INT +#define MULTI_ARG_4_SF2_SI_I V4SF_FTYPE_V4SF_V4SF_V4SI_INT +#define MULTI_ARG_4_SF2_SI_I1 V8SF_FTYPE_V8SF_V8SF_V8SI_INT #define MULTI_ARG_3_SF V4SF_FTYPE_V4SF_V4SF_V4SF #define MULTI_ARG_3_DF V2DF_FTYPE_V2DF_V2DF_V2DF #define MULTI_ARG_3_SF2 V8SF_FTYPE_V8SF_V8SF_V8SF @@ -22230,43 +25337,38 @@ static const struct builtin_description bdesc_multi_arg[] = { - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmaddv4sf4, "__builtin_ia32_vfmaddss", IX86_BUILTIN_VFMADDSS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmaddv2df4, "__builtin_ia32_vfmaddsd", IX86_BUILTIN_VFMADDSD, UNKNOWN, (int)MULTI_ARG_3_DF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddv4sf4, "__builtin_ia32_vfmaddps", IX86_BUILTIN_VFMADDPS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddv2df4, "__builtin_ia32_vfmaddpd", IX86_BUILTIN_VFMADDPD, UNKNOWN, (int)MULTI_ARG_3_DF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmsubv4sf4, "__builtin_ia32_vfmsubss", IX86_BUILTIN_VFMSUBSS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmsubv2df4, "__builtin_ia32_vfmsubsd", IX86_BUILTIN_VFMSUBSD, UNKNOWN, (int)MULTI_ARG_3_DF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubv4sf4, "__builtin_ia32_vfmsubps", IX86_BUILTIN_VFMSUBPS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubv2df4, "__builtin_ia32_vfmsubpd", IX86_BUILTIN_VFMSUBPD, UNKNOWN, (int)MULTI_ARG_3_DF }, - - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfnmaddv4sf4, "__builtin_ia32_vfnmaddss", IX86_BUILTIN_VFNMADDSS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfnmaddv2df4, "__builtin_ia32_vfnmaddsd", IX86_BUILTIN_VFNMADDSD, UNKNOWN, (int)MULTI_ARG_3_DF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmaddv4sf4, "__builtin_ia32_vfnmaddps", IX86_BUILTIN_VFNMADDPS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmaddv2df4, "__builtin_ia32_vfnmaddpd", IX86_BUILTIN_VFNMADDPD, UNKNOWN, (int)MULTI_ARG_3_DF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfnmsubv4sf4, "__builtin_ia32_vfnmsubss", IX86_BUILTIN_VFNMSUBSS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfnmsubv2df4, "__builtin_ia32_vfnmsubsd", IX86_BUILTIN_VFNMSUBSD, UNKNOWN, (int)MULTI_ARG_3_DF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmsubv4sf4, "__builtin_ia32_vfnmsubps", IX86_BUILTIN_VFNMSUBPS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmsubv2df4, "__builtin_ia32_vfnmsubpd", IX86_BUILTIN_VFNMSUBPD, UNKNOWN, (int)MULTI_ARG_3_DF }, - - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddsubv4sf4, "__builtin_ia32_vfmaddsubps", IX86_BUILTIN_VFMADDSUBPS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddsubv2df4, "__builtin_ia32_vfmaddsubpd", IX86_BUILTIN_VFMADDSUBPD, UNKNOWN, (int)MULTI_ARG_3_DF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubaddv4sf4, "__builtin_ia32_vfmsubaddps", IX86_BUILTIN_VFMSUBADDPS, UNKNOWN, (int)MULTI_ARG_3_SF }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubaddv2df4, "__builtin_ia32_vfmsubaddpd", IX86_BUILTIN_VFMSUBADDPD, UNKNOWN, (int)MULTI_ARG_3_DF }, - - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddv8sf4256, "__builtin_ia32_vfmaddps256", IX86_BUILTIN_VFMADDPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddv4df4256, "__builtin_ia32_vfmaddpd256", IX86_BUILTIN_VFMADDPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubv8sf4256, "__builtin_ia32_vfmsubps256", IX86_BUILTIN_VFMSUBPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubv4df4256, "__builtin_ia32_vfmsubpd256", IX86_BUILTIN_VFMSUBPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 }, - - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmaddv8sf4256, "__builtin_ia32_vfnmaddps256", IX86_BUILTIN_VFNMADDPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmaddv4df4256, "__builtin_ia32_vfnmaddpd256", IX86_BUILTIN_VFNMADDPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmsubv8sf4256, "__builtin_ia32_vfnmsubps256", IX86_BUILTIN_VFNMSUBPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fnmsubv4df4256, "__builtin_ia32_vfnmsubpd256", IX86_BUILTIN_VFNMSUBPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 }, - - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddsubv8sf4, "__builtin_ia32_vfmaddsubps256", IX86_BUILTIN_VFMADDSUBPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmaddsubv4df4, "__builtin_ia32_vfmaddsubpd256", IX86_BUILTIN_VFMADDSUBPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubaddv8sf4, "__builtin_ia32_vfmsubaddps256", IX86_BUILTIN_VFMSUBADDPS256, UNKNOWN, (int)MULTI_ARG_3_SF2 }, - { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmsubaddv4df4, "__builtin_ia32_vfmsubaddpd256", IX86_BUILTIN_VFMSUBADDPD256, UNKNOWN, (int)MULTI_ARG_3_DF2 }, + { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmadd_v4sf, + "__builtin_ia32_vfmaddss", IX86_BUILTIN_VFMADDSS, + UNKNOWN, (int)MULTI_ARG_3_SF }, + { OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_vmfmadd_v2df, + "__builtin_ia32_vfmaddsd", IX86_BUILTIN_VFMADDSD, + UNKNOWN, (int)MULTI_ARG_3_DF }, + + { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v4sf, + "__builtin_ia32_vfmaddps", IX86_BUILTIN_VFMADDPS, + UNKNOWN, (int)MULTI_ARG_3_SF }, + { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v2df, + "__builtin_ia32_vfmaddpd", IX86_BUILTIN_VFMADDPD, + UNKNOWN, (int)MULTI_ARG_3_DF }, + { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v8sf, + "__builtin_ia32_vfmaddps256", IX86_BUILTIN_VFMADDPS256, + UNKNOWN, (int)MULTI_ARG_3_SF2 }, + { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fma4i_fmadd_v4df, + "__builtin_ia32_vfmaddpd256", IX86_BUILTIN_VFMADDPD256, + UNKNOWN, (int)MULTI_ARG_3_DF2 }, + + { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v4sf, + "__builtin_ia32_vfmaddsubps", IX86_BUILTIN_VFMADDSUBPS, + UNKNOWN, (int)MULTI_ARG_3_SF }, + { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v2df, + "__builtin_ia32_vfmaddsubpd", IX86_BUILTIN_VFMADDSUBPD, + UNKNOWN, (int)MULTI_ARG_3_DF }, + { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v8sf, + "__builtin_ia32_vfmaddsubps256", IX86_BUILTIN_VFMADDSUBPS256, + UNKNOWN, (int)MULTI_ARG_3_SF2 }, + { OPTION_MASK_ISA_FMA | OPTION_MASK_ISA_FMA4, CODE_FOR_fmaddsub_v4df, + "__builtin_ia32_vfmaddsubpd256", IX86_BUILTIN_VFMADDSUBPD256, + UNKNOWN, (int)MULTI_ARG_3_DF2 }, { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov", IX86_BUILTIN_VPCMOV, UNKNOWN, (int)MULTI_ARG_3_DI }, { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcmov_v2di, "__builtin_ia32_vpcmov_v2di", IX86_BUILTIN_VPCMOV_V2DI, UNKNOWN, (int)MULTI_ARG_3_DI }, @@ -22320,8 +25422,8 @@ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vmfrczv2df2, "__builtin_ia32_vfrczsd", IX86_BUILTIN_VFRCZSD, UNKNOWN, (int)MULTI_ARG_2_DF }, { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4sf2, "__builtin_ia32_vfrczps", IX86_BUILTIN_VFRCZPS, UNKNOWN, (int)MULTI_ARG_1_SF }, { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv2df2, "__builtin_ia32_vfrczpd", IX86_BUILTIN_VFRCZPD, UNKNOWN, (int)MULTI_ARG_1_DF }, - { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2256, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 }, - { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2256, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 }, + { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv8sf2, "__builtin_ia32_vfrczps256", IX86_BUILTIN_VFRCZPS256, UNKNOWN, (int)MULTI_ARG_1_SF2 }, + { OPTION_MASK_ISA_XOP, CODE_FOR_xop_frczv4df2, "__builtin_ia32_vfrczpd256", IX86_BUILTIN_VFRCZPD256, UNKNOWN, (int)MULTI_ARG_1_DF2 }, { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbw, "__builtin_ia32_vphaddbw", IX86_BUILTIN_VPHADDBW, UNKNOWN, (int)MULTI_ARG_1_QI_HI }, { OPTION_MASK_ISA_XOP, CODE_FOR_xop_phaddbd, "__builtin_ia32_vphaddbd", IX86_BUILTIN_VPHADDBD, UNKNOWN, (int)MULTI_ARG_1_QI_SI }, @@ -22421,6 +25523,11 @@ { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv4si3, "__builtin_ia32_vpcomtrueud", IX86_BUILTIN_VPCOMTRUEUD, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_SI_TF }, { OPTION_MASK_ISA_XOP, CODE_FOR_xop_pcom_tfv2di3, "__builtin_ia32_vpcomtrueuq", IX86_BUILTIN_VPCOMTRUEUQ, (enum rtx_code) PCOM_TRUE, (int)MULTI_ARG_2_DI_TF }, + { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v2df3, "__builtin_ia32_vpermil2pd", IX86_BUILTIN_VPERMIL2PD, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I }, + { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4sf3, "__builtin_ia32_vpermil2ps", IX86_BUILTIN_VPERMIL2PS, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I }, + { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v4df3, "__builtin_ia32_vpermil2pd256", IX86_BUILTIN_VPERMIL2PD256, UNKNOWN, (int)MULTI_ARG_4_DF2_DI_I1 }, + { OPTION_MASK_ISA_XOP, CODE_FOR_xop_vpermil2v8sf3, "__builtin_ia32_vpermil2ps256", IX86_BUILTIN_VPERMIL2PS256, UNKNOWN, (int)MULTI_ARG_4_SF2_SI_I1 }, + }; /* Set up all the MMX/SSE builtins, even builtins for instructions that are not @@ -22536,6 +25643,15 @@ def_builtin_const (OPTION_MASK_ISA_PCLMUL, "__builtin_ia32_pclmulqdq128", V2DI_FTYPE_V2DI_V2DI_INT, IX86_BUILTIN_PCLMULQDQ128); + /* RDRND */ + def_builtin (OPTION_MASK_ISA_RDRND, "__builtin_ia32_rdrand16_step", + INT_FTYPE_PUSHORT, IX86_BUILTIN_RDRAND16_STEP); + def_builtin (OPTION_MASK_ISA_RDRND, "__builtin_ia32_rdrand32_step", + INT_FTYPE_PUNSIGNED, IX86_BUILTIN_RDRAND32_STEP); + def_builtin (OPTION_MASK_ISA_RDRND | OPTION_MASK_ISA_64BIT, + "__builtin_ia32_rdrand64_step", INT_FTYPE_PULONGLONG, + IX86_BUILTIN_RDRAND64_STEP); + /* MMX access to the vec_init patterns. */ def_builtin_const (OPTION_MASK_ISA_MMX, "__builtin_ia32_vec_init_v2si", V2SI_FTYPE_INT_INT, IX86_BUILTIN_VEC_INIT_V2SI); @@ -22666,13 +25782,13 @@ TYPE_PRECISION (float80_type_node) = 80; layout_type (float80_type_node); } - (*lang_hooks.types.register_builtin_type) (float80_type_node, "__float80"); + lang_hooks.types.register_builtin_type (float80_type_node, "__float80"); /* The __float128 type. */ float128_type_node = make_node (REAL_TYPE); TYPE_PRECISION (float128_type_node) = 128; layout_type (float128_type_node); - (*lang_hooks.types.register_builtin_type) (float128_type_node, "__float128"); + lang_hooks.types.register_builtin_type (float128_type_node, "__float128"); /* This macro is built by i386-builtin-types.awk. */ DEFINE_BUILTIN_PRIMITIVE_TYPES; @@ -22709,6 +25825,10 @@ if (TARGET_64BIT) ix86_init_builtins_va_builtins_abi (); + +#ifdef SUBTARGET_INIT_BUILTINS + SUBTARGET_INIT_BUILTINS; +#endif } /* Return the ix86 builtin for CODE. */ @@ -22754,7 +25874,7 @@ if (optimize || !target || GET_MODE (target) != tmode - || ! (*insn_data[icode].operand[0].predicate) (target, tmode)) + || !insn_data[icode].operand[0].predicate (target, tmode)) target = gen_reg_rtx (tmode); if (GET_MODE (op1) == SImode && mode1 == TImode) @@ -22764,9 +25884,9 @@ op1 = gen_lowpart (TImode, x); } - if (!(*insn_data[icode].operand[1].predicate) (op0, mode0)) + if (!insn_data[icode].operand[1].predicate (op0, mode0)) op0 = copy_to_mode_reg (mode0, op0); - if (!(*insn_data[icode].operand[2].predicate) (op1, mode1)) + if (!insn_data[icode].operand[2].predicate (op1, mode1)) op1 = copy_to_mode_reg (mode1, op1); pat = GEN_FCN (icode) (target, op0, op1); @@ -22801,6 +25921,14 @@ switch (m_type) { + case MULTI_ARG_4_DF2_DI_I: + case MULTI_ARG_4_DF2_DI_I1: + case MULTI_ARG_4_SF2_SI_I: + case MULTI_ARG_4_SF2_SI_I1: + nargs = 4; + last_arg_constant = true; + break; + case MULTI_ARG_3_SF: case MULTI_ARG_3_DF: case MULTI_ARG_3_SF2: @@ -22876,7 +26004,7 @@ if (optimize || !target || GET_MODE (target) != tmode - || ! (*insn_data[icode].operand[0].predicate) (target, tmode)) + || !insn_data[icode].operand[0].predicate (target, tmode)) target = gen_reg_rtx (tmode); gcc_assert (nargs <= 4); @@ -22909,7 +26037,7 @@ gcc_assert (GET_MODE (op) == mode || GET_MODE (op) == VOIDmode); if (optimize - || ! (*insn_data[icode].operand[i+adjust+1].predicate) (op, mode) + || !insn_data[icode].operand[i+adjust+1].predicate (op, mode) || num_memory > 1) op = force_reg (mode, op); } @@ -22944,6 +26072,10 @@ pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op); break; + case 4: + pat = GEN_FCN (icode) (target, args[0].op, args[1].op, args[2].op, args[3].op); + break; + default: gcc_unreachable (); } @@ -22970,18 +26102,18 @@ if (optimize || !target || GET_MODE (target) != tmode - || ! (*insn_data[icode].operand[0].predicate) (target, tmode)) + || !insn_data[icode].operand[0].predicate (target, tmode)) target = gen_reg_rtx (tmode); if (VECTOR_MODE_P (mode0)) op0 = safe_vector_operand (op0, mode0); if ((optimize && !register_operand (op0, mode0)) - || ! (*insn_data[icode].operand[1].predicate) (op0, mode0)) + || !insn_data[icode].operand[1].predicate (op0, mode0)) op0 = copy_to_mode_reg (mode0, op0); op1 = op0; - if (! (*insn_data[icode].operand[2].predicate) (op1, mode0)) + if (!insn_data[icode].operand[2].predicate (op1, mode0)) op1 = copy_to_mode_reg (mode0, op1); pat = GEN_FCN (icode) (target, op0, op1); @@ -23025,14 +26157,14 @@ if (optimize || !target || GET_MODE (target) != tmode - || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode)) + || !insn_data[d->icode].operand[0].predicate (target, tmode)) target = gen_reg_rtx (tmode); if ((optimize && !register_operand (op0, mode0)) - || ! (*insn_data[d->icode].operand[1].predicate) (op0, mode0)) + || !insn_data[d->icode].operand[1].predicate (op0, mode0)) op0 = copy_to_mode_reg (mode0, op0); if ((optimize && !register_operand (op1, mode1)) - || ! (*insn_data[d->icode].operand[2].predicate) (op1, mode1)) + || !insn_data[d->icode].operand[2].predicate (op1, mode1)) op1 = copy_to_mode_reg (mode1, op1); op2 = gen_rtx_fmt_ee (comparison, mode0, op0, op1); @@ -23077,10 +26209,10 @@ target = gen_rtx_SUBREG (QImode, target, 0); if ((optimize && !register_operand (op0, mode0)) - || !(*insn_data[d->icode].operand[0].predicate) (op0, mode0)) + || !insn_data[d->icode].operand[0].predicate (op0, mode0)) op0 = copy_to_mode_reg (mode0, op0); if ((optimize && !register_operand (op1, mode1)) - || !(*insn_data[d->icode].operand[1].predicate) (op1, mode1)) + || !insn_data[d->icode].operand[1].predicate (op1, mode1)) op1 = copy_to_mode_reg (mode1, op1); pat = GEN_FCN (d->icode) (op0, op1); @@ -23121,10 +26253,10 @@ target = gen_rtx_SUBREG (QImode, target, 0); if ((optimize && !register_operand (op0, mode0)) - || !(*insn_data[d->icode].operand[0].predicate) (op0, mode0)) + || !insn_data[d->icode].operand[0].predicate (op0, mode0)) op0 = copy_to_mode_reg (mode0, op0); if ((optimize && !register_operand (op1, mode1)) - || !(*insn_data[d->icode].operand[1].predicate) (op1, mode1)) + || !insn_data[d->icode].operand[1].predicate (op1, mode1)) op1 = copy_to_mode_reg (mode1, op1); pat = GEN_FCN (d->icode) (op0, op1); @@ -23173,17 +26305,17 @@ if (VECTOR_MODE_P (modev4)) op2 = safe_vector_operand (op2, modev4); - if (! (*insn_data[d->icode].operand[2].predicate) (op0, modev2)) + if (!insn_data[d->icode].operand[2].predicate (op0, modev2)) op0 = copy_to_mode_reg (modev2, op0); - if (! (*insn_data[d->icode].operand[3].predicate) (op1, modei3)) + if (!insn_data[d->icode].operand[3].predicate (op1, modei3)) op1 = copy_to_mode_reg (modei3, op1); if ((optimize && !register_operand (op2, modev4)) - || !(*insn_data[d->icode].operand[4].predicate) (op2, modev4)) + || !insn_data[d->icode].operand[4].predicate (op2, modev4)) op2 = copy_to_mode_reg (modev4, op2); - if (! (*insn_data[d->icode].operand[5].predicate) (op3, modei5)) + if (!insn_data[d->icode].operand[5].predicate (op3, modei5)) op3 = copy_to_mode_reg (modei5, op3); - if (! (*insn_data[d->icode].operand[6].predicate) (op4, modeimm)) + if (!insn_data[d->icode].operand[6].predicate (op4, modeimm)) { error ("the fifth argument must be a 8-bit immediate"); return const0_rtx; @@ -23193,7 +26325,7 @@ { if (optimize || !target || GET_MODE (target) != tmode0 - || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode0)) + || !insn_data[d->icode].operand[0].predicate (target, tmode0)) target = gen_reg_rtx (tmode0); scratch1 = gen_reg_rtx (tmode1); @@ -23204,7 +26336,7 @@ { if (optimize || !target || GET_MODE (target) != tmode1 - || ! (*insn_data[d->icode].operand[1].predicate) (target, tmode1)) + || !insn_data[d->icode].operand[1].predicate (target, tmode1)) target = gen_reg_rtx (tmode1); scratch0 = gen_reg_rtx (tmode0); @@ -23272,13 +26404,13 @@ if (VECTOR_MODE_P (modev3)) op1 = safe_vector_operand (op1, modev3); - if (! (*insn_data[d->icode].operand[2].predicate) (op0, modev2)) + if (!insn_data[d->icode].operand[2].predicate (op0, modev2)) op0 = copy_to_mode_reg (modev2, op0); if ((optimize && !register_operand (op1, modev3)) - || !(*insn_data[d->icode].operand[3].predicate) (op1, modev3)) + || !insn_data[d->icode].operand[3].predicate (op1, modev3)) op1 = copy_to_mode_reg (modev3, op1); - if (! (*insn_data[d->icode].operand[4].predicate) (op2, modeimm)) + if (!insn_data[d->icode].operand[4].predicate (op2, modeimm)) { error ("the third argument must be a 8-bit immediate"); return const0_rtx; @@ -23288,7 +26420,7 @@ { if (optimize || !target || GET_MODE (target) != tmode0 - || ! (*insn_data[d->icode].operand[0].predicate) (target, tmode0)) + || !insn_data[d->icode].operand[0].predicate (target, tmode0)) target = gen_reg_rtx (tmode0); scratch1 = gen_reg_rtx (tmode1); @@ -23299,7 +26431,7 @@ { if (optimize || !target || GET_MODE (target) != tmode1 - || ! (*insn_data[d->icode].operand[1].predicate) (target, tmode1)) + || !insn_data[d->icode].operand[1].predicate (target, tmode1)) target = gen_reg_rtx (tmode1); scratch0 = gen_reg_rtx (tmode0); @@ -23357,7 +26489,7 @@ } args[4]; bool last_arg_count = false; enum insn_code icode = d->icode; - const struct insn_data *insn_p = &insn_data[icode]; + const struct insn_data_d *insn_p = &insn_data[icode]; enum machine_mode tmode = insn_p->operand[0].mode; enum machine_mode rmode = VOIDmode; bool swap = false; @@ -23395,6 +26527,7 @@ case V8SF_FTYPE_V8SF: case V8SF_FTYPE_V8SI: case V8SF_FTYPE_V4SF: + case V8SF_FTYPE_V8HI: case V4SI_FTYPE_V4SI: case V4SI_FTYPE_V16QI: case V4SI_FTYPE_V4SF: @@ -23411,6 +26544,7 @@ case V4SF_FTYPE_V4SI: case V4SF_FTYPE_V8SF: case V4SF_FTYPE_V4DF: + case V4SF_FTYPE_V8HI: case V4SF_FTYPE_V2DF: case V2DI_FTYPE_V2DI: case V2DI_FTYPE_V16QI: @@ -23513,6 +26647,8 @@ nargs_constant = 1; break; case V8HI_FTYPE_V8HI_INT: + case V8HI_FTYPE_V8SF_INT: + case V8HI_FTYPE_V4SF_INT: case V8SF_FTYPE_V8SF_INT: case V4SI_FTYPE_V4SI_INT: case V4SI_FTYPE_V8SI_INT: @@ -23537,8 +26673,8 @@ case V8HI_FTYPE_V8HI_V8HI_INT: case V8SI_FTYPE_V8SI_V8SI_INT: case V8SI_FTYPE_V8SI_V4SI_INT: - case V8SF_FTYPE_V8SF_V8SF_INT: - case V8SF_FTYPE_V8SF_V4SF_INT: + case V8SF_FTYPE_V8SF_V8SF_INT: + case V8SF_FTYPE_V8SF_V4SF_INT: case V4SI_FTYPE_V4SI_V4SI_INT: case V4DF_FTYPE_V4DF_V4DF_INT: case V4DF_FTYPE_V4DF_V2DF_INT: @@ -23562,6 +26698,13 @@ nargs = 3; nargs_constant = 2; break; + case V2DF_FTYPE_V2DF_V2DF_V2DI_INT: + case V4DF_FTYPE_V4DF_V4DF_V4DI_INT: + case V4SF_FTYPE_V4SF_V4SF_V4SI_INT: + case V8SF_FTYPE_V8SF_V8SF_V8SI_INT: + nargs = 4; + nargs_constant = 1; + break; case V2DI_FTYPE_V2DI_V2DI_UINT_UINT: nargs = 4; nargs_constant = 2; @@ -23583,7 +26726,7 @@ if (optimize || target == 0 || GET_MODE (target) != tmode - || ! (*insn_p->operand[0].predicate) (target, tmode)) + || !insn_p->operand[0].predicate (target, tmode)) target = gen_reg_rtx (tmode); real_target = target; } @@ -23598,7 +26741,7 @@ tree arg = CALL_EXPR_ARG (exp, i); rtx op = expand_normal (arg); enum machine_mode mode = insn_p->operand[i + 1].mode; - bool match = (*insn_p->operand[i + 1].predicate) (op, mode); + bool match = insn_p->operand[i + 1].predicate (op, mode); if (last_arg_count && (i + 1) == nargs) { @@ -23608,7 +26751,7 @@ if (!match) { op = simplify_gen_subreg (SImode, op, GET_MODE (op), 0); - if (!(*insn_p->operand[i + 1].predicate) (op, mode)) + if (!insn_p->operand[i + 1].predicate (op, mode)) op = copy_to_reg (op); } } @@ -23631,6 +26774,10 @@ case CODE_FOR_sse4_1_blendpd: case CODE_FOR_avx_vpermilv2df: + case CODE_FOR_xop_vpermil2v2df3: + case CODE_FOR_xop_vpermil2v4sf3: + case CODE_FOR_xop_vpermil2v4df3: + case CODE_FOR_xop_vpermil2v8sf3: error ("the last argument must be a 2-bit immediate"); return const0_rtx; @@ -23740,16 +26887,26 @@ } args[3]; enum insn_code icode = d->icode; bool last_arg_constant = false; - const struct insn_data *insn_p = &insn_data[icode]; + const struct insn_data_d *insn_p = &insn_data[icode]; enum machine_mode tmode = insn_p->operand[0].mode; enum { load, store } klass; switch ((enum ix86_builtin_func_type) d->flag) { case VOID_FTYPE_VOID: + if (icode == CODE_FOR_avx_vzeroupper) + target = GEN_INT (vzeroupper_intrinsic); emit_insn (GEN_FCN (icode) (target)); return 0; + case VOID_FTYPE_UINT64: + case VOID_FTYPE_UNSIGNED: + nargs = 0; + klass = store; + memory = 0; + break; + break; case UINT64_FTYPE_VOID: + case UNSIGNED_FTYPE_VOID: nargs = 0; klass = load; memory = 0; @@ -23791,18 +26948,18 @@ klass = load; memory = 1; break; - case V8SF_FTYPE_PCV8SF_V8SF: - case V4DF_FTYPE_PCV4DF_V4DF: - case V4SF_FTYPE_PCV4SF_V4SF: - case V2DF_FTYPE_PCV2DF_V2DF: + case V8SF_FTYPE_PCV8SF_V8SI: + case V4DF_FTYPE_PCV4DF_V4DI: + case V4SF_FTYPE_PCV4SF_V4SI: + case V2DF_FTYPE_PCV2DF_V2DI: nargs = 2; klass = load; memory = 0; break; - case VOID_FTYPE_PV8SF_V8SF_V8SF: - case VOID_FTYPE_PV4DF_V4DF_V4DF: - case VOID_FTYPE_PV4SF_V4SF_V4SF: - case VOID_FTYPE_PV2DF_V2DF_V2DF: + case VOID_FTYPE_PV8SF_V8SI_V8SF: + case VOID_FTYPE_PV4DF_V4DI_V4DF: + case VOID_FTYPE_PV4SF_V4SI_V4SF: + case VOID_FTYPE_PV2DF_V2DI_V2DF: nargs = 2; klass = store; /* Reserve memory operand for target. */ @@ -23828,7 +26985,10 @@ arg = CALL_EXPR_ARG (exp, 0); op = expand_normal (arg); gcc_assert (target == 0); - target = gen_rtx_MEM (tmode, copy_to_mode_reg (Pmode, op)); + if (memory) + target = gen_rtx_MEM (tmode, copy_to_mode_reg (Pmode, op)); + else + target = force_reg (tmode, op); arg_adjust = 1; } else @@ -23837,7 +26997,7 @@ if (optimize || target == 0 || GET_MODE (target) != tmode - || ! (*insn_p->operand[0].predicate) (target, tmode)) + || !insn_p->operand[0].predicate (target, tmode)) target = gen_reg_rtx (tmode); } @@ -23848,7 +27008,7 @@ arg = CALL_EXPR_ARG (exp, i + arg_adjust); op = expand_normal (arg); - match = (*insn_p->operand[i + 1].predicate) (op, mode); + match = insn_p->operand[i + 1].predicate (op, mode); if (last_arg_constant && (i + 1) == nargs) { @@ -24098,11 +27258,11 @@ op0 = force_reg (Pmode, op0); op0 = gen_rtx_MEM (mode1, op0); - if (! (*insn_data[icode].operand[0].predicate) (op0, mode0)) + if (!insn_data[icode].operand[0].predicate (op0, mode0)) op0 = copy_to_mode_reg (mode0, op0); - if (! (*insn_data[icode].operand[1].predicate) (op1, mode1)) + if (!insn_data[icode].operand[1].predicate (op1, mode1)) op1 = copy_to_mode_reg (mode1, op1); - if (! (*insn_data[icode].operand[2].predicate) (op2, mode2)) + if (!insn_data[icode].operand[2].predicate (op2, mode2)) op2 = copy_to_mode_reg (mode2, op2); pat = GEN_FCN (icode) (op0, op1, op2); if (! pat) @@ -24126,7 +27286,7 @@ arg0 = CALL_EXPR_ARG (exp, 0); op0 = expand_normal (arg0); icode = CODE_FOR_sse2_clflush; - if (! (*insn_data[icode].operand[0].predicate) (op0, Pmode)) + if (!insn_data[icode].operand[0].predicate (op0, Pmode)) op0 = copy_to_mode_reg (Pmode, op0); emit_insn (gen_sse2_clflush (op0)); @@ -24145,7 +27305,7 @@ op1 = copy_to_mode_reg (SImode, op1); if (!REG_P (op2)) op2 = copy_to_mode_reg (SImode, op2); - emit_insn ((*ix86_gen_monitor) (op0, op1, op2)); + emit_insn (ix86_gen_monitor (op0, op1, op2)); return 0; case IX86_BUILTIN_MWAIT: @@ -24219,7 +27379,7 @@ arg0 = CALL_EXPR_ARG (exp, 0); op0 = expand_normal (arg0); icode = CODE_FOR_lwp_llwpcb; - if (! (*insn_data[icode].operand[0].predicate) (op0, Pmode)) + if (!insn_data[icode].operand[0].predicate (op0, Pmode)) op0 = copy_to_mode_reg (Pmode, op0); emit_insn (gen_lwp_llwpcb (op0)); return 0; @@ -24227,11 +27387,82 @@ case IX86_BUILTIN_SLWPCB: icode = CODE_FOR_lwp_slwpcb; if (!target - || ! (*insn_data[icode].operand[0].predicate) (target, Pmode)) + || !insn_data[icode].operand[0].predicate (target, Pmode)) target = gen_reg_rtx (Pmode); emit_insn (gen_lwp_slwpcb (target)); return target; + case IX86_BUILTIN_BEXTRI32: + case IX86_BUILTIN_BEXTRI64: + arg0 = CALL_EXPR_ARG (exp, 0); + arg1 = CALL_EXPR_ARG (exp, 1); + op0 = expand_normal (arg0); + op1 = expand_normal (arg1); + icode = (fcode == IX86_BUILTIN_BEXTRI32 + ? CODE_FOR_tbm_bextri_si + : CODE_FOR_tbm_bextri_di); + if (!CONST_INT_P (op1)) + { + error ("last argument must be an immediate"); + return const0_rtx; + } + else + { + unsigned char length = (INTVAL (op1) >> 8) & 0xFF; + unsigned char lsb_index = INTVAL (op1) & 0xFF; + op1 = GEN_INT (length); + op2 = GEN_INT (lsb_index); + pat = GEN_FCN (icode) (target, op0, op1, op2); + if (pat) + emit_insn (pat); + return target; + } + + case IX86_BUILTIN_RDRAND16_STEP: + icode = CODE_FOR_rdrandhi_1; + mode0 = HImode; + goto rdrand_step; + + case IX86_BUILTIN_RDRAND32_STEP: + icode = CODE_FOR_rdrandsi_1; + mode0 = SImode; + goto rdrand_step; + + case IX86_BUILTIN_RDRAND64_STEP: + icode = CODE_FOR_rdranddi_1; + mode0 = DImode; + +rdrand_step: + op0 = gen_reg_rtx (mode0); + emit_insn (GEN_FCN (icode) (op0)); + + op1 = gen_reg_rtx (SImode); + emit_move_insn (op1, CONST1_RTX (SImode)); + + /* Emit SImode conditional move. */ + if (mode0 == HImode) + { + op2 = gen_reg_rtx (SImode); + emit_insn (gen_zero_extendhisi2 (op2, op0)); + } + else if (mode0 == SImode) + op2 = op0; + else + op2 = gen_rtx_SUBREG (SImode, op0, 0); + + pat = gen_rtx_GEU (VOIDmode, gen_rtx_REG (CCCmode, FLAGS_REG), + const0_rtx); + emit_insn (gen_rtx_SET (VOIDmode, op1, + gen_rtx_IF_THEN_ELSE (SImode, pat, op2, op1))); + emit_move_insn (target, op1); + + arg0 = CALL_EXPR_ARG (exp, 0); + op1 = expand_normal (arg0); + if (!address_operand (op1, VOIDmode)) + op1 = copy_addr_to_reg (op1); + emit_move_insn (gen_rtx_MEM (mode0, op1), op0); + return target; + default: break; } @@ -24287,14 +27518,16 @@ if it is not available. */ static tree -ix86_builtin_vectorized_function (unsigned int fn, tree type_out, +ix86_builtin_vectorized_function (tree fndecl, tree type_out, tree type_in) { enum machine_mode in_mode, out_mode; int in_n, out_n; + enum built_in_function fn = DECL_FUNCTION_CODE (fndecl); if (TREE_CODE (type_out) != VECTOR_TYPE - || TREE_CODE (type_in) != VECTOR_TYPE) + || TREE_CODE (type_in) != VECTOR_TYPE + || DECL_BUILT_IN_CLASS (fndecl) != BUILT_IN_NORMAL) return NULL_TREE; out_mode = TYPE_MODE (TREE_TYPE (type_out)); @@ -24305,15 +27538,23 @@ switch (fn) { case BUILT_IN_SQRT: - if (out_mode == DFmode && out_n == 2 - && in_mode == DFmode && in_n == 2) - return ix86_builtins[IX86_BUILTIN_SQRTPD]; + if (out_mode == DFmode && in_mode == DFmode) + { + if (out_n == 2 && in_n == 2) + return ix86_builtins[IX86_BUILTIN_SQRTPD]; + else if (out_n == 4 && in_n == 4) + return ix86_builtins[IX86_BUILTIN_SQRTPD256]; + } break; case BUILT_IN_SQRTF: - if (out_mode == SFmode && out_n == 4 - && in_mode == SFmode && in_n == 4) - return ix86_builtins[IX86_BUILTIN_SQRTPS_NR]; + if (out_mode == SFmode && in_mode == SFmode) + { + if (out_n == 4 && in_n == 4) + return ix86_builtins[IX86_BUILTIN_SQRTPS_NR]; + else if (out_n == 8 && in_n == 8) + return ix86_builtins[IX86_BUILTIN_SQRTPS_NR256]; + } break; case BUILT_IN_LRINT: @@ -24323,31 +27564,63 @@ break; case BUILT_IN_LRINTF: - if (out_mode == SImode && out_n == 4 - && in_mode == SFmode && in_n == 4) - return ix86_builtins[IX86_BUILTIN_CVTPS2DQ]; + if (out_mode == SImode && in_mode == SFmode) + { + if (out_n == 4 && in_n == 4) + return ix86_builtins[IX86_BUILTIN_CVTPS2DQ]; + else if (out_n == 8 && in_n == 8) + return ix86_builtins[IX86_BUILTIN_CVTPS2DQ256]; + } break; case BUILT_IN_COPYSIGN: - if (out_mode == DFmode && out_n == 2 - && in_mode == DFmode && in_n == 2) - return ix86_builtins[IX86_BUILTIN_CPYSGNPD]; + if (out_mode == DFmode && in_mode == DFmode) + { + if (out_n == 2 && in_n == 2) + return ix86_builtins[IX86_BUILTIN_CPYSGNPD]; + else if (out_n == 4 && in_n == 4) + return ix86_builtins[IX86_BUILTIN_CPYSGNPD256]; + } break; case BUILT_IN_COPYSIGNF: - if (out_mode == SFmode && out_n == 4 - && in_mode == SFmode && in_n == 4) - return ix86_builtins[IX86_BUILTIN_CPYSGNPS]; - break; - - default: - ; + if (out_mode == SFmode && in_mode == SFmode) + { + if (out_n == 4 && in_n == 4) + return ix86_builtins[IX86_BUILTIN_CPYSGNPS]; + else if (out_n == 8 && in_n == 8) + return ix86_builtins[IX86_BUILTIN_CPYSGNPS256]; + } + break; + + case BUILT_IN_FMA: + if (out_mode == DFmode && in_mode == DFmode) + { + if (out_n == 2 && in_n == 2) + return ix86_builtins[IX86_BUILTIN_VFMADDPD]; + if (out_n == 4 && in_n == 4) + return ix86_builtins[IX86_BUILTIN_VFMADDPD256]; + } + break; + + case BUILT_IN_FMAF: + if (out_mode == SFmode && in_mode == SFmode) + { + if (out_n == 4 && in_n == 4) + return ix86_builtins[IX86_BUILTIN_VFMADDPS]; + if (out_n == 8 && in_n == 8) + return ix86_builtins[IX86_BUILTIN_VFMADDPS256]; + } + break; + + default: + break; } /* Dispatch to a handler for a vectorization library. */ if (ix86_veclib_handler) - return (*ix86_veclib_handler) ((enum built_in_function) fn, type_out, - type_in); + return ix86_veclib_handler ((enum built_in_function) fn, type_out, + type_in); return NULL_TREE; } @@ -24552,43 +27825,92 @@ /* Returns a decl of a function that implements conversion of an integer vector - into a floating-point vector, or vice-versa. TYPE is the type of the integer - side of the conversion. + into a floating-point vector, or vice-versa. DEST_TYPE and SRC_TYPE + are the types involved when converting according to CODE. Return NULL_TREE if it is not available. */ static tree -ix86_vectorize_builtin_conversion (unsigned int code, tree type) -{ - if (! (TARGET_SSE2 && TREE_CODE (type) == VECTOR_TYPE)) +ix86_vectorize_builtin_conversion (unsigned int code, + tree dest_type, tree src_type) +{ + if (! TARGET_SSE2) return NULL_TREE; switch (code) { case FLOAT_EXPR: - switch (TYPE_MODE (type)) + switch (TYPE_MODE (src_type)) { case V4SImode: - return TYPE_UNSIGNED (type) - ? ix86_builtins[IX86_BUILTIN_CVTUDQ2PS] - : ix86_builtins[IX86_BUILTIN_CVTDQ2PS]; + switch (TYPE_MODE (dest_type)) + { + case V4SFmode: + return (TYPE_UNSIGNED (src_type) + ? ix86_builtins[IX86_BUILTIN_CVTUDQ2PS] + : ix86_builtins[IX86_BUILTIN_CVTDQ2PS]); + case V4DFmode: + return (TYPE_UNSIGNED (src_type) + ? NULL_TREE + : ix86_builtins[IX86_BUILTIN_CVTDQ2PD256]); + default: + return NULL_TREE; + } + break; + case V8SImode: + switch (TYPE_MODE (dest_type)) + { + case V8SFmode: + return (TYPE_UNSIGNED (src_type) + ? NULL_TREE + : ix86_builtins[IX86_BUILTIN_CVTDQ2PS256]); + default: + return NULL_TREE; + } + break; default: return NULL_TREE; } case FIX_TRUNC_EXPR: - switch (TYPE_MODE (type)) + switch (TYPE_MODE (dest_type)) { case V4SImode: - return TYPE_UNSIGNED (type) - ? NULL_TREE - : ix86_builtins[IX86_BUILTIN_CVTTPS2DQ]; + switch (TYPE_MODE (src_type)) + { + case V4SFmode: + return (TYPE_UNSIGNED (dest_type) + ? NULL_TREE + : ix86_builtins[IX86_BUILTIN_CVTTPS2DQ]); + case V4DFmode: + return (TYPE_UNSIGNED (dest_type) + ? NULL_TREE + : ix86_builtins[IX86_BUILTIN_CVTTPD2DQ256]); + default: + return NULL_TREE; + } + break; + + case V8SImode: + switch (TYPE_MODE (src_type)) + { + case V8SFmode: + return (TYPE_UNSIGNED (dest_type) + ? NULL_TREE + : ix86_builtins[IX86_BUILTIN_CVTTPS2DQ256]); + default: + return NULL_TREE; + } + break; + default: return NULL_TREE; } + default: return NULL_TREE; - - } + } + + return NULL_TREE; } /* Returns a code for a target-specific builtin that implements @@ -24611,6 +27933,9 @@ case IX86_BUILTIN_SQRTPS_NR: return ix86_builtins[IX86_BUILTIN_RSQRTPS_NR]; + case IX86_BUILTIN_SQRTPS_NR256: + return ix86_builtins[IX86_BUILTIN_RSQRTPS_NR256]; + default: return NULL_TREE; } @@ -24652,7 +27977,7 @@ if (!CONST_INT_P (er)) return 0; ei = INTVAL (er); - if (ei >= 2 * nelt) + if (ei >= nelt) return 0; ipar[i] = ei; } @@ -24763,7 +28088,7 @@ rtx result; gcc_assert (reload_completed); - if (!TARGET_64BIT_MS_ABI && TARGET_RED_ZONE) + if (ix86_using_red_zone ()) { result = gen_rtx_MEM (mode, gen_rtx_PLUS (Pmode, @@ -24771,7 +28096,7 @@ GEN_INT (-RED_ZONE_SIZE))); emit_move_insn (result, operand); } - else if ((TARGET_64BIT_MS_ABI || !TARGET_RED_ZONE) && TARGET_64BIT) + else if (TARGET_64BIT) { switch (mode) { @@ -24799,7 +28124,7 @@ case DImode: { rtx operands[2]; - split_di (&operand, 1, operands, operands + 1); + split_double_mode (mode, &operand, 1, operands, operands + 1); emit_insn ( gen_rtx_SET (VOIDmode, gen_rtx_MEM (SImode, @@ -24838,7 +28163,7 @@ void ix86_free_from_memory (enum machine_mode mode) { - if (!TARGET_RED_ZONE || TARGET_64BIT_MS_ABI) + if (!ix86_using_red_zone ()) { int size; @@ -24857,25 +28182,28 @@ /* Implement TARGET_IRA_COVER_CLASSES. If -mfpmath=sse, we prefer SSE_REGS to FLOAT_REGS if their costs for a pseudo are the same. */ -static const enum reg_class * +static const reg_class_t * i386_ira_cover_classes (void) { - static const enum reg_class sse_fpmath_classes[] = { + static const reg_class_t sse_fpmath_classes[] = { GENERAL_REGS, SSE_REGS, MMX_REGS, FLOAT_REGS, LIM_REG_CLASSES }; - static const enum reg_class no_sse_fpmath_classes[] = { + static const reg_class_t no_sse_fpmath_classes[] = { GENERAL_REGS, FLOAT_REGS, MMX_REGS, SSE_REGS, LIM_REG_CLASSES }; return TARGET_SSE_MATH ? sse_fpmath_classes : no_sse_fpmath_classes; } -/* Put float CONST_DOUBLE in the constant pool instead of fp regs. +/* Implement TARGET_PREFERRED_RELOAD_CLASS. + + Put float CONST_DOUBLE in the constant pool instead of fp regs. QImode must go into class Q_REGS. Narrow ALL_REGS to GENERAL_REGS. This supports allowing movsf and movdf to do mem-to-mem moves through integer regs. */ -enum reg_class -ix86_preferred_reload_class (rtx x, enum reg_class regclass) + +static reg_class_t +ix86_preferred_reload_class (rtx x, reg_class_t regclass) { enum machine_mode mode = GET_MODE (x); @@ -24948,8 +28276,8 @@ /* Discourage putting floating-point values in SSE registers unless SSE math is being used, and likewise for the 387 registers. */ -enum reg_class -ix86_preferred_output_reload_class (rtx x, enum reg_class regclass) +static reg_class_t +ix86_preferred_output_reload_class (rtx x, reg_class_t regclass) { enum machine_mode mode = GET_MODE (x); @@ -24973,14 +28301,15 @@ return regclass; } -static enum reg_class -ix86_secondary_reload (bool in_p, rtx x, enum reg_class rclass, +static reg_class_t +ix86_secondary_reload (bool in_p, rtx x, reg_class_t rclass, enum machine_mode mode, secondary_reload_info *sri ATTRIBUTE_UNUSED) { /* QImode spills from non-QI registers require intermediate register on 32bit targets. */ - if (!in_p && mode == QImode && !TARGET_64BIT + if (!TARGET_64BIT + && !in_p && mode == QImode && (rclass == GENERAL_REGS || rclass == LEGACY_REGS || rclass == INDEX_REGS)) @@ -25000,9 +28329,74 @@ return Q_REGS; } + /* This condition handles corner case where an expression involving + pointers gets vectorized. We're trying to use the address of a + stack slot as a vector initializer. + + (set (reg:V2DI 74 [ vect_cst_.2 ]) + (vec_duplicate:V2DI (reg/f:DI 20 frame))) + + Eventually frame gets turned into sp+offset like this: + + (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74]) + (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp) + (const_int 392 [0x188])))) + + That later gets turned into: + + (set (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74]) + (vec_duplicate:V2DI (plus:DI (reg/f:DI 7 sp) + (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64])))) + + We'll have the following reload recorded: + + Reload 0: reload_in (DI) = + (plus:DI (reg/f:DI 7 sp) + (mem/u/c/i:DI (symbol_ref/u:DI ("*.LC0") [flags 0x2]) [0 S8 A64])) + reload_out (V2DI) = (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74]) + SSE_REGS, RELOAD_OTHER (opnum = 0), can't combine + reload_in_reg: (plus:DI (reg/f:DI 7 sp) (const_int 392 [0x188])) + reload_out_reg: (reg:V2DI 21 xmm0 [orig:74 vect_cst_.2 ] [74]) + reload_reg_rtx: (reg:V2DI 22 xmm1) + + Which isn't going to work since SSE instructions can't handle scalar + additions. Returning GENERAL_REGS forces the addition into integer + register and reload can handle subsequent reloads without problems. */ + + if (in_p && GET_CODE (x) == PLUS + && SSE_CLASS_P (rclass) + && SCALAR_INT_MODE_P (mode)) + return GENERAL_REGS; + return NO_REGS; } +/* Implement TARGET_CLASS_LIKELY_SPILLED_P. */ + +static bool +ix86_class_likely_spilled_p (reg_class_t rclass) +{ + switch (rclass) + { + case AREG: + case DREG: + case CREG: + case BREG: + case AD_REGS: + case SIREG: + case DIREG: + case SSE_FIRST_REG: + case FP_TOP_REG: + case FP_SECOND_REG: + return true; + + default: + break; + } + + return false; +} + /* If we are copying between general and FP registers, we need a memory location. The same is true for SSE and MMX registers. @@ -25016,9 +28410,9 @@ When STRICT is false, we are being called from REGISTER_MOVE_COST, so do not enforce these sanity checks. */ -static inline int +static inline bool inline_secondary_memory_needed (enum reg_class class1, enum reg_class class2, - enum machine_mode mode, int strict) + enum machine_mode mode, int strict) { if (MAYBE_FLOAT_CLASS_P (class1) != FLOAT_CLASS_P (class1) || MAYBE_FLOAT_CLASS_P (class2) != FLOAT_CLASS_P (class2) @@ -25059,7 +28453,7 @@ return false; } -int +bool ix86_secondary_memory_needed (enum reg_class class1, enum reg_class class2, enum machine_mode mode, int strict) { @@ -25226,10 +28620,11 @@ } } -int -ix86_memory_move_cost (enum machine_mode mode, enum reg_class regclass, int in) -{ - return inline_memory_move_cost (mode, regclass, in); +static int +ix86_memory_move_cost (enum machine_mode mode, reg_class_t regclass, + bool in) +{ + return inline_memory_move_cost (mode, (enum reg_class) regclass, in ? 1 : 0); } @@ -25240,10 +28635,13 @@ on some machines it is expensive to move between registers if they are not general registers. */ -int -ix86_register_move_cost (enum machine_mode mode, enum reg_class class1, - enum reg_class class2) -{ +static int +ix86_register_move_cost (enum machine_mode mode, reg_class_t class1_i, + reg_class_t class2_i) +{ + enum reg_class class1 = (enum reg_class) class1_i; + enum reg_class class2 = (enum reg_class) class2_i; + /* In case we require secondary memory, compute cost of the store followed by load. In order to avoid bad register allocation choices, we need for this to be *at least* as high as the symmetric MEMORY_MOVE_COST. */ @@ -25538,6 +28936,31 @@ } return false; + case FMA: + { + rtx sub; + + gcc_assert (FLOAT_MODE_P (mode)); + gcc_assert (TARGET_FMA || TARGET_FMA4); + + /* ??? SSE scalar/vector cost should be used here. */ + /* ??? Bald assumption that fma has the same cost as fmul. */ + *total = cost->fmul; + *total += rtx_cost (XEXP (x, 1), FMA, speed); + + /* Negate in op0 or op2 is free: FMS, FNMA, FNMS. */ + sub = XEXP (x, 0); + if (GET_CODE (sub) == NEG) + sub = XEXP (x, 0); + *total += rtx_cost (sub, FMA, speed); + + sub = XEXP (x, 2); + if (GET_CODE (sub) == NEG) + sub = XEXP (x, 0); + *total += rtx_cost (sub, FMA, speed); + return true; + } + case MULT: if (SSE_FLOAT_MODE_P (mode) && TARGET_SSE_MATH) { @@ -25801,7 +29224,7 @@ gcc_assert (!TARGET_64BIT); /* Lose our funky encoding stuff so it doesn't contaminate the stub. */ - symb = (*targetm.strip_name_encoding) (symb); + symb = targetm.strip_name_encoding (symb); length = strlen (stub); binder_name = XALLOCAVEC (char, length + 32); @@ -25813,47 +29236,85 @@ sprintf (lazy_ptr_name, "L%d$lz", label); - if (MACHOPIC_PURE) + if (MACHOPIC_ATT_STUB) + switch_to_section (darwin_sections[machopic_picsymbol_stub3_section]); + else if (MACHOPIC_PURE) + { + if (TARGET_DEEP_BRANCH_PREDICTION) + switch_to_section (darwin_sections[machopic_picsymbol_stub2_section]); + else switch_to_section (darwin_sections[machopic_picsymbol_stub_section]); + } else switch_to_section (darwin_sections[machopic_symbol_stub_section]); fprintf (file, "%s:\n", stub); fprintf (file, "\t.indirect_symbol %s\n", symbol_name); - if (MACHOPIC_PURE) - { - fprintf (file, "\tcall\tLPC$%d\nLPC$%d:\tpopl\t%%eax\n", label, label); - fprintf (file, "\tmovl\t%s-LPC$%d(%%eax),%%edx\n", lazy_ptr_name, label); - fprintf (file, "\tjmp\t*%%edx\n"); + if (MACHOPIC_ATT_STUB) + { + fprintf (file, "\thlt ; hlt ; hlt ; hlt ; hlt\n"); + } + else if (MACHOPIC_PURE) + { + /* PIC stub. */ + if (TARGET_DEEP_BRANCH_PREDICTION) + { + /* 25-byte PIC stub using "CALL get_pc_thunk". */ + rtx tmp = gen_rtx_REG (SImode, 2 /* ECX */); + output_set_got (tmp, NULL_RTX); /* "CALL ___<cpu>.get_pc_thunk.cx". */ + fprintf (file, "LPC$%d:\tmovl\t%s-LPC$%d(%%ecx),%%ecx\n", label, lazy_ptr_name, label); + } + else + { + /* 26-byte PIC stub using inline picbase: "CALL L42 ! L42: pop %eax". */ + fprintf (file, "\tcall LPC$%d\nLPC$%d:\tpopl %%ecx\n", label, label); + fprintf (file, "\tmovl %s-LPC$%d(%%ecx),%%ecx\n", lazy_ptr_name, label); + } + fprintf (file, "\tjmp\t*%%ecx\n"); } else fprintf (file, "\tjmp\t*%s\n", lazy_ptr_name); + /* The AT&T-style ("self-modifying") stub is not lazily bound, thus + it needs no stub-binding-helper. */ + if (MACHOPIC_ATT_STUB) + return; + fprintf (file, "%s:\n", binder_name); if (MACHOPIC_PURE) { - fprintf (file, "\tlea\t%s-LPC$%d(%%eax),%%eax\n", lazy_ptr_name, label); - fputs ("\tpushl\t%eax\n", file); + fprintf (file, "\tlea\t%s-%s(%%ecx),%%ecx\n", lazy_ptr_name, binder_name); + fprintf (file, "\tpushl\t%%ecx\n"); } else fprintf (file, "\tpushl\t$%s\n", lazy_ptr_name); fputs ("\tjmp\tdyld_stub_binding_helper\n", file); + /* N.B. Keep the correspondence of these + 'symbol_ptr/symbol_ptr2/symbol_ptr3' sections consistent with the + old-pic/new-pic/non-pic stubs; altering this will break + compatibility with existing dylibs. */ + if (MACHOPIC_PURE) + { + /* PIC stubs. */ + if (TARGET_DEEP_BRANCH_PREDICTION) + /* 25-byte PIC stub using "CALL get_pc_thunk". */ + switch_to_section (darwin_sections[machopic_lazy_symbol_ptr2_section]); + else + /* 26-byte PIC stub using inline picbase: "CALL L42 ! L42: pop %ebx". */ switch_to_section (darwin_sections[machopic_lazy_symbol_ptr_section]); + } + else + /* 16-byte -mdynamic-no-pic stub. */ + switch_to_section(darwin_sections[machopic_lazy_symbol_ptr3_section]); + fprintf (file, "%s:\n", lazy_ptr_name); fprintf (file, "\t.indirect_symbol %s\n", symbol_name); fprintf (file, ASM_LONG "%s\n", binder_name); } - -void -darwin_x86_file_end (void) -{ - darwin_file_end (); - ix86_file_end (); -} #endif /* TARGET_MACHO */ /* Order the registers for register allocator. */ @@ -25900,6 +29361,58 @@ reg_alloc_order [pos++] = 0; } +/* Handle a "callee_pop_aggregate_return" attribute; arguments as + in struct attribute_spec handler. */ +static tree +ix86_handle_callee_pop_aggregate_return (tree *node, tree name, + tree args, + int flags ATTRIBUTE_UNUSED, + bool *no_add_attrs) +{ + if (TREE_CODE (*node) != FUNCTION_TYPE + && TREE_CODE (*node) != METHOD_TYPE + && TREE_CODE (*node) != FIELD_DECL + && TREE_CODE (*node) != TYPE_DECL) + { + warning (OPT_Wattributes, "%qE attribute only applies to functions", + name); + *no_add_attrs = true; + return NULL_TREE; + } + if (TARGET_64BIT) + { + warning (OPT_Wattributes, "%qE attribute only available for 32-bit", + name); + *no_add_attrs = true; + return NULL_TREE; + } + if (is_attribute_p ("callee_pop_aggregate_return", name)) + { + tree cst; + + cst = TREE_VALUE (args); + if (TREE_CODE (cst) != INTEGER_CST) + { + warning (OPT_Wattributes, + "%qE attribute requires an integer constant argument", + name); + *no_add_attrs = true; + } + else if (compare_tree_int (cst, 0) != 0 + && compare_tree_int (cst, 1) != 0) + { + warning (OPT_Wattributes, + "argument to %qE attribute is neither zero, nor one", + name); + *no_add_attrs = true; + } + + return NULL_TREE; + } + + return NULL_TREE; +} + /* Handle a "ms_abi" or "sysv" attribute; arguments as in struct attribute_spec.handler. */ static tree @@ -25995,29 +29508,16 @@ warning (OPT_Wattributes, "%qE attribute only applies to functions", name); *no_add_attrs = true; - return NULL_TREE; - } - - if (TARGET_64BIT) - { - warning (OPT_Wattributes, "%qE attribute only available for 32-bit", - name); - return NULL_TREE; - } - -#ifndef HAVE_AS_IX86_SWAP - sorry ("ms_hook_prologue attribute needs assembler swap suffix support"); -#endif - - return NULL_TREE; + } + return NULL_TREE; } static bool ix86_ms_bitfield_layout_p (const_tree record_type) { - return (TARGET_MS_BITFIELD_LAYOUT && - !lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (record_type))) - || lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (record_type)); + return ((TARGET_MS_BITFIELD_LAYOUT + && !lookup_attribute ("gcc_struct", TYPE_ATTRIBUTES (record_type))) + || lookup_attribute ("ms_struct", TYPE_ATTRIBUTES (record_type))); } /* Returns an expression indicating where the this parameter is @@ -26049,6 +29549,13 @@ if (lookup_attribute ("fastcall", TYPE_ATTRIBUTES (type))) regno = aggr ? DX_REG : CX_REG; + else if (lookup_attribute ("thiscall", TYPE_ATTRIBUTES (type))) + { + regno = CX_REG; + if (aggr) + return gen_rtx_MEM (SImode, + plus_constant (stack_pointer_rtx, 4)); + } else { regno = AX_REG; @@ -26086,7 +29593,7 @@ return false; /* Need a free register for GOT references. */ - if (flag_pic && !(*targetm.binds_local_p) (function)) + if (flag_pic && !targetm.binds_local_p (function)) return false; /* Otherwise ok. */ @@ -26100,7 +29607,7 @@ *(*this + vcall_offset) should be added to THIS. */ static void -x86_output_mi_thunk (FILE *file ATTRIBUTE_UNUSED, +x86_output_mi_thunk (FILE *file, tree thunk ATTRIBUTE_UNUSED, HOST_WIDE_INT delta, HOST_WIDE_INT vcall_offset, tree function) { @@ -26108,6 +29615,9 @@ rtx this_param = x86_this_parameter (function); rtx this_reg, tmp; + /* Make sure unwind info is emitted for the thunk if needed. */ + final_start_function (emit_barrier (), file, 1); + /* If VCALL_OFFSET, we'll need THIS in a register. Might as well pull it in now and let DELTA benefit. */ if (REG_P (this_param)) @@ -26125,10 +29635,7 @@ /* Adjust the this parameter by a fixed constant. */ if (delta) { - /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'. - Exceptions: -128 encodes smaller than 128, so swap sign and op. */ - bool sub = delta < 0 || delta == 128; - xops[0] = GEN_INT (sub ? -delta : delta); + xops[0] = GEN_INT (delta); xops[1] = this_reg ? this_reg : this_param; if (TARGET_64BIT) { @@ -26140,12 +29647,12 @@ xops[0] = tmp; xops[1] = this_param; } - if (sub) + if (x86_maybe_negate_const_int (&xops[0], DImode)) output_asm_insn ("sub{q}\t{%0, %1|%1, %0}", xops); else output_asm_insn ("add{q}\t{%0, %1|%1, %0}", xops); } - else if (sub) + else if (x86_maybe_negate_const_int (&xops[0], SImode)) output_asm_insn ("sub{l}\t{%0, %1|%1, %0}", xops); else output_asm_insn ("add{l}\t{%0, %1|%1, %0}", xops); @@ -26160,7 +29667,9 @@ { int tmp_regno = CX_REG; if (lookup_attribute ("fastcall", - TYPE_ATTRIBUTES (TREE_TYPE (function)))) + TYPE_ATTRIBUTES (TREE_TYPE (function))) + || lookup_attribute ("thiscall", + TYPE_ATTRIBUTES (TREE_TYPE (function)))) tmp_regno = AX_REG; tmp = gen_rtx_REG (SImode, tmp_regno); } @@ -26194,7 +29703,8 @@ xops[0] = XEXP (DECL_RTL (function), 0); if (TARGET_64BIT) { - if (!flag_pic || (*targetm.binds_local_p) (function)) + if (!flag_pic || targetm.binds_local_p (function) + || DEFAULT_ABI == MS_ABI) output_asm_insn ("jmp\t%P0", xops); /* All thunks should be in the same object as their target, and thus binds_local_p should be true. */ @@ -26211,17 +29721,18 @@ } else { - if (!flag_pic || (*targetm.binds_local_p) (function)) + if (!flag_pic || targetm.binds_local_p (function)) output_asm_insn ("jmp\t%P0", xops); else #if TARGET_MACHO if (TARGET_MACHO) { rtx sym_ref = XEXP (DECL_RTL (function), 0); - tmp = (gen_rtx_SYMBOL_REF + if (TARGET_MACHO_BRANCH_ISLANDS) + sym_ref = (gen_rtx_SYMBOL_REF (Pmode, machopic_indirection_name (sym_ref, /*stub_p=*/true))); - tmp = gen_rtx_MEM (QImode, tmp); + tmp = gen_rtx_MEM (QImode, sym_ref); xops[0] = tmp; output_asm_insn ("jmp\t%0", xops); } @@ -26236,6 +29747,7 @@ output_asm_insn ("jmp\t{*}%1", xops); } } + final_end_function (); } static void @@ -26274,36 +29786,38 @@ void x86_function_profiler (FILE *file, int labelno ATTRIBUTE_UNUSED) { + const char *mcount_name = (flag_fentry ? MCOUNT_NAME_BEFORE_PROLOGUE + : MCOUNT_NAME); + if (TARGET_64BIT) { #ifndef NO_PROFILE_COUNTERS - fprintf (file, "\tleaq\t" LPREFIX "P%d@(%%rip),%%r11\n", labelno); + fprintf (file, "\tleaq\t%sP%d(%%rip),%%r11\n", LPREFIX, labelno); #endif if (DEFAULT_ABI == SYSV_ABI && flag_pic) - fputs ("\tcall\t*" MCOUNT_NAME "@GOTPCREL(%rip)\n", file); - else - fputs ("\tcall\t" MCOUNT_NAME "\n", file); + fprintf (file, "\tcall\t*%s@GOTPCREL(%%rip)\n", mcount_name); + else + fprintf (file, "\tcall\t%s\n", mcount_name); } else if (flag_pic) { #ifndef NO_PROFILE_COUNTERS - fprintf (file, "\tleal\t" LPREFIX "P%d@GOTOFF(%%ebx),%%" PROFILE_COUNT_REGISTER "\n", - labelno); -#endif - fputs ("\tcall\t*" MCOUNT_NAME "@GOT(%ebx)\n", file); + fprintf (file, "\tleal\t%sP%d@GOTOFF(%%ebx),%%" PROFILE_COUNT_REGISTER "\n", + LPREFIX, labelno); +#endif + fprintf (file, "\tcall\t*%s@GOT(%%ebx)\n", mcount_name); } else { #ifndef NO_PROFILE_COUNTERS - fprintf (file, "\tmovl\t$" LPREFIX "P%d,%%" PROFILE_COUNT_REGISTER "\n", - labelno); -#endif - fputs ("\tcall\t" MCOUNT_NAME "\n", file); - } -} - -#ifdef ASM_OUTPUT_MAX_SKIP_PAD + fprintf (file, "\tmovl\t$%sP%d,%%" PROFILE_COUNT_REGISTER "\n", + LPREFIX, labelno); +#endif + fprintf (file, "\tcall\t%s\n", mcount_name); + } +} + /* We don't have exact information about the insn sizes, but we may assume quite safely that we are informed about all 1 byte insns and memory address sizes. This is enough to eliminate unnecessary padding in @@ -26365,6 +29879,8 @@ return 2; } +#ifdef ASM_OUTPUT_MAX_SKIP_PAD + /* AMD K8 core mispredicts jumps when there are more than 3 jumps in 16 byte window. */ @@ -26507,9 +30023,9 @@ && ((JUMP_P (prev) && any_condjump_p (prev)) || CALL_P (prev))) replace = true; - /* Empty functions get branch mispredict even when the jump destination - is not visible to us. */ - if (!prev && cfun->function_frequency > FUNCTION_FREQUENCY_UNLIKELY_EXECUTED) + /* Empty functions get branch mispredict even when + the jump destination is not visible to us. */ + if (!prev && !optimize_function_for_size_p (cfun)) replace = true; } if (replace) @@ -26520,20 +30036,144 @@ } } +/* Count the minimum number of instructions in BB. Return 4 if the + number of instructions >= 4. */ + +static int +ix86_count_insn_bb (basic_block bb) +{ + rtx insn; + int insn_count = 0; + + /* Count number of instructions in this block. Return 4 if the number + of instructions >= 4. */ + FOR_BB_INSNS (bb, insn) + { + /* Only happen in exit blocks. */ + if (JUMP_P (insn) + && GET_CODE (PATTERN (insn)) == RETURN) + break; + + if (NONDEBUG_INSN_P (insn) + && GET_CODE (PATTERN (insn)) != USE + && GET_CODE (PATTERN (insn)) != CLOBBER) + { + insn_count++; + if (insn_count >= 4) + return insn_count; + } + } + + return insn_count; +} + + +/* Count the minimum number of instructions in code path in BB. + Return 4 if the number of instructions >= 4. */ + +static int +ix86_count_insn (basic_block bb) +{ + edge e; + edge_iterator ei; + int min_prev_count; + + /* Only bother counting instructions along paths with no + more than 2 basic blocks between entry and exit. Given + that BB has an edge to exit, determine if a predecessor + of BB has an edge from entry. If so, compute the number + of instructions in the predecessor block. If there + happen to be multiple such blocks, compute the minimum. */ + min_prev_count = 4; + FOR_EACH_EDGE (e, ei, bb->preds) + { + edge prev_e; + edge_iterator prev_ei; + + if (e->src == ENTRY_BLOCK_PTR) + { + min_prev_count = 0; + break; + } + FOR_EACH_EDGE (prev_e, prev_ei, e->src->preds) + { + if (prev_e->src == ENTRY_BLOCK_PTR) + { + int count = ix86_count_insn_bb (e->src); + if (count < min_prev_count) + min_prev_count = count; + break; + } + } + } + + if (min_prev_count < 4) + min_prev_count += ix86_count_insn_bb (bb); + + return min_prev_count; +} + +/* Pad short funtion to 4 instructions. */ + +static void +ix86_pad_short_function (void) +{ + edge e; + edge_iterator ei; + + FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR->preds) + { + rtx ret = BB_END (e->src); + if (JUMP_P (ret) && GET_CODE (PATTERN (ret)) == RETURN) + { + int insn_count = ix86_count_insn (e->src); + + /* Pad short function. */ + if (insn_count < 4) + { + rtx insn = ret; + + /* Find epilogue. */ + while (insn + && (!NOTE_P (insn) + || NOTE_KIND (insn) != NOTE_INSN_EPILOGUE_BEG)) + insn = PREV_INSN (insn); + + if (!insn) + insn = ret; + + /* Two NOPs count as one instruction. */ + insn_count = 2 * (4 - insn_count); + emit_insn_before (gen_nops (GEN_INT (insn_count)), insn); + } + } + } +} + /* Implement machine specific optimizations. We implement padding of returns for K8 CPUs and pass to avoid 4 jumps in the single 16 byte window. */ static void ix86_reorg (void) { + /* We are freeing block_for_insn in the toplev to keep compatibility + with old MDEP_REORGS that are not CFG based. Recompute it now. */ + compute_bb_for_insn (); + if (optimize && optimize_function_for_speed_p (cfun)) { - if (TARGET_PAD_RETURNS) + if (TARGET_PAD_SHORT_FUNCTION) + ix86_pad_short_function (); + else if (TARGET_PAD_RETURNS) ix86_pad_returns (); #ifdef ASM_OUTPUT_MAX_SKIP_PAD if (TARGET_FOUR_JUMP_LIMIT) ix86_avoid_jump_mispredicts (); #endif } + + /* Run the vzeroupper optimization if needed. */ + if (TARGET_VZEROUPPER) + move_or_delete_vzeroupper (); } /* Return nonzero when QImode register that must be represented via REX prefix @@ -26571,6 +30211,52 @@ extended_reg_mentioned_1, NULL); } +/* If profitable, negate (without causing overflow) integer constant + of mode MODE at location LOC. Return true in this case. */ +bool +x86_maybe_negate_const_int (rtx *loc, enum machine_mode mode) +{ + HOST_WIDE_INT val; + + if (!CONST_INT_P (*loc)) + return false; + + switch (mode) + { + case DImode: + /* DImode x86_64 constants must fit in 32 bits. */ + gcc_assert (x86_64_immediate_operand (*loc, mode)); + + mode = SImode; + break; + + case SImode: + case HImode: + case QImode: + break; + + default: + gcc_unreachable (); + } + + /* Avoid overflows. */ + if (mode_signbit_p (mode, *loc)) + return false; + + val = INTVAL (*loc); + + /* Make things pretty and `subl $4,%eax' rather than `addl $-4,%eax'. + Exceptions: -128 encodes smaller than 128, so swap sign and op. */ + if ((val < 0 && val != -128) + || val == 128) + { + *loc = GEN_INT (-val); + return true; + } + + return false; +} + /* Generate an unsigned DImode/SImode to FP conversion. This is the same code optabs would emit if we didn't have TFmode patterns. */ @@ -26674,8 +30360,16 @@ insn = emit_insn (gen_rtx_SET (VOIDmode, target, dup)); if (recog_memoized (insn) < 0) { + rtx seq; /* If that fails, force VAL into a register. */ + + start_sequence (); XEXP (dup, 0) = force_reg (GET_MODE_INNER (mode), val); + seq = get_insns (); + end_sequence (); + if (seq) + emit_insn_before (seq, insn); + ok = recog_memoized (insn) >= 0; gcc_assert (ok); } @@ -26831,7 +30525,7 @@ emit_insn (gen_rtx_SET (VOIDmode, target, CONST0_RTX (mode))); var = force_reg (GET_MODE_INNER (mode), var); ix86_expand_vector_set (mmx_ok, target, var, one_var); - return true; + return true; } switch (mode) @@ -27165,7 +30859,7 @@ rtx (*gen_load_even) (rtx, rtx, rtx); rtx (*gen_interleave_first_low) (rtx, rtx, rtx); rtx (*gen_interleave_second_low) (rtx, rtx, rtx); - + switch (mode) { case V8HImode: @@ -27189,7 +30883,7 @@ default: gcc_unreachable (); } - + for (i = 0; i < n; i++) { /* Extend the odd elment to SImode using a paradoxical SUBREG. */ @@ -27208,12 +30902,12 @@ /* Cast the V4SImode vector back to a vector in orignal mode. */ op0 = gen_reg_rtx (mode); emit_move_insn (op0, gen_lowpart (mode, op1)); - + /* Load even elements into the second positon. */ - emit_insn ((*gen_load_even) (op0, - force_reg (inner_mode, - ops [i + i + 1]), - const1_rtx)); + emit_insn (gen_load_even (op0, + force_reg (inner_mode, + ops [i + i + 1]), + const1_rtx)); /* Cast vector to FIRST_IMODE vector. */ ops[i] = gen_reg_rtx (first_imode); @@ -27224,7 +30918,7 @@ for (i = j = 0; i < n; i += 2, j++) { op0 = gen_reg_rtx (first_imode); - emit_insn ((*gen_interleave_first_low) (op0, ops[i], ops[i + 1])); + emit_insn (gen_interleave_first_low (op0, ops[i], ops[i + 1])); /* Cast FIRST_IMODE vector to SECOND_IMODE vector. */ ops[j] = gen_reg_rtx (second_imode); @@ -27238,8 +30932,8 @@ for (i = j = 0; i < n / 2; i += 2, j++) { op0 = gen_reg_rtx (second_imode); - emit_insn ((*gen_interleave_second_low) (op0, ops[i], - ops[i + 1])); + emit_insn (gen_interleave_second_low (op0, ops[i], + ops[i + 1])); /* Cast the SECOND_IMODE vector to the THIRD_IMODE vector. */ @@ -27252,8 +30946,8 @@ case V2DImode: op0 = gen_reg_rtx (second_imode); - emit_insn ((*gen_interleave_second_low) (op0, ops[0], - ops[1])); + emit_insn (gen_interleave_second_low (op0, ops[0], + ops[1])); /* Cast the SECOND_IMODE vector back to a vector on original mode. */ @@ -27331,7 +31025,7 @@ break; /* Don't use ix86_expand_vector_init_interleave if we can't - move from GPR to SSE register directly. */ + move from GPR to SSE register directly. */ if (!TARGET_INTER_UNIT_MOVES) break; @@ -27684,13 +31378,13 @@ /* Extract the half. */ tmp = gen_reg_rtx (half_mode); - emit_insn ((*gen_extract[j][i]) (tmp, target)); + emit_insn (gen_extract[j][i] (tmp, target)); /* Put val in tmp at elt. */ ix86_expand_vector_set (false, tmp, val, elt); /* Put it back. */ - emit_insn ((*gen_insert[j][i]) (target, target, tmp)); + emit_insn (gen_insert[j][i] (target, target, tmp)); return; default: @@ -28075,7 +31769,7 @@ two = CONST_DOUBLE_FROM_REAL_VALUE (dconst2, SFmode); if (VECTOR_MODE_P (mode)) - two = ix86_build_const_vector (SFmode, true, two); + two = ix86_build_const_vector (mode, true, two); two = force_reg (mode, two); @@ -28122,8 +31816,8 @@ if (VECTOR_MODE_P (mode)) { - mthree = ix86_build_const_vector (SFmode, true, mthree); - mhalf = ix86_build_const_vector (SFmode, true, mhalf); + mthree = ix86_build_const_vector (mode, true, mthree); + mhalf = ix86_build_const_vector (mode, true, mhalf); } /* sqrt(a) = -0.5 * a * rsqrtss(a) * (a * rsqrtss(a) * rsqrtss(a) - 3.0) @@ -28268,7 +31962,16 @@ rtx sgn = gen_reg_rtx (mode); if (mask == NULL_RTX) { - mask = ix86_build_signbit_mask (mode, VECTOR_MODE_P (mode), false); + enum machine_mode vmode; + + if (mode == SFmode) + vmode = V4SFmode; + else if (mode == DFmode) + vmode = V2DFmode; + else + vmode = mode; + + mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), false); if (!VECTOR_MODE_P (mode)) { /* We need to generate a scalar mode mask in this case. */ @@ -28292,11 +31995,17 @@ static rtx ix86_expand_sse_fabs (rtx op0, rtx *smask) { - enum machine_mode mode = GET_MODE (op0); + enum machine_mode vmode, mode = GET_MODE (op0); rtx xa, mask; xa = gen_reg_rtx (mode); - mask = ix86_build_signbit_mask (mode, VECTOR_MODE_P (mode), true); + if (mode == SFmode) + vmode = V4SFmode; + else if (mode == DFmode) + vmode = V2DFmode; + else + vmode = mode; + mask = ix86_build_signbit_mask (vmode, VECTOR_MODE_P (mode), true); if (!VECTOR_MODE_P (mode)) { /* We need to generate a scalar mode mask in this case. */ @@ -28845,6 +32554,9 @@ /* Fastcall attribute says callee is responsible for popping arguments if they are not variable. */ { "fastcall", 0, 0, false, true, true, ix86_handle_cconv_attribute }, + /* Thiscall attribute says callee is responsible for popping arguments + if they are not variable. */ + { "thiscall", 0, 0, false, true, true, ix86_handle_cconv_attribute }, /* Cdecl attribute says the callee is a normal C declaration */ { "cdecl", 0, 0, false, true, true, ix86_handle_cconv_attribute }, /* Regparm attribute specifies how many integer arguments are to be @@ -28870,33 +32582,62 @@ { "ms_abi", 0, 0, false, true, true, ix86_handle_abi_attribute }, { "sysv_abi", 0, 0, false, true, true, ix86_handle_abi_attribute }, { "ms_hook_prologue", 0, 0, true, false, false, ix86_handle_fndecl_attribute }, + { "callee_pop_aggregate_return", 1, 1, false, true, true, + ix86_handle_callee_pop_aggregate_return }, /* End element. */ { NULL, 0, 0, false, false, false, NULL } }; /* Implement targetm.vectorize.builtin_vectorization_cost. */ static int -ix86_builtin_vectorization_cost (bool runtime_test) -{ - /* If the branch of the runtime test is taken - i.e. - the vectorized - version is skipped - this incurs a misprediction cost (because the - vectorized version is expected to be the fall-through). So we subtract - the latency of a mispredicted branch from the costs that are incured - when the vectorized version is executed. - - TODO: The values in individual target tables have to be tuned or new - fields may be needed. For eg. on K8, the default branch path is the - not-taken path. If the taken path is predicted correctly, the minimum - penalty of going down the taken-path is 1 cycle. If the taken-path is - not predicted correctly, then the minimum penalty is 10 cycles. */ - - if (runtime_test) - { - return (-(ix86_cost->cond_taken_branch_cost)); - } - else - return 0; -} +ix86_builtin_vectorization_cost (enum vect_cost_for_stmt type_of_cost, + tree vectype ATTRIBUTE_UNUSED, + int misalign ATTRIBUTE_UNUSED) +{ + switch (type_of_cost) + { + case scalar_stmt: + return ix86_cost->scalar_stmt_cost; + + case scalar_load: + return ix86_cost->scalar_load_cost; + + case scalar_store: + return ix86_cost->scalar_store_cost; + + case vector_stmt: + return ix86_cost->vec_stmt_cost; + + case vector_load: + return ix86_cost->vec_align_load_cost; + + case vector_store: + return ix86_cost->vec_store_cost; + + case vec_to_scalar: + return ix86_cost->vec_to_scalar_cost; + + case scalar_to_vec: + return ix86_cost->scalar_to_vec_cost; + + case unaligned_load: + case unaligned_store: + return ix86_cost->vec_unalign_load_cost; + + case cond_branch_taken: + return ix86_cost->cond_taken_branch_cost; + + case cond_branch_not_taken: + return ix86_cost->cond_not_taken_branch_cost; + + case vec_perm: + return 1; + + default: + gcc_unreachable (); + } +} + /* Implement targetm.vectorize.builtin_vec_perm. */ @@ -29115,8 +32856,8 @@ do_subreg: vmode = V8HImode; target = gen_lowpart (vmode, target); - op0 = gen_lowpart (vmode, target); - op1 = gen_lowpart (vmode, target); + op0 = gen_lowpart (vmode, op0); + op1 = gen_lowpart (vmode, op1); break; default: @@ -29124,7 +32865,7 @@ } /* This matches five different patterns with the different modes. */ - x = gen_rtx_VEC_MERGE (vmode, op0, op1, GEN_INT (mask)); + x = gen_rtx_VEC_MERGE (vmode, op1, op0, GEN_INT (mask)); x = gen_rtx_SET (VOIDmode, target, x); emit_insn (x); @@ -29236,7 +32977,12 @@ input where SEL+CONCAT may not. */ if (d->op0 == d->op1) { - if (expand_vselect (d->target, d->op0, d->perm, nelt)) + int mask = nelt - 1; + + for (i = 0; i < nelt; i++) + perm2[i] = d->perm[i] & mask; + + if (expand_vselect (d->target, d->op0, perm2, nelt)) return true; /* There are plenty of patterns in sse.md that are written for @@ -29247,8 +32993,8 @@ every other permutation operand. */ for (i = 0; i < nelt; i += 2) { - perm2[i] = d->perm[i]; - perm2[i+1] = d->perm[i+1] + nelt; + perm2[i] = d->perm[i] & mask; + perm2[i + 1] = (d->perm[i + 1] & mask) + nelt; } if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt)) return true; @@ -29256,11 +33002,12 @@ /* Recognize shufps, which means adding {0, 0, nelt, nelt}. */ if (nelt >= 4) { - memcpy (perm2, d->perm, nelt); - for (i = 2; i < nelt; i += 4) - { - perm2[i+0] += nelt; - perm2[i+1] += nelt; + for (i = 0; i < nelt; i += 4) + { + perm2[i + 0] = d->perm[i + 0] & mask; + perm2[i + 1] = d->perm[i + 1] & mask; + perm2[i + 2] = (d->perm[i + 2] & mask) + nelt; + perm2[i + 3] = (d->perm[i + 3] & mask) + nelt; } if (expand_vselect_vconcat (d->target, d->op0, d->op0, perm2, nelt)) @@ -29557,7 +33304,7 @@ nelt = d->nelt; eltsz = GET_MODE_SIZE (GET_MODE_INNER (d->vmode)); - + /* Generate two permutation masks. If the required element is within the given vector it is shuffled into the proper lane. If the required element is in the other vector, force a zero into the lane by setting @@ -29603,7 +33350,7 @@ static bool expand_vec_perm_even_odd_1 (struct expand_vec_perm_d *d, unsigned odd) { - rtx t1, t2, t3, t4; + rtx t1, t2, t3; switch (d->vmode) { @@ -29625,34 +33372,34 @@ case V8SFmode: { - static const unsigned char perm1[8] = { 0, 2, 1, 3, 5, 6, 5, 7 }; - static const unsigned char perme[8] = { 0, 1, 8, 9, 4, 5, 12, 13 }; - static const unsigned char permo[8] = { 2, 3, 10, 11, 6, 7, 14, 15 }; + int mask = odd ? 0xdd : 0x88; t1 = gen_reg_rtx (V8SFmode); t2 = gen_reg_rtx (V8SFmode); t3 = gen_reg_rtx (V8SFmode); - t4 = gen_reg_rtx (V8SFmode); /* Shuffle within the 128-bit lanes to produce: - { 0 2 1 3 4 6 5 7 } and { 8 a 9 b c e d f }. */ - expand_vselect (t1, d->op0, perm1, 8); - expand_vselect (t2, d->op1, perm1, 8); + { 0 2 8 a 4 6 c e } | { 1 3 9 b 5 7 d f }. */ + emit_insn (gen_avx_shufps256 (t1, d->op0, d->op1, + GEN_INT (mask))); /* Shuffle the lanes around to produce: - { 0 2 1 3 8 a 9 b } and { 4 6 5 7 c e d f }. */ - emit_insn (gen_avx_vperm2f128v8sf3 (t3, t1, t2, GEN_INT (0x20))); - emit_insn (gen_avx_vperm2f128v8sf3 (t4, t1, t2, GEN_INT (0x31))); - - /* Now a vpermil2p will produce the result required. */ - /* ??? The vpermil2p requires a vector constant. Another option - is a unpck[lh]ps to merge the two vectors to produce - { 0 4 2 6 8 c a e } or { 1 5 3 7 9 d b f }. Then use another - vpermilps to get the elements into the final order. */ - d->op0 = t3; - d->op1 = t4; - memcpy (d->perm, odd ? permo: perme, 8); - expand_vec_perm_vpermil (d); + { 4 6 c e 0 2 8 a } and { 5 7 d f 1 3 9 b }. */ + emit_insn (gen_avx_vperm2f128v8sf3 (t2, t1, t1, + GEN_INT (0x3))); + + /* Shuffle within the 128-bit lanes to produce: + { 0 2 4 6 4 6 0 2 } | { 1 3 5 7 5 7 1 3 }. */ + emit_insn (gen_avx_shufps256 (t3, t1, t2, GEN_INT (0x44))); + + /* Shuffle within the 128-bit lanes to produce: + { 8 a c e c e 8 a } | { 9 b d f d f 9 b }. */ + emit_insn (gen_avx_shufps256 (t2, t1, t2, GEN_INT (0xee))); + + /* Shuffle the lanes around to produce: + { 0 2 4 6 8 a c e } | { 1 3 5 7 9 b d f }. */ + emit_insn (gen_avx_vperm2f128v8sf3 (d->target, t3, t2, + GEN_INT (0x20))); } break; @@ -29955,7 +33702,7 @@ d.op1 = d.op0; break; } - + d.target = gen_reg_rtx (d.vmode); if (ix86_expand_vec_perm_builtin_1 (&d)) return d.target; @@ -30027,7 +33774,7 @@ an error generated from the extract. */ gcc_assert (vec_mask > 0 && vec_mask <= 3); one_vec = (vec_mask != 3); - + /* Implementable with shufps or pshufd. */ if (one_vec && (d.vmode == V4SFmode || d.vmode == V4SImode)) return true; @@ -30073,7 +33820,7 @@ /* This function returns the calling abi specific va_list type node. It returns the FNDECL specific va_list type. */ -tree +static tree ix86_fn_abi_va_list (tree fndecl) { if (!TARGET_64BIT) @@ -30089,18 +33836,20 @@ /* Returns the canonical va_list type specified by TYPE. If there is no valid TYPE provided, it return NULL_TREE. */ -tree +static tree ix86_canonical_va_list_type (tree type) { tree wtype, htype; /* Resolve references and pointers to va_list type. */ - if (INDIRECT_REF_P (type)) + if (TREE_CODE (type) == MEM_REF) type = TREE_TYPE (type); else if (POINTER_TYPE_P (type) && POINTER_TYPE_P (TREE_TYPE(type))) type = TREE_TYPE (type); - - if (TARGET_64BIT) + else if (POINTER_TYPE_P (type) && TREE_CODE (TREE_TYPE (type)) == ARRAY_TYPE) + type = TREE_TYPE (type); + + if (TARGET_64BIT && va_list_type_node != NULL_TREE) { wtype = va_list_type_node; gcc_assert (wtype != NULL_TREE); @@ -30162,33 +33911,881 @@ } /* Iterate through the target-specific builtin types for va_list. - IDX denotes the iterator, *PTREE is set to the result type of - the va_list builtin, and *PNAME to its internal type. - Returns zero if there is no element for this index, otherwise - IDX should be increased upon the next call. - Note, do not iterate a base builtin's name like __builtin_va_list. - Used from c_common_nodes_and_builtins. */ - -int + IDX denotes the iterator, *PTREE is set to the result type of + the va_list builtin, and *PNAME to its internal type. + Returns zero if there is no element for this index, otherwise + IDX should be increased upon the next call. + Note, do not iterate a base builtin's name like __builtin_va_list. + Used from c_common_nodes_and_builtins. */ + +static int ix86_enum_va_list (int idx, const char **pname, tree *ptree) { - if (!TARGET_64BIT) - return 0; - switch (idx) { - case 0: - *ptree = ms_va_list_type_node; - *pname = "__builtin_ms_va_list"; - break; - case 1: - *ptree = sysv_va_list_type_node; - *pname = "__builtin_sysv_va_list"; - break; - default: - return 0; - } + if (TARGET_64BIT) + { + switch (idx) + { + default: + break; + + case 0: + *ptree = ms_va_list_type_node; + *pname = "__builtin_ms_va_list"; + return 1; + + case 1: + *ptree = sysv_va_list_type_node; + *pname = "__builtin_sysv_va_list"; + return 1; + } + } + + return 0; +} + +#undef TARGET_SCHED_DISPATCH +#define TARGET_SCHED_DISPATCH has_dispatch +#undef TARGET_SCHED_DISPATCH_DO +#define TARGET_SCHED_DISPATCH_DO do_dispatch + +/* The size of the dispatch window is the total number of bytes of + object code allowed in a window. */ +#define DISPATCH_WINDOW_SIZE 16 + +/* Number of dispatch windows considered for scheduling. */ +#define MAX_DISPATCH_WINDOWS 3 + +/* Maximum number of instructions in a window. */ +#define MAX_INSN 4 + +/* Maximum number of immediate operands in a window. */ +#define MAX_IMM 4 + +/* Maximum number of immediate bits allowed in a window. */ +#define MAX_IMM_SIZE 128 + +/* Maximum number of 32 bit immediates allowed in a window. */ +#define MAX_IMM_32 4 + +/* Maximum number of 64 bit immediates allowed in a window. */ +#define MAX_IMM_64 2 + +/* Maximum total of loads or prefetches allowed in a window. */ +#define MAX_LOAD 2 + +/* Maximum total of stores allowed in a window. */ +#define MAX_STORE 1 + +#undef BIG +#define BIG 100 + + +/* Dispatch groups. Istructions that affect the mix in a dispatch window. */ +enum dispatch_group { + disp_no_group = 0, + disp_load, + disp_store, + disp_load_store, + disp_prefetch, + disp_imm, + disp_imm_32, + disp_imm_64, + disp_branch, + disp_cmp, + disp_jcc, + disp_last +}; + +/* Number of allowable groups in a dispatch window. It is an array + indexed by dispatch_group enum. 100 is used as a big number, + because the number of these kind of operations does not have any + effect in dispatch window, but we need them for other reasons in + the table. */ +static unsigned int num_allowable_groups[disp_last] = { + 0, 2, 1, 1, 2, 4, 4, 2, 1, BIG, BIG +}; + +char group_name[disp_last + 1][16] = { + "disp_no_group", "disp_load", "disp_store", "disp_load_store", + "disp_prefetch", "disp_imm", "disp_imm_32", "disp_imm_64", + "disp_branch", "disp_cmp", "disp_jcc", "disp_last" +}; + +/* Instruction path. */ +enum insn_path { + no_path = 0, + path_single, /* Single micro op. */ + path_double, /* Double micro op. */ + path_multi, /* Instructions with more than 2 micro op.. */ + last_path +}; + +/* sched_insn_info defines a window to the instructions scheduled in + the basic block. It contains a pointer to the insn_info table and + the instruction scheduled. + + Windows are allocated for each basic block and are linked + together. */ +typedef struct sched_insn_info_s { + rtx insn; + enum dispatch_group group; + enum insn_path path; + int byte_len; + int imm_bytes; +} sched_insn_info; + +/* Linked list of dispatch windows. This is a two way list of + dispatch windows of a basic block. It contains information about + the number of uops in the window and the total number of + instructions and of bytes in the object code for this dispatch + window. */ +typedef struct dispatch_windows_s { + int num_insn; /* Number of insn in the window. */ + int num_uops; /* Number of uops in the window. */ + int window_size; /* Number of bytes in the window. */ + int window_num; /* Window number between 0 or 1. */ + int num_imm; /* Number of immediates in an insn. */ + int num_imm_32; /* Number of 32 bit immediates in an insn. */ + int num_imm_64; /* Number of 64 bit immediates in an insn. */ + int imm_size; /* Total immediates in the window. */ + int num_loads; /* Total memory loads in the window. */ + int num_stores; /* Total memory stores in the window. */ + int violation; /* Violation exists in window. */ + sched_insn_info *window; /* Pointer to the window. */ + struct dispatch_windows_s *next; + struct dispatch_windows_s *prev; +} dispatch_windows; + +/* Immediate valuse used in an insn. */ +typedef struct imm_info_s + { + int imm; + int imm32; + int imm64; + } imm_info; + +static dispatch_windows *dispatch_window_list; +static dispatch_windows *dispatch_window_list1; + +/* Get dispatch group of insn. */ + +static enum dispatch_group +get_mem_group (rtx insn) +{ + enum attr_memory memory; + + if (INSN_CODE (insn) < 0) + return disp_no_group; + memory = get_attr_memory (insn); + if (memory == MEMORY_STORE) + return disp_store; + + if (memory == MEMORY_LOAD) + return disp_load; + + if (memory == MEMORY_BOTH) + return disp_load_store; + + return disp_no_group; +} + +/* Return true if insn is a compare instruction. */ + +static bool +is_cmp (rtx insn) +{ + enum attr_type type; + + type = get_attr_type (insn); + return (type == TYPE_TEST + || type == TYPE_ICMP + || type == TYPE_FCMP + || GET_CODE (PATTERN (insn)) == COMPARE); +} + +/* Return true if a dispatch violation encountered. */ + +static bool +dispatch_violation (void) +{ + if (dispatch_window_list->next) + return dispatch_window_list->next->violation; + return dispatch_window_list->violation; +} + +/* Return true if insn is a branch instruction. */ + +static bool +is_branch (rtx insn) +{ + return (CALL_P (insn) || JUMP_P (insn)); +} + +/* Return true if insn is a prefetch instruction. */ + +static bool +is_prefetch (rtx insn) +{ + return NONJUMP_INSN_P (insn) && GET_CODE (PATTERN (insn)) == PREFETCH; +} + +/* This function initializes a dispatch window and the list container holding a + pointer to the window. */ + +static void +init_window (int window_num) +{ + int i; + dispatch_windows *new_list; + + if (window_num == 0) + new_list = dispatch_window_list; + else + new_list = dispatch_window_list1; + + new_list->num_insn = 0; + new_list->num_uops = 0; + new_list->window_size = 0; + new_list->next = NULL; + new_list->prev = NULL; + new_list->window_num = window_num; + new_list->num_imm = 0; + new_list->num_imm_32 = 0; + new_list->num_imm_64 = 0; + new_list->imm_size = 0; + new_list->num_loads = 0; + new_list->num_stores = 0; + new_list->violation = false; + + for (i = 0; i < MAX_INSN; i++) + { + new_list->window[i].insn = NULL; + new_list->window[i].group = disp_no_group; + new_list->window[i].path = no_path; + new_list->window[i].byte_len = 0; + new_list->window[i].imm_bytes = 0; + } + return; +} + +/* This function allocates and initializes a dispatch window and the + list container holding a pointer to the window. */ + +static dispatch_windows * +allocate_window (void) +{ + dispatch_windows *new_list = XNEW (struct dispatch_windows_s); + new_list->window = XNEWVEC (struct sched_insn_info_s, MAX_INSN + 1); + + return new_list; +} + +/* This routine initializes the dispatch scheduling information. It + initiates building dispatch scheduler tables and constructs the + first dispatch window. */ + +static void +init_dispatch_sched (void) +{ + /* Allocate a dispatch list and a window. */ + dispatch_window_list = allocate_window (); + dispatch_window_list1 = allocate_window (); + init_window (0); + init_window (1); +} + +/* This function returns true if a branch is detected. End of a basic block + does not have to be a branch, but here we assume only branches end a + window. */ + +static bool +is_end_basic_block (enum dispatch_group group) +{ + return group == disp_branch; +} + +/* This function is called when the end of a window processing is reached. */ + +static void +process_end_window (void) +{ + gcc_assert (dispatch_window_list->num_insn <= MAX_INSN); + if (dispatch_window_list->next) + { + gcc_assert (dispatch_window_list1->num_insn <= MAX_INSN); + gcc_assert (dispatch_window_list->window_size + + dispatch_window_list1->window_size <= 48); + init_window (1); + } + init_window (0); +} + +/* Allocates a new dispatch window and adds it to WINDOW_LIST. + WINDOW_NUM is either 0 or 1. A maximum of two windows are generated + for 48 bytes of instructions. Note that these windows are not dispatch + windows that their sizes are DISPATCH_WINDOW_SIZE. */ + +static dispatch_windows * +allocate_next_window (int window_num) +{ + if (window_num == 0) + { + if (dispatch_window_list->next) + init_window (1); + init_window (0); + return dispatch_window_list; + } + + dispatch_window_list->next = dispatch_window_list1; + dispatch_window_list1->prev = dispatch_window_list; + + return dispatch_window_list1; +} + +/* Increment the number of immediate operands of an instruction. */ + +static int +find_constant_1 (rtx *in_rtx, imm_info *imm_values) +{ + if (*in_rtx == 0) + return 0; + + switch ( GET_CODE (*in_rtx)) + { + case CONST: + case SYMBOL_REF: + case CONST_INT: + (imm_values->imm)++; + if (x86_64_immediate_operand (*in_rtx, SImode)) + (imm_values->imm32)++; + else + (imm_values->imm64)++; + break; + + case CONST_DOUBLE: + (imm_values->imm)++; + (imm_values->imm64)++; + break; + + case CODE_LABEL: + if (LABEL_KIND (*in_rtx) == LABEL_NORMAL) + { + (imm_values->imm)++; + (imm_values->imm32)++; + } + break; + + default: + break; + } + + return 0; +} + +/* Compute number of immediate operands of an instruction. */ + +static void +find_constant (rtx in_rtx, imm_info *imm_values) +{ + for_each_rtx (INSN_P (in_rtx) ? &PATTERN (in_rtx) : &in_rtx, + (rtx_function) find_constant_1, (void *) imm_values); +} + +/* Return total size of immediate operands of an instruction along with number + of corresponding immediate-operands. It initializes its parameters to zero + befor calling FIND_CONSTANT. + INSN is the input instruction. IMM is the total of immediates. + IMM32 is the number of 32 bit immediates. IMM64 is the number of 64 + bit immediates. */ + +static int +get_num_immediates (rtx insn, int *imm, int *imm32, int *imm64) +{ + imm_info imm_values = {0, 0, 0}; + + find_constant (insn, &imm_values); + *imm = imm_values.imm; + *imm32 = imm_values.imm32; + *imm64 = imm_values.imm64; + return imm_values.imm32 * 4 + imm_values.imm64 * 8; +} + +/* This function indicates if an operand of an instruction is an + immediate. */ + +static bool +has_immediate (rtx insn) +{ + int num_imm_operand; + int num_imm32_operand; + int num_imm64_operand; + + if (insn) + return get_num_immediates (insn, &num_imm_operand, &num_imm32_operand, + &num_imm64_operand); + return false; +} + +/* Return single or double path for instructions. */ + +static enum insn_path +get_insn_path (rtx insn) +{ + enum attr_amdfam10_decode path = get_attr_amdfam10_decode (insn); + + if ((int)path == 0) + return path_single; + + if ((int)path == 1) + return path_double; + + return path_multi; +} + +/* Return insn dispatch group. */ + +static enum dispatch_group +get_insn_group (rtx insn) +{ + enum dispatch_group group = get_mem_group (insn); + if (group) + return group; + + if (is_branch (insn)) + return disp_branch; + + if (is_cmp (insn)) + return disp_cmp; + + if (has_immediate (insn)) + return disp_imm; + + if (is_prefetch (insn)) + return disp_prefetch; + + return disp_no_group; +} + +/* Count number of GROUP restricted instructions in a dispatch + window WINDOW_LIST. */ + +static int +count_num_restricted (rtx insn, dispatch_windows *window_list) +{ + enum dispatch_group group = get_insn_group (insn); + int imm_size; + int num_imm_operand; + int num_imm32_operand; + int num_imm64_operand; + + if (group == disp_no_group) + return 0; + + if (group == disp_imm) + { + imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand, + &num_imm64_operand); + if (window_list->imm_size + imm_size > MAX_IMM_SIZE + || num_imm_operand + window_list->num_imm > MAX_IMM + || (num_imm32_operand > 0 + && (window_list->num_imm_32 + num_imm32_operand > MAX_IMM_32 + || window_list->num_imm_64 * 2 + num_imm32_operand > MAX_IMM_32)) + || (num_imm64_operand > 0 + && (window_list->num_imm_64 + num_imm64_operand > MAX_IMM_64 + || window_list->num_imm_32 + num_imm64_operand * 2 > MAX_IMM_32)) + || (window_list->imm_size + imm_size == MAX_IMM_SIZE + && num_imm64_operand > 0 + && ((window_list->num_imm_64 > 0 + && window_list->num_insn >= 2) + || window_list->num_insn >= 3))) + return BIG; + + return 1; + } + + if ((group == disp_load_store + && (window_list->num_loads >= MAX_LOAD + || window_list->num_stores >= MAX_STORE)) + || ((group == disp_load + || group == disp_prefetch) + && window_list->num_loads >= MAX_LOAD) + || (group == disp_store + && window_list->num_stores >= MAX_STORE)) + return BIG; + return 1; } +/* This function returns true if insn satisfies dispatch rules on the + last window scheduled. */ + +static bool +fits_dispatch_window (rtx insn) +{ + dispatch_windows *window_list = dispatch_window_list; + dispatch_windows *window_list_next = dispatch_window_list->next; + unsigned int num_restrict; + enum dispatch_group group = get_insn_group (insn); + enum insn_path path = get_insn_path (insn); + int sum; + + /* Make disp_cmp and disp_jcc get scheduled at the latest. These + instructions should be given the lowest priority in the + scheduling process in Haifa scheduler to make sure they will be + scheduled in the same dispatch window as the refrence to them. */ + if (group == disp_jcc || group == disp_cmp) + return false; + + /* Check nonrestricted. */ + if (group == disp_no_group || group == disp_branch) + return true; + + /* Get last dispatch window. */ + if (window_list_next) + window_list = window_list_next; + + if (window_list->window_num == 1) + { + sum = window_list->prev->window_size + window_list->window_size; + + if (sum == 32 + || (min_insn_size (insn) + sum) >= 48) + /* Window 1 is full. Go for next window. */ + return true; + } + + num_restrict = count_num_restricted (insn, window_list); + + if (num_restrict > num_allowable_groups[group]) + return false; + + /* See if it fits in the first window. */ + if (window_list->window_num == 0) + { + /* The first widow should have only single and double path + uops. */ + if (path == path_double + && (window_list->num_uops + 2) > MAX_INSN) + return false; + else if (path != path_single) + return false; + } + return true; +} + +/* Add an instruction INSN with NUM_UOPS micro-operations to the + dispatch window WINDOW_LIST. */ + +static void +add_insn_window (rtx insn, dispatch_windows *window_list, int num_uops) +{ + int byte_len = min_insn_size (insn); + int num_insn = window_list->num_insn; + int imm_size; + sched_insn_info *window = window_list->window; + enum dispatch_group group = get_insn_group (insn); + enum insn_path path = get_insn_path (insn); + int num_imm_operand; + int num_imm32_operand; + int num_imm64_operand; + + if (!window_list->violation && group != disp_cmp + && !fits_dispatch_window (insn)) + window_list->violation = true; + + imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand, + &num_imm64_operand); + + /* Initialize window with new instruction. */ + window[num_insn].insn = insn; + window[num_insn].byte_len = byte_len; + window[num_insn].group = group; + window[num_insn].path = path; + window[num_insn].imm_bytes = imm_size; + + window_list->window_size += byte_len; + window_list->num_insn = num_insn + 1; + window_list->num_uops = window_list->num_uops + num_uops; + window_list->imm_size += imm_size; + window_list->num_imm += num_imm_operand; + window_list->num_imm_32 += num_imm32_operand; + window_list->num_imm_64 += num_imm64_operand; + + if (group == disp_store) + window_list->num_stores += 1; + else if (group == disp_load + || group == disp_prefetch) + window_list->num_loads += 1; + else if (group == disp_load_store) + { + window_list->num_stores += 1; + window_list->num_loads += 1; + } +} + +/* Adds a scheduled instruction, INSN, to the current dispatch window. + If the total bytes of instructions or the number of instructions in + the window exceed allowable, it allocates a new window. */ + +static void +add_to_dispatch_window (rtx insn) +{ + int byte_len; + dispatch_windows *window_list; + dispatch_windows *next_list; + dispatch_windows *window0_list; + enum insn_path path; + enum dispatch_group insn_group; + bool insn_fits; + int num_insn; + int num_uops; + int window_num; + int insn_num_uops; + int sum; + + if (INSN_CODE (insn) < 0) + return; + + byte_len = min_insn_size (insn); + window_list = dispatch_window_list; + next_list = window_list->next; + path = get_insn_path (insn); + insn_group = get_insn_group (insn); + + /* Get the last dispatch window. */ + if (next_list) + window_list = dispatch_window_list->next; + + if (path == path_single) + insn_num_uops = 1; + else if (path == path_double) + insn_num_uops = 2; + else + insn_num_uops = (int) path; + + /* If current window is full, get a new window. + Window number zero is full, if MAX_INSN uops are scheduled in it. + Window number one is full, if window zero's bytes plus window + one's bytes is 32, or if the bytes of the new instruction added + to the total makes it greater than 48, or it has already MAX_INSN + instructions in it. */ + num_insn = window_list->num_insn; + num_uops = window_list->num_uops; + window_num = window_list->window_num; + insn_fits = fits_dispatch_window (insn); + + if (num_insn >= MAX_INSN + || num_uops + insn_num_uops > MAX_INSN + || !(insn_fits)) + { + window_num = ~window_num & 1; + window_list = allocate_next_window (window_num); + } + + if (window_num == 0) + { + add_insn_window (insn, window_list, insn_num_uops); + if (window_list->num_insn >= MAX_INSN + && insn_group == disp_branch) + { + process_end_window (); + return; + } + } + else if (window_num == 1) + { + window0_list = window_list->prev; + sum = window0_list->window_size + window_list->window_size; + if (sum == 32 + || (byte_len + sum) >= 48) + { + process_end_window (); + window_list = dispatch_window_list; + } + + add_insn_window (insn, window_list, insn_num_uops); + } + else + gcc_unreachable (); + + if (is_end_basic_block (insn_group)) + { + /* End of basic block is reached do end-basic-block process. */ + process_end_window (); + return; + } +} + +/* Print the dispatch window, WINDOW_NUM, to FILE. */ + +DEBUG_FUNCTION static void +debug_dispatch_window_file (FILE *file, int window_num) +{ + dispatch_windows *list; + int i; + + if (window_num == 0) + list = dispatch_window_list; + else + list = dispatch_window_list1; + + fprintf (file, "Window #%d:\n", list->window_num); + fprintf (file, " num_insn = %d, num_uops = %d, window_size = %d\n", + list->num_insn, list->num_uops, list->window_size); + fprintf (file, " num_imm = %d, num_imm_32 = %d, num_imm_64 = %d, imm_size = %d\n", + list->num_imm, list->num_imm_32, list->num_imm_64, list->imm_size); + + fprintf (file, " num_loads = %d, num_stores = %d\n", list->num_loads, + list->num_stores); + fprintf (file, " insn info:\n"); + + for (i = 0; i < MAX_INSN; i++) + { + if (!list->window[i].insn) + break; + fprintf (file, " group[%d] = %s, insn[%d] = %p, path[%d] = %d byte_len[%d] = %d, imm_bytes[%d] = %d\n", + i, group_name[list->window[i].group], + i, (void *)list->window[i].insn, + i, list->window[i].path, + i, list->window[i].byte_len, + i, list->window[i].imm_bytes); + } +} + +/* Print to stdout a dispatch window. */ + +DEBUG_FUNCTION void +debug_dispatch_window (int window_num) +{ + debug_dispatch_window_file (stdout, window_num); +} + +/* Print INSN dispatch information to FILE. */ + +DEBUG_FUNCTION static void +debug_insn_dispatch_info_file (FILE *file, rtx insn) +{ + int byte_len; + enum insn_path path; + enum dispatch_group group; + int imm_size; + int num_imm_operand; + int num_imm32_operand; + int num_imm64_operand; + + if (INSN_CODE (insn) < 0) + return; + + byte_len = min_insn_size (insn); + path = get_insn_path (insn); + group = get_insn_group (insn); + imm_size = get_num_immediates (insn, &num_imm_operand, &num_imm32_operand, + &num_imm64_operand); + + fprintf (file, " insn info:\n"); + fprintf (file, " group = %s, path = %d, byte_len = %d\n", + group_name[group], path, byte_len); + fprintf (file, " num_imm = %d, num_imm_32 = %d, num_imm_64 = %d, imm_size = %d\n", + num_imm_operand, num_imm32_operand, num_imm64_operand, imm_size); +} + +/* Print to STDERR the status of the ready list with respect to + dispatch windows. */ + +DEBUG_FUNCTION void +debug_ready_dispatch (void) +{ + int i; + int no_ready = number_in_ready (); + + fprintf (stdout, "Number of ready: %d\n", no_ready); + + for (i = 0; i < no_ready; i++) + debug_insn_dispatch_info_file (stdout, get_ready_element (i)); +} + +/* This routine is the driver of the dispatch scheduler. */ + +static void +do_dispatch (rtx insn, int mode) +{ + if (mode == DISPATCH_INIT) + init_dispatch_sched (); + else if (mode == ADD_TO_DISPATCH_WINDOW) + add_to_dispatch_window (insn); +} + +/* Return TRUE if Dispatch Scheduling is supported. */ + +static bool +has_dispatch (rtx insn, int action) +{ + if (ix86_tune == PROCESSOR_BDVER1 && flag_dispatch_scheduler) + switch (action) + { + default: + return false; + + case IS_DISPATCH_ON: + return true; + break; + + case IS_CMP: + return is_cmp (insn); + + case DISPATCH_VIOLATION: + return dispatch_violation (); + + case FITS_DISPATCH_WINDOW: + return fits_dispatch_window (insn); + } + + return false; +} + +/* ??? No autovectorization into MMX or 3DNOW until we can reliably + place emms and femms instructions. */ + +static enum machine_mode +ix86_preferred_simd_mode (enum machine_mode mode) +{ + /* Disable double precision vectorizer if needed. */ + if (mode == DFmode && !TARGET_VECTORIZE_DOUBLE) + return word_mode; + + if (!TARGET_AVX && !TARGET_SSE) + return word_mode; + + switch (mode) + { + case SFmode: + return (TARGET_AVX && !flag_prefer_avx128) ? V8SFmode : V4SFmode; + case DFmode: + return (TARGET_AVX && !flag_prefer_avx128) ? V4DFmode : V2DFmode; + case DImode: + return V2DImode; + case SImode: + return V4SImode; + case HImode: + return V8HImode; + case QImode: + return V16QImode; + + default:; + } + + return word_mode; +} + +/* If AVX is enabled then try vectorizing with both 256bit and 128bit + vectors. */ + +static unsigned int +ix86_autovectorize_vector_sizes (void) +{ + return TARGET_AVX ? 32 | 16 : 0; +} + /* Initialize the GCC target structure. */ #undef TARGET_RETURN_IN_MEMORY #define TARGET_RETURN_IN_MEMORY ix86_return_in_memory @@ -30250,6 +34847,9 @@ #define TARGET_ASM_ALIGNED_DI_OP ASM_QUAD #endif +#undef TARGET_PROFILE_BEFORE_PROLOGUE +#define TARGET_PROFILE_BEFORE_PROLOGUE ix86_profile_before_prologue + #undef TARGET_ASM_UNALIGNED_HI_OP #define TARGET_ASM_UNALIGNED_HI_OP TARGET_ASM_ALIGNED_HI_OP #undef TARGET_ASM_UNALIGNED_SI_OP @@ -30257,6 +34857,17 @@ #undef TARGET_ASM_UNALIGNED_DI_OP #define TARGET_ASM_UNALIGNED_DI_OP TARGET_ASM_ALIGNED_DI_OP +#undef TARGET_PRINT_OPERAND +#define TARGET_PRINT_OPERAND ix86_print_operand +#undef TARGET_PRINT_OPERAND_ADDRESS +#define TARGET_PRINT_OPERAND_ADDRESS ix86_print_operand_address +#undef TARGET_PRINT_OPERAND_PUNCT_VALID_P +#define TARGET_PRINT_OPERAND_PUNCT_VALID_P ix86_print_operand_punct_valid_p +#undef TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA +#define TARGET_ASM_OUTPUT_ADDR_CONST_EXTRA i386_asm_output_addr_const_extra + +#undef TARGET_SCHED_INIT_GLOBAL +#define TARGET_SCHED_INIT_GLOBAL ix86_sched_init_global #undef TARGET_SCHED_ADJUST_COST #define TARGET_SCHED_ADJUST_COST ix86_adjust_cost #undef TARGET_SCHED_ISSUE_RATE @@ -30304,12 +34915,22 @@ #define TARGET_DEFAULT_TARGET_FLAGS \ (TARGET_DEFAULT \ | TARGET_SUBTARGET_DEFAULT \ - | TARGET_TLS_DIRECT_SEG_REFS_DEFAULT \ - | MASK_FUSED_MADD) + | TARGET_TLS_DIRECT_SEG_REFS_DEFAULT) #undef TARGET_HANDLE_OPTION #define TARGET_HANDLE_OPTION ix86_handle_option +#undef TARGET_OPTION_OVERRIDE +#define TARGET_OPTION_OVERRIDE ix86_option_override +#undef TARGET_OPTION_OPTIMIZATION_TABLE +#define TARGET_OPTION_OPTIMIZATION_TABLE ix86_option_optimization_table +#undef TARGET_OPTION_INIT_STRUCT +#define TARGET_OPTION_INIT_STRUCT ix86_option_init_struct + +#undef TARGET_REGISTER_MOVE_COST +#define TARGET_REGISTER_MOVE_COST ix86_register_move_cost +#undef TARGET_MEMORY_MOVE_COST +#define TARGET_MEMORY_MOVE_COST ix86_memory_move_cost #undef TARGET_RTX_COSTS #define TARGET_RTX_COSTS ix86_rtx_costs #undef TARGET_ADDRESS_COST @@ -30329,6 +34950,9 @@ #undef TARGET_BUILD_BUILTIN_VA_LIST #define TARGET_BUILD_BUILTIN_VA_LIST ix86_build_builtin_va_list +#undef TARGET_ENUM_VA_LIST_P +#define TARGET_ENUM_VA_LIST_P ix86_enum_va_list + #undef TARGET_FN_ABI_VA_LIST #define TARGET_FN_ABI_VA_LIST ix86_fn_abi_va_list @@ -30349,6 +34973,12 @@ #define TARGET_SETUP_INCOMING_VARARGS ix86_setup_incoming_varargs #undef TARGET_MUST_PASS_IN_STACK #define TARGET_MUST_PASS_IN_STACK ix86_must_pass_in_stack +#undef TARGET_FUNCTION_ARG_ADVANCE +#define TARGET_FUNCTION_ARG_ADVANCE ix86_function_arg_advance +#undef TARGET_FUNCTION_ARG +#define TARGET_FUNCTION_ARG ix86_function_arg +#undef TARGET_FUNCTION_ARG_BOUNDARY +#define TARGET_FUNCTION_ARG_BOUNDARY ix86_function_arg_boundary #undef TARGET_PASS_BY_REFERENCE #define TARGET_PASS_BY_REFERENCE ix86_pass_by_reference #undef TARGET_INTERNAL_ARG_POINTER @@ -30363,6 +34993,8 @@ #define TARGET_STATIC_CHAIN ix86_static_chain #undef TARGET_TRAMPOLINE_INIT #define TARGET_TRAMPOLINE_INIT ix86_trampoline_init +#undef TARGET_RETURN_POPS_ARGS +#define TARGET_RETURN_POPS_ARGS ix86_return_pops_args #undef TARGET_GIMPLIFY_VA_ARG_EXPR #define TARGET_GIMPLIFY_VA_ARG_EXPR ix86_gimplify_va_arg @@ -30392,12 +35024,25 @@ #undef TARGET_STACK_PROTECT_FAIL #define TARGET_STACK_PROTECT_FAIL ix86_stack_protect_fail +#undef TARGET_SUPPORTS_SPLIT_STACK +#define TARGET_SUPPORTS_SPLIT_STACK ix86_supports_split_stack + #undef TARGET_FUNCTION_VALUE #define TARGET_FUNCTION_VALUE ix86_function_value +#undef TARGET_FUNCTION_VALUE_REGNO_P +#define TARGET_FUNCTION_VALUE_REGNO_P ix86_function_value_regno_p + #undef TARGET_SECONDARY_RELOAD #define TARGET_SECONDARY_RELOAD ix86_secondary_reload +#undef TARGET_PREFERRED_RELOAD_CLASS +#define TARGET_PREFERRED_RELOAD_CLASS ix86_preferred_reload_class +#undef TARGET_PREFERRED_OUTPUT_RELOAD_CLASS +#define TARGET_PREFERRED_OUTPUT_RELOAD_CLASS ix86_preferred_output_reload_class +#undef TARGET_CLASS_LIKELY_SPILLED_P +#define TARGET_CLASS_LIKELY_SPILLED_P ix86_class_likely_spilled_p + #undef TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST #define TARGET_VECTORIZE_BUILTIN_VECTORIZATION_COST \ ix86_builtin_vectorization_cost @@ -30407,6 +35052,12 @@ #undef TARGET_VECTORIZE_BUILTIN_VEC_PERM_OK #define TARGET_VECTORIZE_BUILTIN_VEC_PERM_OK \ ix86_vectorize_builtin_vec_perm_ok +#undef TARGET_VECTORIZE_PREFERRED_SIMD_MODE +#define TARGET_VECTORIZE_PREFERRED_SIMD_MODE \ + ix86_preferred_simd_mode +#undef TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES +#define TARGET_VECTORIZE_AUTOVECTORIZE_VECTOR_SIZES \ + ix86_autovectorize_vector_sizes #undef TARGET_SET_CURRENT_FUNCTION #define TARGET_SET_CURRENT_FUNCTION ix86_set_current_function @@ -30441,6 +35092,20 @@ #undef TARGET_CAN_ELIMINATE #define TARGET_CAN_ELIMINATE ix86_can_eliminate +#undef TARGET_EXTRA_LIVE_ON_ENTRY +#define TARGET_EXTRA_LIVE_ON_ENTRY ix86_live_on_entry + +#undef TARGET_ASM_CODE_END +#define TARGET_ASM_CODE_END ix86_code_end + +#undef TARGET_CONDITIONAL_REGISTER_USAGE +#define TARGET_CONDITIONAL_REGISTER_USAGE ix86_conditional_register_usage + +#if TARGET_MACHO +#undef TARGET_INIT_LIBFUNCS +#define TARGET_INIT_LIBFUNCS darwin_rename_builtins +#endif + struct gcc_target targetm = TARGET_INITIALIZER; #include "gt-i386.h"