Mercurial > hg > CbC > CbC_gcc
diff gcc/lto-cgraph.c @ 111:04ced10e8804
gcc 7
author | kono |
---|---|
date | Fri, 27 Oct 2017 22:46:09 +0900 |
parents | f6334be47118 |
children | 84e7813d76e9 |
line wrap: on
line diff
--- a/gcc/lto-cgraph.c Sun Aug 21 07:07:55 2011 +0900 +++ b/gcc/lto-cgraph.c Fri Oct 27 22:46:09 2017 +0900 @@ -1,7 +1,7 @@ /* Write and read the cgraph to the memory mapped representation of a .o file. - Copyright 2009, 2010 Free Software Foundation, Inc. + Copyright (C) 2009-2017 Free Software Foundation, Inc. Contributed by Kenneth Zadeck <zadeck@naturalbridge.com> This file is part of GCC. @@ -23,57 +23,65 @@ #include "config.h" #include "system.h" #include "coretypes.h" -#include "tm.h" +#include "backend.h" +#include "rtl.h" #include "tree.h" -#include "expr.h" -#include "flags.h" -#include "params.h" -#include "input.h" -#include "hashtab.h" -#include "langhooks.h" -#include "basic-block.h" -#include "tree-flow.h" +#include "gimple.h" +#include "predict.h" +#include "stringpool.h" +#include "tree-streamer.h" #include "cgraph.h" -#include "function.h" -#include "ggc.h" -#include "diagnostic-core.h" -#include "except.h" -#include "vec.h" -#include "timevar.h" -#include "output.h" -#include "pointer-set.h" -#include "lto-streamer.h" -#include "gcov-io.h" +#include "tree-pass.h" +#include "profile.h" +#include "context.h" +#include "pass_manager.h" +#include "ipa-utils.h" +#include "omp-offload.h" +#include "ipa-chkp.h" +#include "stringpool.h" +#include "attribs.h" -static void output_varpool (cgraph_node_set, varpool_node_set); -static void output_cgraph_opt_summary (cgraph_node_set set); -static void input_cgraph_opt_summary (VEC (cgraph_node_ptr, heap) * nodes); +/* True when asm nodes has been output. */ +bool asm_nodes_output = false; + +static void output_cgraph_opt_summary (void); +static void input_cgraph_opt_summary (vec<symtab_node *> nodes); +/* Number of LDPR values known to GCC. */ +#define LDPR_NUM_KNOWN (LDPR_PREVAILING_DEF_IRONLY_EXP + 1) + +/* All node orders are ofsetted by ORDER_BASE. */ +static int order_base; /* Cgraph streaming is organized as set of record whose type is indicated by a tag. */ -enum LTO_cgraph_tags +enum LTO_symtab_tags { /* Must leave 0 for the stopper. */ /* Cgraph node without body available. */ - LTO_cgraph_unavail_node = 1, + LTO_symtab_unavail_node = 1, /* Cgraph node with function body. */ - LTO_cgraph_analyzed_node, + LTO_symtab_analyzed_node, /* Cgraph edges. */ - LTO_cgraph_edge, - LTO_cgraph_indirect_edge + LTO_symtab_edge, + LTO_symtab_indirect_edge, + LTO_symtab_variable, + LTO_symtab_last_tag }; -/* Create a new cgraph encoder. */ +/* Create a new symtab encoder. + if FOR_INPUT, the encoder allocate only datastructures needed + to read the symtab. */ -lto_cgraph_encoder_t -lto_cgraph_encoder_new (void) +lto_symtab_encoder_t +lto_symtab_encoder_new (bool for_input) { - lto_cgraph_encoder_t encoder = XCNEW (struct lto_cgraph_encoder_d); - encoder->map = pointer_map_create (); - encoder->nodes = NULL; - encoder->body = pointer_set_create (); + lto_symtab_encoder_t encoder = XCNEW (struct lto_symtab_encoder_d); + + if (!for_input) + encoder->map = new hash_map<symtab_node *, size_t>; + encoder->nodes.create (0); return encoder; } @@ -81,214 +89,191 @@ /* Delete ENCODER and its components. */ void -lto_cgraph_encoder_delete (lto_cgraph_encoder_t encoder) +lto_symtab_encoder_delete (lto_symtab_encoder_t encoder) { - VEC_free (cgraph_node_ptr, heap, encoder->nodes); - pointer_map_destroy (encoder->map); - pointer_set_destroy (encoder->body); + encoder->nodes.release (); + if (encoder->map) + delete encoder->map; free (encoder); } -/* Return the existing reference number of NODE in the cgraph encoder in +/* Return the existing reference number of NODE in the symtab encoder in output block OB. Assign a new reference if this is the first time NODE is encoded. */ int -lto_cgraph_encoder_encode (lto_cgraph_encoder_t encoder, - struct cgraph_node *node) +lto_symtab_encoder_encode (lto_symtab_encoder_t encoder, + symtab_node *node) { int ref; - void **slot; - slot = pointer_map_contains (encoder->map, node); - if (!slot) + if (!encoder->map) { - ref = VEC_length (cgraph_node_ptr, encoder->nodes); - slot = pointer_map_insert (encoder->map, node); - *slot = (void *) (intptr_t) ref; - VEC_safe_push (cgraph_node_ptr, heap, encoder->nodes, node); + lto_encoder_entry entry = {node, false, false, false}; + + ref = encoder->nodes.length (); + encoder->nodes.safe_push (entry); + return ref; + } + + size_t *slot = encoder->map->get (node); + if (!slot || !*slot) + { + lto_encoder_entry entry = {node, false, false, false}; + ref = encoder->nodes.length (); + if (!slot) + encoder->map->put (node, ref + 1); + encoder->nodes.safe_push (entry); } else - ref = (int) (intptr_t) *slot; + ref = *slot - 1; return ref; } -#define LCC_NOT_FOUND (-1) +/* Remove NODE from encoder. */ -/* Look up NODE in encoder. Return NODE's reference if it has been encoded - or LCC_NOT_FOUND if it is not there. */ +bool +lto_symtab_encoder_delete_node (lto_symtab_encoder_t encoder, + symtab_node *node) +{ + int index; + lto_encoder_entry last_node; + + size_t *slot = encoder->map->get (node); + if (slot == NULL || !*slot) + return false; -int -lto_cgraph_encoder_lookup (lto_cgraph_encoder_t encoder, - struct cgraph_node *node) -{ - void **slot = pointer_map_contains (encoder->map, node); - return (slot ? (int) (intptr_t) *slot : LCC_NOT_FOUND); + index = *slot - 1; + gcc_checking_assert (encoder->nodes[index].node == node); + + /* Remove from vector. We do this by swapping node with the last element + of the vector. */ + last_node = encoder->nodes.pop (); + if (last_node.node != node) + { + gcc_assert (encoder->map->put (last_node.node, index + 1)); + + /* Move the last element to the original spot of NODE. */ + encoder->nodes[index] = last_node; + } + + /* Remove element from hash table. */ + encoder->map->remove (node); + return true; } -/* Return the cgraph node corresponding to REF using ENCODER. */ +/* Return TRUE if we should encode the body of NODE (if any). */ -struct cgraph_node * -lto_cgraph_encoder_deref (lto_cgraph_encoder_t encoder, int ref) +bool +lto_symtab_encoder_encode_body_p (lto_symtab_encoder_t encoder, + struct cgraph_node *node) { - if (ref == LCC_NOT_FOUND) - return NULL; - - return VEC_index (cgraph_node_ptr, encoder->nodes, ref); + int index = lto_symtab_encoder_lookup (encoder, node); + return encoder->nodes[index].body; } +/* Specify that we encode the body of NODE in this partition. */ + +static void +lto_set_symtab_encoder_encode_body (lto_symtab_encoder_t encoder, + struct cgraph_node *node) +{ + int index = lto_symtab_encoder_encode (encoder, node); + gcc_checking_assert (encoder->nodes[index].node == node); + encoder->nodes[index].body = true; +} /* Return TRUE if we should encode initializer of NODE (if any). */ bool -lto_cgraph_encoder_encode_body_p (lto_cgraph_encoder_t encoder, - struct cgraph_node *node) +lto_symtab_encoder_encode_initializer_p (lto_symtab_encoder_t encoder, + varpool_node *node) { - return pointer_set_contains (encoder->body, node); -} - -/* Return TRUE if we should encode body of NODE (if any). */ - -static void -lto_set_cgraph_encoder_encode_body (lto_cgraph_encoder_t encoder, - struct cgraph_node *node) -{ - pointer_set_insert (encoder->body, node); + int index = lto_symtab_encoder_lookup (encoder, node); + if (index == LCC_NOT_FOUND) + return false; + return encoder->nodes[index].initializer; } -/* Create a new varpool encoder. */ +/* Specify that we should encode initializer of NODE (if any). */ -lto_varpool_encoder_t -lto_varpool_encoder_new (void) +static void +lto_set_symtab_encoder_encode_initializer (lto_symtab_encoder_t encoder, + varpool_node *node) { - lto_varpool_encoder_t encoder = XCNEW (struct lto_varpool_encoder_d); - encoder->map = pointer_map_create (); - encoder->initializer = pointer_set_create (); - encoder->nodes = NULL; - return encoder; -} - - -/* Delete ENCODER and its components. */ - -void -lto_varpool_encoder_delete (lto_varpool_encoder_t encoder) -{ - VEC_free (varpool_node_ptr, heap, encoder->nodes); - pointer_map_destroy (encoder->map); - pointer_set_destroy (encoder->initializer); - free (encoder); + int index = lto_symtab_encoder_lookup (encoder, node); + encoder->nodes[index].initializer = true; } - -/* Return the existing reference number of NODE in the varpool encoder in - output block OB. Assign a new reference if this is the first time - NODE is encoded. */ - -int -lto_varpool_encoder_encode (lto_varpool_encoder_t encoder, - struct varpool_node *node) -{ - int ref; - void **slot; +/* Return TRUE if NODE is in this partition. */ - slot = pointer_map_contains (encoder->map, node); - if (!slot) - { - ref = VEC_length (varpool_node_ptr, encoder->nodes); - slot = pointer_map_insert (encoder->map, node); - *slot = (void *) (intptr_t) ref; - VEC_safe_push (varpool_node_ptr, heap, encoder->nodes, node); - } - else - ref = (int) (intptr_t) *slot; - - return ref; +bool +lto_symtab_encoder_in_partition_p (lto_symtab_encoder_t encoder, + symtab_node *node) +{ + int index = lto_symtab_encoder_lookup (encoder, node); + if (index == LCC_NOT_FOUND) + return false; + return encoder->nodes[index].in_partition; } -/* Look up NODE in encoder. Return NODE's reference if it has been encoded - or LCC_NOT_FOUND if it is not there. */ - -int -lto_varpool_encoder_lookup (lto_varpool_encoder_t encoder, - struct varpool_node *node) -{ - void **slot = pointer_map_contains (encoder->map, node); - return (slot ? (int) (intptr_t) *slot : LCC_NOT_FOUND); -} - - -/* Return the varpool node corresponding to REF using ENCODER. */ - -struct varpool_node * -lto_varpool_encoder_deref (lto_varpool_encoder_t encoder, int ref) -{ - if (ref == LCC_NOT_FOUND) - return NULL; +/* Specify that NODE is in this partition. */ - return VEC_index (varpool_node_ptr, encoder->nodes, ref); -} - - -/* Return TRUE if we should encode initializer of NODE (if any). */ - -bool -lto_varpool_encoder_encode_initializer_p (lto_varpool_encoder_t encoder, - struct varpool_node *node) +void +lto_set_symtab_encoder_in_partition (lto_symtab_encoder_t encoder, + symtab_node *node) { - return pointer_set_contains (encoder->initializer, node); -} - -/* Return TRUE if we should encode initializer of NODE (if any). */ - -static void -lto_set_varpool_encoder_encode_initializer (lto_varpool_encoder_t encoder, - struct varpool_node *node) -{ - pointer_set_insert (encoder->initializer, node); + int index = lto_symtab_encoder_encode (encoder, node); + encoder->nodes[index].in_partition = true; } /* Output the cgraph EDGE to OB using ENCODER. */ static void lto_output_edge (struct lto_simple_output_block *ob, struct cgraph_edge *edge, - lto_cgraph_encoder_t encoder) + lto_symtab_encoder_t encoder) { unsigned int uid; intptr_t ref; struct bitpack_d bp; if (edge->indirect_unknown_callee) - lto_output_uleb128_stream (ob->main_stream, LTO_cgraph_indirect_edge); + streamer_write_enum (ob->main_stream, LTO_symtab_tags, LTO_symtab_last_tag, + LTO_symtab_indirect_edge); else - lto_output_uleb128_stream (ob->main_stream, LTO_cgraph_edge); + streamer_write_enum (ob->main_stream, LTO_symtab_tags, LTO_symtab_last_tag, + LTO_symtab_edge); - ref = lto_cgraph_encoder_lookup (encoder, edge->caller); + ref = lto_symtab_encoder_lookup (encoder, edge->caller); gcc_assert (ref != LCC_NOT_FOUND); - lto_output_sleb128_stream (ob->main_stream, ref); + streamer_write_hwi_stream (ob->main_stream, ref); if (!edge->indirect_unknown_callee) { - ref = lto_cgraph_encoder_lookup (encoder, edge->callee); + ref = lto_symtab_encoder_lookup (encoder, edge->callee); gcc_assert (ref != LCC_NOT_FOUND); - lto_output_sleb128_stream (ob->main_stream, ref); + streamer_write_hwi_stream (ob->main_stream, ref); } - lto_output_sleb128_stream (ob->main_stream, edge->count); + edge->count.stream_out (ob->main_stream); bp = bitpack_create (ob->main_stream); - uid = (!gimple_has_body_p (edge->caller->decl) - ? edge->lto_stmt_uid : gimple_uid (edge->call_stmt)); - bp_pack_value (&bp, uid, HOST_BITS_PER_INT); - bp_pack_value (&bp, edge->inline_failed, HOST_BITS_PER_INT); - bp_pack_value (&bp, edge->frequency, HOST_BITS_PER_INT); - bp_pack_value (&bp, edge->loop_nest, 30); + uid = (!gimple_has_body_p (edge->caller->decl) || edge->caller->thunk.thunk_p + ? edge->lto_stmt_uid : gimple_uid (edge->call_stmt) + 1); + bp_pack_enum (&bp, cgraph_inline_failed_t, + CIF_N_REASONS, edge->inline_failed); + bp_pack_var_len_unsigned (&bp, uid); + bp_pack_var_len_unsigned (&bp, edge->frequency); bp_pack_value (&bp, edge->indirect_inlining_edge, 1); + bp_pack_value (&bp, edge->speculative, 1); bp_pack_value (&bp, edge->call_stmt_cannot_inline_p, 1); + gcc_assert (!edge->call_stmt_cannot_inline_p + || edge->inline_failed != CIF_BODY_NOT_AVAILABLE); bp_pack_value (&bp, edge->can_throw_external, 1); + bp_pack_value (&bp, edge->in_polymorphic_cdtor, 1); if (edge->indirect_unknown_callee) { int flags = edge->indirect_info->ecf_flags; @@ -305,32 +290,35 @@ | ECF_LEAF | ECF_NOVOPS))); } - lto_output_bitpack (&bp); + streamer_write_bitpack (&bp); + if (edge->indirect_unknown_callee) + { + streamer_write_hwi_stream (ob->main_stream, + edge->indirect_info->common_target_id); + if (edge->indirect_info->common_target_id) + streamer_write_hwi_stream + (ob->main_stream, edge->indirect_info->common_target_probability); + } } -/* Return if LIST contain references from other partitions. */ +/* Return if NODE contain references from other partitions. */ bool -referenced_from_other_partition_p (struct ipa_ref_list *list, cgraph_node_set set, - varpool_node_set vset) +referenced_from_other_partition_p (symtab_node *node, lto_symtab_encoder_t encoder) { int i; - struct ipa_ref *ref; - for (i = 0; ipa_ref_list_refering_iterate (list, i, ref); i++) + struct ipa_ref *ref = NULL; + + for (i = 0; node->iterate_referring (i, ref); i++) { - if (ref->refering_type == IPA_REF_CGRAPH) - { - if (ipa_ref_refering_node (ref)->in_other_partition - || !cgraph_node_in_set_p (ipa_ref_refering_node (ref), set)) - return true; - } - else - { - if (ipa_ref_refering_varpool_node (ref)->in_other_partition - || !varpool_node_in_set_p (ipa_ref_refering_varpool_node (ref), - vset)) - return true; - } + /* Ignore references from non-offloadable nodes while streaming NODE into + offload LTO section. */ + if (!ref->referring->need_lto_streaming) + continue; + + if (ref->referring->in_other_partition + || !lto_symtab_encoder_in_partition_p (encoder, ref->referring)) + return true; } return false; } @@ -338,53 +326,50 @@ /* Return true when node is reachable from other partition. */ bool -reachable_from_other_partition_p (struct cgraph_node *node, cgraph_node_set set) +reachable_from_other_partition_p (struct cgraph_node *node, lto_symtab_encoder_t encoder) { struct cgraph_edge *e; - if (!node->analyzed) + if (!node->definition) return false; if (node->global.inlined_to) return false; for (e = node->callers; e; e = e->next_caller) - if (e->caller->in_other_partition - || !cgraph_node_in_set_p (e->caller, set)) - return true; + { + /* Ignore references from non-offloadable nodes while streaming NODE into + offload LTO section. */ + if (!e->caller->need_lto_streaming) + continue; + + if (e->caller->in_other_partition + || !lto_symtab_encoder_in_partition_p (encoder, e->caller)) + return true; + } return false; } -/* Return if LIST contain references from other partitions. */ +/* Return if NODE contain references from other partitions. */ bool -referenced_from_this_partition_p (struct ipa_ref_list *list, cgraph_node_set set, - varpool_node_set vset) +referenced_from_this_partition_p (symtab_node *node, + lto_symtab_encoder_t encoder) { int i; - struct ipa_ref *ref; - for (i = 0; ipa_ref_list_refering_iterate (list, i, ref); i++) - { - if (ref->refering_type == IPA_REF_CGRAPH) - { - if (cgraph_node_in_set_p (ipa_ref_refering_node (ref), set)) - return true; - } - else - { - if (varpool_node_in_set_p (ipa_ref_refering_varpool_node (ref), - vset)) - return true; - } - } + struct ipa_ref *ref = NULL; + + for (i = 0; node->iterate_referring (i, ref); i++) + if (lto_symtab_encoder_in_partition_p (encoder, ref->referring)) + return true; return false; } /* Return true when node is reachable from other partition. */ bool -reachable_from_this_partition_p (struct cgraph_node *node, cgraph_node_set set) +reachable_from_this_partition_p (struct cgraph_node *node, lto_symtab_encoder_t encoder) { struct cgraph_edge *e; for (e = node->callers; e; e = e->next_caller) - if (cgraph_node_in_set_p (e->caller, set)) + if (lto_symtab_encoder_in_partition_p (encoder, e->caller)) return true; return false; } @@ -399,24 +384,31 @@ static void lto_output_node (struct lto_simple_output_block *ob, struct cgraph_node *node, - lto_cgraph_encoder_t encoder, cgraph_node_set set, - varpool_node_set vset) + lto_symtab_encoder_t encoder) { unsigned int tag; struct bitpack_d bp; bool boundary_p; intptr_t ref; bool in_other_partition = false; - struct cgraph_node *clone_of; + struct cgraph_node *clone_of, *ultimate_clone_of; + ipa_opt_pass_d *pass; + int i; + const char *comdat; + const char *section; + tree group; - boundary_p = !cgraph_node_in_set_p (node, set); + boundary_p = !lto_symtab_encoder_in_partition_p (encoder, node); - if (node->analyzed && !boundary_p) - tag = LTO_cgraph_analyzed_node; + if (node->analyzed && (!boundary_p || node->alias + || (node->thunk.thunk_p && !node->global.inlined_to))) + tag = LTO_symtab_analyzed_node; else - tag = LTO_cgraph_unavail_node; + tag = LTO_symtab_unavail_node; - lto_output_uleb128_stream (ob->main_stream, tag); + streamer_write_enum (ob->main_stream, LTO_symtab_tags, LTO_symtab_last_tag, + tag); + streamer_write_hwi_stream (ob->main_stream, node->order); /* In WPA mode, we only output part of the call-graph. Also, we fake cgraph node attributes. There are two cases that we care. @@ -428,7 +420,8 @@ Cherry-picked nodes: These are nodes we pulled from other translation units into SET during IPA-inlining. We make them as local static nodes to prevent clashes with other local statics. */ - if (boundary_p && node->analyzed) + if (boundary_p && node->analyzed + && node->get_partitioning_class () == SYMBOL_PARTITION) { /* Inline clones can not be part of boundary. gcc_assert (!node->global.inlined_to); @@ -443,187 +436,230 @@ clone_of = node->clone_of; while (clone_of - && (ref = lto_cgraph_encoder_lookup (encoder, clone_of)) == LCC_NOT_FOUND) + && (ref = lto_symtab_encoder_lookup (encoder, clone_of)) == LCC_NOT_FOUND) if (clone_of->prev_sibling_clone) clone_of = clone_of->prev_sibling_clone; else clone_of = clone_of->clone_of; - if (LTO_cgraph_analyzed_node) + /* See if body of the master function is output. If not, we are seeing only + an declaration and we do not need to pass down clone tree. */ + ultimate_clone_of = clone_of; + while (ultimate_clone_of && ultimate_clone_of->clone_of) + ultimate_clone_of = ultimate_clone_of->clone_of; + + if (clone_of && !lto_symtab_encoder_encode_body_p (encoder, ultimate_clone_of)) + clone_of = NULL; + + if (tag == LTO_symtab_analyzed_node) gcc_assert (clone_of || !node->clone_of); if (!clone_of) - lto_output_sleb128_stream (ob->main_stream, LCC_NOT_FOUND); + streamer_write_hwi_stream (ob->main_stream, LCC_NOT_FOUND); else - lto_output_sleb128_stream (ob->main_stream, ref); + streamer_write_hwi_stream (ob->main_stream, ref); lto_output_fn_decl_index (ob->decl_state, ob->main_stream, node->decl); - lto_output_sleb128_stream (ob->main_stream, node->count); - lto_output_sleb128_stream (ob->main_stream, node->count_materialization_scale); + node->count.stream_out (ob->main_stream); + streamer_write_hwi_stream (ob->main_stream, node->count_materialization_scale); - if (tag == LTO_cgraph_analyzed_node) + streamer_write_hwi_stream (ob->main_stream, + node->ipa_transforms_to_apply.length ()); + FOR_EACH_VEC_ELT (node->ipa_transforms_to_apply, i, pass) + streamer_write_hwi_stream (ob->main_stream, pass->static_pass_number); + + if (tag == LTO_symtab_analyzed_node) { - lto_output_sleb128_stream (ob->main_stream, - node->local.inline_summary.estimated_self_stack_size); - lto_output_sleb128_stream (ob->main_stream, - node->local.inline_summary.self_size); - lto_output_sleb128_stream (ob->main_stream, - node->local.inline_summary.size_inlining_benefit); - lto_output_sleb128_stream (ob->main_stream, - node->local.inline_summary.self_time); - lto_output_sleb128_stream (ob->main_stream, - node->local.inline_summary.time_inlining_benefit); if (node->global.inlined_to) { - ref = lto_cgraph_encoder_lookup (encoder, node->global.inlined_to); + ref = lto_symtab_encoder_lookup (encoder, node->global.inlined_to); gcc_assert (ref != LCC_NOT_FOUND); } else ref = LCC_NOT_FOUND; - lto_output_sleb128_stream (ob->main_stream, ref); + streamer_write_hwi_stream (ob->main_stream, ref); } - if (node->same_comdat_group && !boundary_p) + group = node->get_comdat_group (); + if (group) + comdat = IDENTIFIER_POINTER (group); + else + comdat = ""; + streamer_write_data_stream (ob->main_stream, comdat, strlen (comdat) + 1); + + if (group) { - ref = lto_cgraph_encoder_lookup (encoder, node->same_comdat_group); - gcc_assert (ref != LCC_NOT_FOUND); + if (node->same_comdat_group) + { + ref = LCC_NOT_FOUND; + for (struct symtab_node *n = node->same_comdat_group; + ref == LCC_NOT_FOUND && n != node; n = n->same_comdat_group) + ref = lto_symtab_encoder_lookup (encoder, n); + } + else + ref = LCC_NOT_FOUND; + streamer_write_hwi_stream (ob->main_stream, ref); } - else - ref = LCC_NOT_FOUND; - lto_output_sleb128_stream (ob->main_stream, ref); + + section = node->get_section (); + if (!section) + section = ""; + + streamer_write_hwi_stream (ob->main_stream, node->tp_first_run); bp = bitpack_create (ob->main_stream); bp_pack_value (&bp, node->local.local, 1); - bp_pack_value (&bp, node->local.externally_visible, 1); - bp_pack_value (&bp, node->local.finalized, 1); - bp_pack_value (&bp, node->local.inlinable, 1); + bp_pack_value (&bp, node->externally_visible, 1); + bp_pack_value (&bp, node->no_reorder, 1); + bp_pack_value (&bp, node->definition, 1); bp_pack_value (&bp, node->local.versionable, 1); bp_pack_value (&bp, node->local.can_change_signature, 1); - bp_pack_value (&bp, node->local.disregard_inline_limits, 1); bp_pack_value (&bp, node->local.redefined_extern_inline, 1); - bp_pack_value (&bp, node->local.vtable_method, 1); - bp_pack_value (&bp, node->needed, 1); + bp_pack_value (&bp, node->force_output, 1); + bp_pack_value (&bp, node->forced_by_abi, 1); + bp_pack_value (&bp, node->unique_name, 1); + bp_pack_value (&bp, node->body_removed, 1); + bp_pack_value (&bp, node->implicit_section, 1); bp_pack_value (&bp, node->address_taken, 1); - bp_pack_value (&bp, node->abstract_and_needed, 1); - bp_pack_value (&bp, tag == LTO_cgraph_analyzed_node - && !DECL_EXTERNAL (node->decl) - && !DECL_COMDAT (node->decl) - && (reachable_from_other_partition_p (node, set) - || referenced_from_other_partition_p (&node->ref_list, set, vset)), 1); + bp_pack_value (&bp, tag == LTO_symtab_analyzed_node + && node->get_partitioning_class () == SYMBOL_PARTITION + && (reachable_from_other_partition_p (node, encoder) + || referenced_from_other_partition_p (node, encoder)), 1); bp_pack_value (&bp, node->lowered, 1); bp_pack_value (&bp, in_other_partition, 1); bp_pack_value (&bp, node->alias, 1); - bp_pack_value (&bp, node->finalized_by_frontend, 1); + bp_pack_value (&bp, node->transparent_alias, 1); + bp_pack_value (&bp, node->weakref, 1); bp_pack_value (&bp, node->frequency, 2); bp_pack_value (&bp, node->only_called_at_startup, 1); bp_pack_value (&bp, node->only_called_at_exit, 1); - lto_output_bitpack (&bp); - lto_output_uleb128_stream (ob->main_stream, node->resolution); + bp_pack_value (&bp, node->tm_clone, 1); + bp_pack_value (&bp, node->calls_comdat_local, 1); + bp_pack_value (&bp, node->icf_merged, 1); + bp_pack_value (&bp, node->nonfreeing_fn, 1); + bp_pack_value (&bp, node->thunk.thunk_p, 1); + bp_pack_value (&bp, node->parallelized_function, 1); + bp_pack_enum (&bp, ld_plugin_symbol_resolution, + LDPR_NUM_KNOWN, node->resolution); + bp_pack_value (&bp, node->instrumentation_clone, 1); + bp_pack_value (&bp, node->split_part, 1); + streamer_write_bitpack (&bp); + streamer_write_data_stream (ob->main_stream, section, strlen (section) + 1); - if (node->same_body) + if (node->thunk.thunk_p) { - struct cgraph_node *alias; - unsigned long alias_count = 1; - for (alias = node->same_body; alias->next; alias = alias->next) - alias_count++; - lto_output_uleb128_stream (ob->main_stream, alias_count); - do - { - lto_output_fn_decl_index (ob->decl_state, ob->main_stream, - alias->decl); - if (alias->thunk.thunk_p) - { - lto_output_uleb128_stream - (ob->main_stream, - 1 + (alias->thunk.this_adjusting != 0) * 2 - + (alias->thunk.virtual_offset_p != 0) * 4); - lto_output_uleb128_stream (ob->main_stream, - alias->thunk.fixed_offset); - lto_output_uleb128_stream (ob->main_stream, - alias->thunk.virtual_value); - lto_output_fn_decl_index (ob->decl_state, ob->main_stream, - alias->thunk.alias); - } - else - { - lto_output_uleb128_stream (ob->main_stream, 0); - lto_output_fn_decl_index (ob->decl_state, ob->main_stream, - alias->thunk.alias); - } - gcc_assert (cgraph_get_node (alias->thunk.alias) == node); - lto_output_uleb128_stream (ob->main_stream, alias->resolution); - alias = alias->previous; - } - while (alias); + streamer_write_uhwi_stream + (ob->main_stream, + 1 + (node->thunk.this_adjusting != 0) * 2 + + (node->thunk.virtual_offset_p != 0) * 4 + + (node->thunk.add_pointer_bounds_args != 0) * 8); + streamer_write_uhwi_stream (ob->main_stream, node->thunk.fixed_offset); + streamer_write_uhwi_stream (ob->main_stream, node->thunk.virtual_value); } - else - lto_output_uleb128_stream (ob->main_stream, 0); + streamer_write_hwi_stream (ob->main_stream, node->profile_id); + if (DECL_STATIC_CONSTRUCTOR (node->decl)) + streamer_write_hwi_stream (ob->main_stream, node->get_init_priority ()); + if (DECL_STATIC_DESTRUCTOR (node->decl)) + streamer_write_hwi_stream (ob->main_stream, node->get_fini_priority ()); + + if (node->instrumentation_clone) + lto_output_fn_decl_index (ob->decl_state, ob->main_stream, node->orig_decl); } /* Output the varpool NODE to OB. If NODE is not in SET, then NODE is a boundary. */ static void -lto_output_varpool_node (struct lto_simple_output_block *ob, struct varpool_node *node, - lto_varpool_encoder_t varpool_encoder, - cgraph_node_set set, varpool_node_set vset) +lto_output_varpool_node (struct lto_simple_output_block *ob, varpool_node *node, + lto_symtab_encoder_t encoder) { - bool boundary_p = !varpool_node_in_set_p (node, vset) && node->analyzed; + bool boundary_p = !lto_symtab_encoder_in_partition_p (encoder, node); + bool encode_initializer_p + = (node->definition + && lto_symtab_encoder_encode_initializer_p (encoder, node)); struct bitpack_d bp; - struct varpool_node *alias; - int count = 0; int ref; + const char *comdat; + const char *section; + tree group; + gcc_assert (!encode_initializer_p || node->definition); + gcc_assert (boundary_p || encode_initializer_p); + + streamer_write_enum (ob->main_stream, LTO_symtab_tags, LTO_symtab_last_tag, + LTO_symtab_variable); + streamer_write_hwi_stream (ob->main_stream, node->order); lto_output_var_decl_index (ob->decl_state, ob->main_stream, node->decl); bp = bitpack_create (ob->main_stream); bp_pack_value (&bp, node->externally_visible, 1); + bp_pack_value (&bp, node->no_reorder, 1); bp_pack_value (&bp, node->force_output, 1); - bp_pack_value (&bp, node->finalized, 1); + bp_pack_value (&bp, node->forced_by_abi, 1); + bp_pack_value (&bp, node->unique_name, 1); + bp_pack_value (&bp, + node->body_removed + || (!encode_initializer_p && !node->alias && node->definition), + 1); + bp_pack_value (&bp, node->implicit_section, 1); + bp_pack_value (&bp, node->writeonly, 1); + bp_pack_value (&bp, node->definition && (encode_initializer_p || node->alias), + 1); bp_pack_value (&bp, node->alias, 1); - gcc_assert (!node->alias || !node->extra_name); - gcc_assert (node->finalized || !node->analyzed); - gcc_assert (node->needed); + bp_pack_value (&bp, node->transparent_alias, 1); + bp_pack_value (&bp, node->weakref, 1); + bp_pack_value (&bp, node->analyzed && (!boundary_p || node->alias), 1); + gcc_assert (node->definition || !node->analyzed); /* Constant pool initializers can be de-unified into individual ltrans units. FIXME: Alternatively at -Os we may want to avoid generating for them the local labels and share them across LTRANS partitions. */ - if (DECL_IN_CONSTANT_POOL (node->decl) - && !DECL_COMDAT (node->decl)) + if (node->get_partitioning_class () != SYMBOL_PARTITION) { bp_pack_value (&bp, 0, 1); /* used_from_other_parition. */ bp_pack_value (&bp, 0, 1); /* in_other_partition. */ } else { + bp_pack_value (&bp, node->definition + && referenced_from_other_partition_p (node, encoder), 1); bp_pack_value (&bp, node->analyzed - && referenced_from_other_partition_p (&node->ref_list, - set, vset), 1); - bp_pack_value (&bp, boundary_p, 1); /* in_other_partition. */ - } - /* Also emit any extra name aliases. */ - for (alias = node->extra_name; alias; alias = alias->next) - count++; - bp_pack_value (&bp, count != 0, 1); - lto_output_bitpack (&bp); - if (node->same_comdat_group && !boundary_p) - { - ref = lto_varpool_encoder_lookup (varpool_encoder, node->same_comdat_group); - gcc_assert (ref != LCC_NOT_FOUND); + && boundary_p && !DECL_EXTERNAL (node->decl), 1); + /* in_other_partition. */ } + bp_pack_value (&bp, node->tls_model, 3); + bp_pack_value (&bp, node->used_by_single_function, 1); + bp_pack_value (&bp, node->dynamically_initialized, 1); + bp_pack_value (&bp, node->need_bounds_init, 1); + streamer_write_bitpack (&bp); + + group = node->get_comdat_group (); + if (group) + comdat = IDENTIFIER_POINTER (group); else - ref = LCC_NOT_FOUND; - lto_output_sleb128_stream (ob->main_stream, ref); - lto_output_uleb128_stream (ob->main_stream, node->resolution); + comdat = ""; + streamer_write_data_stream (ob->main_stream, comdat, strlen (comdat) + 1); - if (count) + if (group) { - lto_output_uleb128_stream (ob->main_stream, count); - for (alias = node->extra_name; alias; alias = alias->next) + if (node->same_comdat_group) { - lto_output_var_decl_index (ob->decl_state, ob->main_stream, alias->decl); - lto_output_uleb128_stream (ob->main_stream, alias->resolution); + ref = LCC_NOT_FOUND; + for (struct symtab_node *n = node->same_comdat_group; + ref == LCC_NOT_FOUND && n != node; n = n->same_comdat_group) + ref = lto_symtab_encoder_lookup (encoder, n); } + else + ref = LCC_NOT_FOUND; + streamer_write_hwi_stream (ob->main_stream, ref); } + + section = node->get_section (); + if (!section) + section = ""; + streamer_write_data_stream (ob->main_stream, section, strlen (section) + 1); + + streamer_write_enum (ob->main_stream, ld_plugin_symbol_resolution, + LDPR_NUM_KNOWN, node->resolution); } /* Output the varpool NODE to OB. @@ -631,26 +667,27 @@ static void lto_output_ref (struct lto_simple_output_block *ob, struct ipa_ref *ref, - lto_cgraph_encoder_t encoder, - lto_varpool_encoder_t varpool_encoder) + lto_symtab_encoder_t encoder) { struct bitpack_d bp; + int nref; + int uid = ref->lto_stmt_uid; + struct cgraph_node *node; + bp = bitpack_create (ob->main_stream); - bp_pack_value (&bp, ref->refered_type, 1); - bp_pack_value (&bp, ref->use, 2); - lto_output_bitpack (&bp); - if (ref->refered_type == IPA_REF_CGRAPH) + bp_pack_value (&bp, ref->use, 3); + bp_pack_value (&bp, ref->speculative, 1); + streamer_write_bitpack (&bp); + nref = lto_symtab_encoder_lookup (encoder, ref->referred); + gcc_assert (nref != LCC_NOT_FOUND); + streamer_write_hwi_stream (ob->main_stream, nref); + + node = dyn_cast <cgraph_node *> (ref->referring); + if (node) { - int nref = lto_cgraph_encoder_lookup (encoder, ipa_ref_node (ref)); - gcc_assert (nref != LCC_NOT_FOUND); - lto_output_sleb128_stream (ob->main_stream, nref); - } - else - { - int nref = lto_varpool_encoder_lookup (varpool_encoder, - ipa_ref_varpool_node (ref)); - gcc_assert (nref != LCC_NOT_FOUND); - lto_output_sleb128_stream (ob->main_stream, nref); + if (ref->stmt) + uid = gimple_uid (ref->stmt) + 1; + streamer_write_hwi_stream (ob->main_stream, uid); } } @@ -659,50 +696,47 @@ static void output_profile_summary (struct lto_simple_output_block *ob) { + unsigned h_ix; + struct bitpack_d bp; + if (profile_info) { - /* We do not output num, sum_all and run_max, they are not used by - GCC profile feedback and they are difficult to merge from multiple - units. */ + /* We do not output num and run_max, they are not used by + GCC profile feedback and they are difficult to merge from multiple + units. */ gcc_assert (profile_info->runs); - lto_output_uleb128_stream (ob->main_stream, profile_info->runs); - lto_output_uleb128_stream (ob->main_stream, profile_info->sum_max); + streamer_write_uhwi_stream (ob->main_stream, profile_info->runs); + streamer_write_gcov_count_stream (ob->main_stream, profile_info->sum_max); + + /* sum_all is needed for computing the working set with the + histogram. */ + streamer_write_gcov_count_stream (ob->main_stream, profile_info->sum_all); + + /* Create and output a bitpack of non-zero histogram entries indices. */ + bp = bitpack_create (ob->main_stream); + for (h_ix = 0; h_ix < GCOV_HISTOGRAM_SIZE; h_ix++) + bp_pack_value (&bp, profile_info->histogram[h_ix].num_counters > 0, 1); + streamer_write_bitpack (&bp); + /* Now stream out only those non-zero entries. */ + for (h_ix = 0; h_ix < GCOV_HISTOGRAM_SIZE; h_ix++) + { + if (!profile_info->histogram[h_ix].num_counters) + continue; + streamer_write_gcov_count_stream (ob->main_stream, + profile_info->histogram[h_ix].num_counters); + streamer_write_gcov_count_stream (ob->main_stream, + profile_info->histogram[h_ix].min_value); + streamer_write_gcov_count_stream (ob->main_stream, + profile_info->histogram[h_ix].cum_value); + } + /* IPA-profile computes hot bb threshold based on cumulated + whole program profile. We need to stream it down to ltrans. */ + if (flag_wpa) + streamer_write_gcov_count_stream (ob->main_stream, + get_hot_bb_threshold ()); } else - lto_output_uleb128_stream (ob->main_stream, 0); -} - -/* Add NODE into encoder as well as nodes it is cloned from. - Do it in a way so clones appear first. */ - -static void -add_node_to (lto_cgraph_encoder_t encoder, struct cgraph_node *node, - bool include_body) -{ - if (node->clone_of) - add_node_to (encoder, node->clone_of, include_body); - else if (include_body) - lto_set_cgraph_encoder_encode_body (encoder, node); - lto_cgraph_encoder_encode (encoder, node); -} - -/* Add all references in LIST to encoders. */ - -static void -add_references (lto_cgraph_encoder_t encoder, - lto_varpool_encoder_t varpool_encoder, - struct ipa_ref_list *list) -{ - int i; - struct ipa_ref *ref; - for (i = 0; ipa_ref_list_reference_iterate (list, i, ref); i++) - if (ref->refered_type == IPA_REF_CGRAPH) - add_node_to (encoder, ipa_ref_node (ref), false); - else - { - struct varpool_node *vnode = ipa_ref_varpool_node (ref); - lto_varpool_encoder_encode (varpool_encoder, vnode); - } + streamer_write_uhwi_stream (ob->main_stream, 0); } /* Output all callees or indirect outgoing edges. EDGE must be the first such @@ -711,7 +745,7 @@ static void output_outgoing_cgraph_edges (struct cgraph_edge *edge, struct lto_simple_output_block *ob, - lto_cgraph_encoder_t encoder) + lto_symtab_encoder_t encoder) { if (!edge) return; @@ -727,170 +761,288 @@ /* Output the part of the cgraph in SET. */ static void -output_refs (cgraph_node_set set, varpool_node_set vset, - lto_cgraph_encoder_t encoder, - lto_varpool_encoder_t varpool_encoder) +output_refs (lto_symtab_encoder_t encoder) { - cgraph_node_set_iterator csi; - varpool_node_set_iterator vsi; struct lto_simple_output_block *ob; int count; struct ipa_ref *ref; - int i; ob = lto_create_simple_output_block (LTO_section_refs); - for (csi = csi_start (set); !csi_end_p (csi); csi_next (&csi)) + for (int i = 0; i < lto_symtab_encoder_size (encoder); i++) { - struct cgraph_node *node = csi_node (csi); + symtab_node *node = lto_symtab_encoder_deref (encoder, i); - count = ipa_ref_list_nreferences (&node->ref_list); + /* IPA_REF_ALIAS and IPA_REF_CHKP references are always preserved + in the boundary. Alias node can't have other references and + can be always handled as if it's not in the boundary. */ + if (!node->alias && !lto_symtab_encoder_in_partition_p (encoder, node)) + { + cgraph_node *cnode = dyn_cast <cgraph_node *> (node); + /* Output IPA_REF_CHKP reference. */ + if (cnode + && cnode->instrumented_version + && !cnode->instrumentation_clone) + { + for (int i = 0; node->iterate_reference (i, ref); i++) + if (ref->use == IPA_REF_CHKP) + { + if (lto_symtab_encoder_lookup (encoder, ref->referred) + != LCC_NOT_FOUND) + { + int nref = lto_symtab_encoder_lookup (encoder, node); + streamer_write_gcov_count_stream (ob->main_stream, 1); + streamer_write_uhwi_stream (ob->main_stream, nref); + lto_output_ref (ob, ref, encoder); + } + break; + } + } + continue; + } + + count = node->ref_list.nreferences (); if (count) { - lto_output_uleb128_stream (ob->main_stream, count); - lto_output_uleb128_stream (ob->main_stream, - lto_cgraph_encoder_lookup (encoder, node)); - for (i = 0; ipa_ref_list_reference_iterate (&node->ref_list, i, ref); i++) - lto_output_ref (ob, ref, encoder, varpool_encoder); + streamer_write_gcov_count_stream (ob->main_stream, count); + streamer_write_uhwi_stream (ob->main_stream, + lto_symtab_encoder_lookup (encoder, node)); + for (int i = 0; node->iterate_reference (i, ref); i++) + lto_output_ref (ob, ref, encoder); } } - lto_output_uleb128_stream (ob->main_stream, 0); - - for (vsi = vsi_start (vset); !vsi_end_p (vsi); vsi_next (&vsi)) - { - struct varpool_node *node = vsi_node (vsi); - - count = ipa_ref_list_nreferences (&node->ref_list); - if (count) - { - lto_output_uleb128_stream (ob->main_stream, count); - lto_output_uleb128_stream (ob->main_stream, - lto_varpool_encoder_lookup (varpool_encoder, - node)); - for (i = 0; ipa_ref_list_reference_iterate (&node->ref_list, i, ref); i++) - lto_output_ref (ob, ref, encoder, varpool_encoder); - } - } - - lto_output_uleb128_stream (ob->main_stream, 0); + streamer_write_uhwi_stream (ob->main_stream, 0); lto_destroy_simple_output_block (ob); } -/* Find out all cgraph and varpool nodes we want to encode in current unit - and insert them to encoders. */ +/* Add NODE into encoder as well as nodes it is cloned from. + Do it in a way so clones appear first. */ + +static void +add_node_to (lto_symtab_encoder_t encoder, struct cgraph_node *node, + bool include_body) +{ + if (node->clone_of) + add_node_to (encoder, node->clone_of, include_body); + else if (include_body) + lto_set_symtab_encoder_encode_body (encoder, node); + lto_symtab_encoder_encode (encoder, node); +} + +/* Add all references in NODE to encoders. */ + +static void +create_references (lto_symtab_encoder_t encoder, symtab_node *node) +{ + int i; + struct ipa_ref *ref = NULL; + for (i = 0; node->iterate_reference (i, ref); i++) + if (is_a <cgraph_node *> (ref->referred)) + add_node_to (encoder, dyn_cast <cgraph_node *> (ref->referred), false); + else + lto_symtab_encoder_encode (encoder, ref->referred); +} + +/* Select what needs to be streamed out. In regular lto mode stream everything. + In offload lto mode stream only nodes marked as offloadable. */ void -compute_ltrans_boundary (struct lto_out_decl_state *state, - cgraph_node_set set, varpool_node_set vset) +select_what_to_stream (void) { - struct cgraph_node *node; - cgraph_node_set_iterator csi; - varpool_node_set_iterator vsi; + struct symtab_node *snode; + FOR_EACH_SYMBOL (snode) + snode->need_lto_streaming = !lto_stream_offload_p || snode->offloadable; +} + +/* Find all symbols we want to stream into given partition and insert them + to encoders. + + The function actually replaces IN_ENCODER by new one. The reason is that + streaming code needs clone's origin to be streamed before clone. This + means that we need to insert the nodes in specific order. This order is + ignored by the partitioning logic earlier. */ + +lto_symtab_encoder_t +compute_ltrans_boundary (lto_symtab_encoder_t in_encoder) +{ struct cgraph_edge *edge; int i; - lto_cgraph_encoder_t encoder; - lto_varpool_encoder_t varpool_encoder; + lto_symtab_encoder_t encoder; + lto_symtab_encoder_iterator lsei; + hash_set<void *> reachable_call_targets; + + encoder = lto_symtab_encoder_new (false); - encoder = state->cgraph_node_encoder = lto_cgraph_encoder_new (); - varpool_encoder = state->varpool_node_encoder = lto_varpool_encoder_new (); - - /* Go over all the nodes in SET and assign references. */ - for (csi = csi_start (set); !csi_end_p (csi); csi_next (&csi)) + /* Go over all entries in the IN_ENCODER and duplicate them to + ENCODER. At the same time insert masters of clones so + every master appears before clone. */ + for (lsei = lsei_start_function_in_partition (in_encoder); + !lsei_end_p (lsei); lsei_next_function_in_partition (&lsei)) { - node = csi_node (csi); + struct cgraph_node *node = lsei_cgraph_node (lsei); + if (!node->need_lto_streaming) + continue; add_node_to (encoder, node, true); - add_references (encoder, varpool_encoder, &node->ref_list); + lto_set_symtab_encoder_in_partition (encoder, node); + create_references (encoder, node); } - for (vsi = vsi_start (vset); !vsi_end_p (vsi); vsi_next (&vsi)) + for (lsei = lsei_start_variable_in_partition (in_encoder); + !lsei_end_p (lsei); lsei_next_variable_in_partition (&lsei)) { - struct varpool_node *vnode = vsi_node (vsi); - gcc_assert (!vnode->alias); - lto_varpool_encoder_encode (varpool_encoder, vnode); - lto_set_varpool_encoder_encode_initializer (varpool_encoder, vnode); - add_references (encoder, varpool_encoder, &vnode->ref_list); + varpool_node *vnode = lsei_varpool_node (lsei); + + if (!vnode->need_lto_streaming) + continue; + lto_set_symtab_encoder_in_partition (encoder, vnode); + lto_set_symtab_encoder_encode_initializer (encoder, vnode); + create_references (encoder, vnode); } /* Pickle in also the initializer of all referenced readonly variables to help folding. Constant pool variables are not shared, so we must pickle those too. */ - for (i = 0; i < lto_varpool_encoder_size (varpool_encoder); i++) + for (i = 0; i < lto_symtab_encoder_size (encoder); i++) { - struct varpool_node *vnode = lto_varpool_encoder_deref (varpool_encoder, i); - if (DECL_INITIAL (vnode->decl) - && !lto_varpool_encoder_encode_initializer_p (varpool_encoder, - vnode) - && const_value_known_p (vnode->decl)) + symtab_node *node = lto_symtab_encoder_deref (encoder, i); + if (varpool_node *vnode = dyn_cast <varpool_node *> (node)) { - lto_set_varpool_encoder_encode_initializer (varpool_encoder, vnode); - add_references (encoder, varpool_encoder, &vnode->ref_list); - } + if (!lto_symtab_encoder_encode_initializer_p (encoder, + vnode) + && (((vnode->ctor_useable_for_folding_p () + && (!DECL_VIRTUAL_P (vnode->decl) + || !flag_wpa + || flag_ltrans_devirtualize)) + || POINTER_BOUNDS_P (vnode->decl)))) + { + lto_set_symtab_encoder_encode_initializer (encoder, vnode); + create_references (encoder, vnode); + } + } } /* Go over all the nodes again to include callees that are not in SET. */ - for (csi = csi_start (set); !csi_end_p (csi); csi_next (&csi)) + for (lsei = lsei_start_function_in_partition (encoder); + !lsei_end_p (lsei); lsei_next_function_in_partition (&lsei)) { - node = csi_node (csi); + struct cgraph_node *node = lsei_cgraph_node (lsei); for (edge = node->callees; edge; edge = edge->next_callee) { struct cgraph_node *callee = edge->callee; - if (!cgraph_node_in_set_p (callee, set)) + if (!lto_symtab_encoder_in_partition_p (encoder, callee)) { /* We should have moved all the inlines. */ gcc_assert (!callee->global.inlined_to); add_node_to (encoder, callee, false); } } + /* Add all possible targets for late devirtualization. */ + if (flag_ltrans_devirtualize || !flag_wpa) + for (edge = node->indirect_calls; edge; edge = edge->next_callee) + if (edge->indirect_info->polymorphic) + { + unsigned int i; + void *cache_token; + bool final; + vec <cgraph_node *>targets + = possible_polymorphic_call_targets + (edge, &final, &cache_token); + if (!reachable_call_targets.add (cache_token)) + { + for (i = 0; i < targets.length (); i++) + { + struct cgraph_node *callee = targets[i]; + + /* Adding an external declarations into the unit serves + no purpose and just increases its boundary. */ + if (callee->definition + && !lto_symtab_encoder_in_partition_p + (encoder, callee)) + { + gcc_assert (!callee->global.inlined_to); + add_node_to (encoder, callee, false); + } + } + } + } } + /* Be sure to also insert alias targert and thunk callees. These needs + to stay to aid local calling conventions. */ + for (i = 0; i < lto_symtab_encoder_size (encoder); i++) + { + symtab_node *node = lto_symtab_encoder_deref (encoder, i); + cgraph_node *cnode = dyn_cast <cgraph_node *> (node); + + if (node->alias && node->analyzed) + create_references (encoder, node); + if (cnode + && cnode->thunk.thunk_p && !cnode->global.inlined_to) + add_node_to (encoder, cnode->callees->callee, false); + while (node->transparent_alias && node->analyzed) + { + node = node->get_alias_target (); + if (is_a <cgraph_node *> (node)) + add_node_to (encoder, dyn_cast <cgraph_node *> (node), + false); + else + lto_symtab_encoder_encode (encoder, node); + } + } + lto_symtab_encoder_delete (in_encoder); + return encoder; } -/* Output the part of the cgraph in SET. */ +/* Output the part of the symtab in SET and VSET. */ void -output_cgraph (cgraph_node_set set, varpool_node_set vset) +output_symtab (void) { struct cgraph_node *node; struct lto_simple_output_block *ob; - cgraph_node_set_iterator csi; int i, n_nodes; - lto_cgraph_encoder_t encoder; - lto_varpool_encoder_t varpool_encoder; - struct cgraph_asm_node *can; - static bool asm_nodes_output = false; + lto_symtab_encoder_t encoder; if (flag_wpa) - output_cgraph_opt_summary (set); + output_cgraph_opt_summary (); - ob = lto_create_simple_output_block (LTO_section_cgraph); + ob = lto_create_simple_output_block (LTO_section_symtab_nodes); output_profile_summary (ob); /* An encoder for cgraph nodes should have been created by ipa_write_summaries_1. */ - gcc_assert (ob->decl_state->cgraph_node_encoder); - gcc_assert (ob->decl_state->varpool_node_encoder); - encoder = ob->decl_state->cgraph_node_encoder; - varpool_encoder = ob->decl_state->varpool_node_encoder; + gcc_assert (ob->decl_state->symtab_node_encoder); + encoder = ob->decl_state->symtab_node_encoder; /* Write out the nodes. We must first output a node and then its clones, otherwise at a time reading back the node there would be nothing to clone from. */ - n_nodes = lto_cgraph_encoder_size (encoder); + n_nodes = lto_symtab_encoder_size (encoder); for (i = 0; i < n_nodes; i++) { - node = lto_cgraph_encoder_deref (encoder, i); - lto_output_node (ob, node, encoder, set, vset); + symtab_node *node = lto_symtab_encoder_deref (encoder, i); + if (cgraph_node *cnode = dyn_cast <cgraph_node *> (node)) + lto_output_node (ob, cnode, encoder); + else + lto_output_varpool_node (ob, dyn_cast<varpool_node *> (node), encoder); } /* Go over the nodes in SET again to write edges. */ - for (csi = csi_start (set); !csi_end_p (csi); csi_next (&csi)) + for (int i = 0; i < lto_symtab_encoder_size (encoder); i++) { - node = csi_node (csi); - output_outgoing_cgraph_edges (node->callees, ob, encoder); - output_outgoing_cgraph_edges (node->indirect_calls, ob, encoder); + node = dyn_cast <cgraph_node *> (lto_symtab_encoder_deref (encoder, i)); + if (node + && ((node->thunk.thunk_p && !node->global.inlined_to) + || lto_symtab_encoder_in_partition_p (encoder, node))) + { + output_outgoing_cgraph_edges (node->callees, ob, encoder); + output_outgoing_cgraph_edges (node->indirect_calls, ob, encoder); + } } - lto_output_uleb128_stream (ob->main_stream, 0); + streamer_write_uhwi_stream (ob->main_stream, 0); + + lto_destroy_simple_output_block (ob); /* Emit toplevel asms. When doing WPA we must output every asm just once. Since we do not partition asm @@ -899,21 +1051,94 @@ if (!asm_nodes_output) { asm_nodes_output = true; - for (can = cgraph_asm_nodes; can; can = can->next) - { - int len = TREE_STRING_LENGTH (can->asm_str); - lto_output_uleb128_stream (ob->main_stream, len); - for (i = 0; i < len; ++i) - lto_output_1_stream (ob->main_stream, - TREE_STRING_POINTER (can->asm_str)[i]); - } + lto_output_toplevel_asms (); } - lto_output_uleb128_stream (ob->main_stream, 0); + output_refs (encoder); +} + +/* Return identifier encoded in IB as a plain string. */ + +static tree +read_identifier (struct lto_input_block *ib) +{ + unsigned int len = strnlen (ib->data + ib->p, ib->len - ib->p - 1); + tree id; + + if (ib->data[ib->p + len]) + lto_section_overrun (ib); + if (!len) + { + ib->p++; + return NULL; + } + id = get_identifier (ib->data + ib->p); + ib->p += len + 1; + return id; +} + +/* Return string encoded in IB, NULL if string is empty. */ + +static const char * +read_string (struct lto_input_block *ib) +{ + unsigned int len = strnlen (ib->data + ib->p, ib->len - ib->p - 1); + const char *str; + if (ib->data[ib->p + len]) + lto_section_overrun (ib); + if (!len) + { + ib->p++; + return NULL; + } + str = ib->data + ib->p; + ib->p += len + 1; + return str; +} + +/* Output function/variable tables that will allow libgomp to look up offload + target code. + OFFLOAD_FUNCS is filled in expand_omp_target, OFFLOAD_VARS is filled in + varpool_node::get_create. In WHOPR (partitioned) mode during the WPA stage + both OFFLOAD_FUNCS and OFFLOAD_VARS are filled by input_offload_tables. */ + +void +output_offload_tables (void) +{ + if (vec_safe_is_empty (offload_funcs) && vec_safe_is_empty (offload_vars)) + return; + + struct lto_simple_output_block *ob + = lto_create_simple_output_block (LTO_section_offload_table); + + for (unsigned i = 0; i < vec_safe_length (offload_funcs); i++) + { + streamer_write_enum (ob->main_stream, LTO_symtab_tags, + LTO_symtab_last_tag, LTO_symtab_unavail_node); + lto_output_fn_decl_index (ob->decl_state, ob->main_stream, + (*offload_funcs)[i]); + } + + for (unsigned i = 0; i < vec_safe_length (offload_vars); i++) + { + streamer_write_enum (ob->main_stream, LTO_symtab_tags, + LTO_symtab_last_tag, LTO_symtab_variable); + lto_output_var_decl_index (ob->decl_state, ob->main_stream, + (*offload_vars)[i]); + } + + streamer_write_uhwi_stream (ob->main_stream, 0); lto_destroy_simple_output_block (ob); - output_varpool (set, vset); - output_refs (set, vset, encoder, varpool_encoder); + + /* In WHOPR mode during the WPA stage the joint offload tables need to be + streamed to one partition only. That's why we free offload_funcs and + offload_vars after the first call of output_offload_tables. */ + if (flag_wpa) + { + vec_free (offload_funcs); + vec_free (offload_vars); + } } /* Overwrite the information in NODE based on FILE_DATA, TAG, FLAGS, @@ -926,42 +1151,28 @@ static void input_overwrite_node (struct lto_file_decl_data *file_data, struct cgraph_node *node, - enum LTO_cgraph_tags tag, - struct bitpack_d *bp, - unsigned int stack_size, - unsigned int self_time, - unsigned int time_inlining_benefit, - unsigned int self_size, - unsigned int size_inlining_benefit, - enum ld_plugin_symbol_resolution resolution) + enum LTO_symtab_tags tag, + struct bitpack_d *bp) { node->aux = (void *) tag; - node->local.inline_summary.estimated_self_stack_size = stack_size; - node->local.inline_summary.self_time = self_time; - node->local.inline_summary.time_inlining_benefit = time_inlining_benefit; - node->local.inline_summary.self_size = self_size; - node->local.inline_summary.size_inlining_benefit = size_inlining_benefit; - node->global.time = self_time; - node->global.size = self_size; - node->global.estimated_stack_size = stack_size; - node->global.estimated_growth = INT_MIN; - node->local.lto_file_data = file_data; + node->lto_file_data = file_data; node->local.local = bp_unpack_value (bp, 1); - node->local.externally_visible = bp_unpack_value (bp, 1); - node->local.finalized = bp_unpack_value (bp, 1); - node->local.inlinable = bp_unpack_value (bp, 1); + node->externally_visible = bp_unpack_value (bp, 1); + node->no_reorder = bp_unpack_value (bp, 1); + node->definition = bp_unpack_value (bp, 1); node->local.versionable = bp_unpack_value (bp, 1); node->local.can_change_signature = bp_unpack_value (bp, 1); - node->local.disregard_inline_limits = bp_unpack_value (bp, 1); node->local.redefined_extern_inline = bp_unpack_value (bp, 1); - node->local.vtable_method = bp_unpack_value (bp, 1); - node->needed = bp_unpack_value (bp, 1); + node->force_output = bp_unpack_value (bp, 1); + node->forced_by_abi = bp_unpack_value (bp, 1); + node->unique_name = bp_unpack_value (bp, 1); + node->body_removed = bp_unpack_value (bp, 1); + node->implicit_section = bp_unpack_value (bp, 1); node->address_taken = bp_unpack_value (bp, 1); - node->abstract_and_needed = bp_unpack_value (bp, 1); - node->reachable_from_other_partition = bp_unpack_value (bp, 1); + node->used_from_other_partition = bp_unpack_value (bp, 1); node->lowered = bp_unpack_value (bp, 1); - node->analyzed = tag == LTO_cgraph_analyzed_node; + node->analyzed = tag == LTO_symtab_analyzed_node; node->in_other_partition = bp_unpack_value (bp, 1); if (node->in_other_partition /* Avoid updating decl when we are seeing just inline clone. @@ -978,35 +1189,34 @@ TREE_STATIC (node->decl) = 0; } node->alias = bp_unpack_value (bp, 1); - node->finalized_by_frontend = bp_unpack_value (bp, 1); + node->transparent_alias = bp_unpack_value (bp, 1); + node->weakref = bp_unpack_value (bp, 1); node->frequency = (enum node_frequency)bp_unpack_value (bp, 2); node->only_called_at_startup = bp_unpack_value (bp, 1); node->only_called_at_exit = bp_unpack_value (bp, 1); - node->resolution = resolution; + node->tm_clone = bp_unpack_value (bp, 1); + node->calls_comdat_local = bp_unpack_value (bp, 1); + node->icf_merged = bp_unpack_value (bp, 1); + node->nonfreeing_fn = bp_unpack_value (bp, 1); + node->thunk.thunk_p = bp_unpack_value (bp, 1); + node->parallelized_function = bp_unpack_value (bp, 1); + node->resolution = bp_unpack_enum (bp, ld_plugin_symbol_resolution, + LDPR_NUM_KNOWN); + node->instrumentation_clone = bp_unpack_value (bp, 1); + node->split_part = bp_unpack_value (bp, 1); + gcc_assert (flag_ltrans + || (!node->in_other_partition + && !node->used_from_other_partition)); } -/* Output the part of the cgraph in SET. */ - -static void -output_varpool (cgraph_node_set set, varpool_node_set vset) -{ - struct lto_simple_output_block *ob = lto_create_simple_output_block (LTO_section_varpool); - lto_varpool_encoder_t varpool_encoder = ob->decl_state->varpool_node_encoder; - int len = lto_varpool_encoder_size (varpool_encoder), i; - - lto_output_uleb128_stream (ob->main_stream, len); +/* Return string alias is alias of. */ - /* Write out the nodes. We must first output a node and then its clones, - otherwise at a time reading back the node there would be nothing to clone - from. */ - for (i = 0; i < len; i++) - { - lto_output_varpool_node (ob, lto_varpool_encoder_deref (varpool_encoder, i), - varpool_encoder, - set, vset); - } - - lto_destroy_simple_output_block (ob); +static tree +get_alias_symbol (tree decl) +{ + tree alias = lookup_attribute ("alias", DECL_ATTRIBUTES (decl)); + return get_identifier (TREE_STRING_POINTER + (TREE_VALUE (TREE_VALUE (alias)))); } /* Read a node from input_block IB. TAG is the node's tag just read. @@ -1015,132 +1225,173 @@ static struct cgraph_node * input_node (struct lto_file_decl_data *file_data, struct lto_input_block *ib, - enum LTO_cgraph_tags tag, - VEC(cgraph_node_ptr, heap) *nodes) + enum LTO_symtab_tags tag, + vec<symtab_node *> nodes) { + gcc::pass_manager *passes = g->get_passes (); tree fn_decl; struct cgraph_node *node; struct bitpack_d bp; - int stack_size = 0; unsigned decl_index; int ref = LCC_NOT_FOUND, ref2 = LCC_NOT_FOUND; - int self_time = 0; - int self_size = 0; - int time_inlining_benefit = 0; - int size_inlining_benefit = 0; - unsigned long same_body_count = 0; int clone_ref; - enum ld_plugin_symbol_resolution resolution; + int order; + int i, count; + tree group; + const char *section; + order = streamer_read_hwi (ib) + order_base; + clone_ref = streamer_read_hwi (ib); - clone_ref = lto_input_sleb128 (ib); - - decl_index = lto_input_uleb128 (ib); + decl_index = streamer_read_uhwi (ib); fn_decl = lto_file_decl_data_get_fn_decl (file_data, decl_index); if (clone_ref != LCC_NOT_FOUND) { - node = cgraph_clone_node (VEC_index (cgraph_node_ptr, nodes, clone_ref), fn_decl, - 0, CGRAPH_FREQ_BASE, 0, false, NULL); + node = dyn_cast<cgraph_node *> (nodes[clone_ref])->create_clone (fn_decl, + profile_count::uninitialized (), CGRAPH_FREQ_BASE, false, + vNULL, false, NULL, NULL); } else - node = cgraph_node (fn_decl); - - node->count = lto_input_sleb128 (ib); - node->count_materialization_scale = lto_input_sleb128 (ib); - - if (tag == LTO_cgraph_analyzed_node) { - stack_size = lto_input_sleb128 (ib); - self_size = lto_input_sleb128 (ib); - size_inlining_benefit = lto_input_sleb128 (ib); - self_time = lto_input_sleb128 (ib); - time_inlining_benefit = lto_input_sleb128 (ib); - - ref = lto_input_sleb128 (ib); + /* Declaration of functions can be already merged with a declaration + from other input file. We keep cgraph unmerged until after streaming + of ipa passes is done. Alays forcingly create a fresh node. */ + node = symtab->create_empty (); + node->decl = fn_decl; + node->register_symbol (); } - ref2 = lto_input_sleb128 (ib); + node->order = order; + if (order >= symtab->order) + symtab->order = order + 1; + + node->count = profile_count::stream_in (ib); + node->count_materialization_scale = streamer_read_hwi (ib); + + count = streamer_read_hwi (ib); + node->ipa_transforms_to_apply = vNULL; + for (i = 0; i < count; i++) + { + opt_pass *pass; + int pid = streamer_read_hwi (ib); + + gcc_assert (pid < passes->passes_by_id_size); + pass = passes->passes_by_id[pid]; + node->ipa_transforms_to_apply.safe_push ((ipa_opt_pass_d *) pass); + } + + if (tag == LTO_symtab_analyzed_node) + ref = streamer_read_hwi (ib); + + group = read_identifier (ib); + if (group) + ref2 = streamer_read_hwi (ib); /* Make sure that we have not read this node before. Nodes that have already been read will have their tag stored in the 'aux' field. Since built-in functions can be referenced in multiple functions, they are expected to be read more than once. */ - if (node->aux && !DECL_IS_BUILTIN (node->decl)) + if (node->aux && !DECL_BUILT_IN (node->decl)) internal_error ("bytecode stream: found multiple instances of cgraph " - "node %d", node->uid); + "node with uid %d", node->uid); - bp = lto_input_bitpack (ib); - resolution = (enum ld_plugin_symbol_resolution)lto_input_uleb128 (ib); - input_overwrite_node (file_data, node, tag, &bp, stack_size, self_time, - time_inlining_benefit, self_size, - size_inlining_benefit, resolution); + node->tp_first_run = streamer_read_uhwi (ib); - /* Store a reference for now, and fix up later to be a pointer. */ - node->global.inlined_to = (cgraph_node_ptr) (intptr_t) ref; + bp = streamer_read_bitpack (ib); + + input_overwrite_node (file_data, node, tag, &bp); /* Store a reference for now, and fix up later to be a pointer. */ - node->same_comdat_group = (cgraph_node_ptr) (intptr_t) ref2; + node->global.inlined_to = (cgraph_node *) (intptr_t) ref; - same_body_count = lto_input_uleb128 (ib); - while (same_body_count-- > 0) + if (group) + { + node->set_comdat_group (group); + /* Store a reference for now, and fix up later to be a pointer. */ + node->same_comdat_group = (symtab_node *) (intptr_t) ref2; + } + else + node->same_comdat_group = (symtab_node *) (intptr_t) LCC_NOT_FOUND; + section = read_string (ib); + if (section) + node->set_section_for_node (section); + + if (node->thunk.thunk_p) { - tree alias_decl; - int type; - struct cgraph_node *alias; - decl_index = lto_input_uleb128 (ib); - alias_decl = lto_file_decl_data_get_fn_decl (file_data, decl_index); - type = lto_input_uleb128 (ib); - if (!type) - { - tree real_alias; - decl_index = lto_input_uleb128 (ib); - real_alias = lto_file_decl_data_get_fn_decl (file_data, decl_index); - alias = cgraph_same_body_alias (node, alias_decl, real_alias); - } - else - { - HOST_WIDE_INT fixed_offset = lto_input_uleb128 (ib); - HOST_WIDE_INT virtual_value = lto_input_uleb128 (ib); - tree real_alias; - decl_index = lto_input_uleb128 (ib); - real_alias = lto_file_decl_data_get_fn_decl (file_data, decl_index); - alias = cgraph_add_thunk (node, alias_decl, fn_decl, type & 2, fixed_offset, - virtual_value, - (type & 4) ? size_int (virtual_value) : NULL_TREE, - real_alias); - } - gcc_assert (alias); - alias->resolution = (enum ld_plugin_symbol_resolution)lto_input_uleb128 (ib); + int type = streamer_read_uhwi (ib); + HOST_WIDE_INT fixed_offset = streamer_read_uhwi (ib); + HOST_WIDE_INT virtual_value = streamer_read_uhwi (ib); + + node->thunk.fixed_offset = fixed_offset; + node->thunk.this_adjusting = (type & 2); + node->thunk.virtual_value = virtual_value; + node->thunk.virtual_offset_p = (type & 4); + node->thunk.add_pointer_bounds_args = (type & 8); } + if (node->alias && !node->analyzed && node->weakref) + node->alias_target = get_alias_symbol (node->decl); + node->profile_id = streamer_read_hwi (ib); + if (DECL_STATIC_CONSTRUCTOR (node->decl)) + node->set_init_priority (streamer_read_hwi (ib)); + if (DECL_STATIC_DESTRUCTOR (node->decl)) + node->set_fini_priority (streamer_read_hwi (ib)); + + if (node->instrumentation_clone) + { + decl_index = streamer_read_uhwi (ib); + fn_decl = lto_file_decl_data_get_fn_decl (file_data, decl_index); + node->orig_decl = fn_decl; + } + return node; } /* Read a node from input_block IB. TAG is the node's tag just read. Return the node read or overwriten. */ -static struct varpool_node * +static varpool_node * input_varpool_node (struct lto_file_decl_data *file_data, struct lto_input_block *ib) { int decl_index; tree var_decl; - struct varpool_node *node; + varpool_node *node; struct bitpack_d bp; - bool aliases_p; - int count; int ref = LCC_NOT_FOUND; + int order; + tree group; + const char *section; - decl_index = lto_input_uleb128 (ib); + order = streamer_read_hwi (ib) + order_base; + decl_index = streamer_read_uhwi (ib); var_decl = lto_file_decl_data_get_var_decl (file_data, decl_index); - node = varpool_node (var_decl); + + /* Declaration of functions can be already merged with a declaration + from other input file. We keep cgraph unmerged until after streaming + of ipa passes is done. Alays forcingly create a fresh node. */ + node = varpool_node::create_empty (); + node->decl = var_decl; + node->register_symbol (); + + node->order = order; + if (order >= symtab->order) + symtab->order = order + 1; node->lto_file_data = file_data; - bp = lto_input_bitpack (ib); + bp = streamer_read_bitpack (ib); node->externally_visible = bp_unpack_value (&bp, 1); + node->no_reorder = bp_unpack_value (&bp, 1); node->force_output = bp_unpack_value (&bp, 1); - node->finalized = bp_unpack_value (&bp, 1); + node->forced_by_abi = bp_unpack_value (&bp, 1); + node->unique_name = bp_unpack_value (&bp, 1); + node->body_removed = bp_unpack_value (&bp, 1); + node->implicit_section = bp_unpack_value (&bp, 1); + node->writeonly = bp_unpack_value (&bp, 1); + node->definition = bp_unpack_value (&bp, 1); node->alias = bp_unpack_value (&bp, 1); - node->analyzed = node->finalized; + node->transparent_alias = bp_unpack_value (&bp, 1); + node->weakref = bp_unpack_value (&bp, 1); + node->analyzed = bp_unpack_value (&bp, 1); node->used_from_other_partition = bp_unpack_value (&bp, 1); node->in_other_partition = bp_unpack_value (&bp, 1); if (node->in_other_partition) @@ -1148,25 +1399,31 @@ DECL_EXTERNAL (node->decl) = 1; TREE_STATIC (node->decl) = 0; } - aliases_p = bp_unpack_value (&bp, 1); - if (node->finalized) - varpool_mark_needed_node (node); - ref = lto_input_sleb128 (ib); - /* Store a reference for now, and fix up later to be a pointer. */ - node->same_comdat_group = (struct varpool_node *) (intptr_t) ref; - node->resolution = (enum ld_plugin_symbol_resolution)lto_input_uleb128 (ib); - if (aliases_p) + if (node->alias && !node->analyzed && node->weakref) + node->alias_target = get_alias_symbol (node->decl); + node->tls_model = (enum tls_model)bp_unpack_value (&bp, 3); + node->used_by_single_function = (enum tls_model)bp_unpack_value (&bp, 1); + node->dynamically_initialized = bp_unpack_value (&bp, 1); + node->need_bounds_init = bp_unpack_value (&bp, 1); + group = read_identifier (ib); + if (group) { - count = lto_input_uleb128 (ib); - for (; count > 0; count --) - { - tree decl = lto_file_decl_data_get_var_decl (file_data, - lto_input_uleb128 (ib)); - struct varpool_node *alias; - alias = varpool_extra_name_alias (decl, var_decl); - alias->resolution = (enum ld_plugin_symbol_resolution)lto_input_uleb128 (ib); - } + node->set_comdat_group (group); + ref = streamer_read_hwi (ib); + /* Store a reference for now, and fix up later to be a pointer. */ + node->same_comdat_group = (symtab_node *) (intptr_t) ref; } + else + node->same_comdat_group = (symtab_node *) (intptr_t) LCC_NOT_FOUND; + section = read_string (ib); + if (section) + node->set_section_for_node (section); + node->resolution = streamer_read_enum (ib, ld_plugin_symbol_resolution, + LDPR_NUM_KNOWN); + gcc_assert (flag_ltrans + || (!node->in_other_partition + && !node->used_from_other_partition)); + return node; } @@ -1175,26 +1432,23 @@ static void input_ref (struct lto_input_block *ib, - struct cgraph_node *refering_node, - struct varpool_node *refering_varpool_node, - VEC(cgraph_node_ptr, heap) *nodes, - VEC(varpool_node_ptr, heap) *varpool_nodes) + symtab_node *referring_node, + vec<symtab_node *> nodes) { - struct cgraph_node *node = NULL; - struct varpool_node *varpool_node = NULL; + symtab_node *node = NULL; struct bitpack_d bp; - enum ipa_ref_type type; enum ipa_ref_use use; + bool speculative; + struct ipa_ref *ref; - bp = lto_input_bitpack (ib); - type = (enum ipa_ref_type) bp_unpack_value (&bp, 1); - use = (enum ipa_ref_use) bp_unpack_value (&bp, 2); - if (type == IPA_REF_CGRAPH) - node = VEC_index (cgraph_node_ptr, nodes, lto_input_sleb128 (ib)); - else - varpool_node = VEC_index (varpool_node_ptr, varpool_nodes, lto_input_sleb128 (ib)); - ipa_record_reference (refering_node, refering_varpool_node, - node, varpool_node, use, NULL); + bp = streamer_read_bitpack (ib); + use = (enum ipa_ref_use) bp_unpack_value (&bp, 3); + speculative = (enum ipa_ref_use) bp_unpack_value (&bp, 1); + node = nodes[streamer_read_hwi (ib)]; + ref = referring_node->create_reference (node, use); + ref->speculative = speculative; + if (is_a <cgraph_node *> (referring_node)) + ref->lto_stmt_uid = streamer_read_hwi (ib); } /* Read an edge from IB. NODES points to a vector of previously read nodes for @@ -1203,51 +1457,50 @@ indirect_unknown_callee set). */ static void -input_edge (struct lto_input_block *ib, VEC(cgraph_node_ptr, heap) *nodes, +input_edge (struct lto_input_block *ib, vec<symtab_node *> nodes, bool indirect) { struct cgraph_node *caller, *callee; struct cgraph_edge *edge; unsigned int stmt_id; - gcov_type count; + profile_count count; int freq; - unsigned int nest; cgraph_inline_failed_t inline_failed; struct bitpack_d bp; int ecf_flags = 0; - caller = VEC_index (cgraph_node_ptr, nodes, lto_input_sleb128 (ib)); + caller = dyn_cast<cgraph_node *> (nodes[streamer_read_hwi (ib)]); if (caller == NULL || caller->decl == NULL_TREE) internal_error ("bytecode stream: no caller found while reading edge"); if (!indirect) { - callee = VEC_index (cgraph_node_ptr, nodes, lto_input_sleb128 (ib)); + callee = dyn_cast<cgraph_node *> (nodes[streamer_read_hwi (ib)]); if (callee == NULL || callee->decl == NULL_TREE) internal_error ("bytecode stream: no callee found while reading edge"); } else callee = NULL; - count = (gcov_type) lto_input_sleb128 (ib); + count = profile_count::stream_in (ib); - bp = lto_input_bitpack (ib); - stmt_id = (unsigned int) bp_unpack_value (&bp, HOST_BITS_PER_INT); - inline_failed = (cgraph_inline_failed_t) bp_unpack_value (&bp, - HOST_BITS_PER_INT); - freq = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT); - nest = (unsigned) bp_unpack_value (&bp, 30); + bp = streamer_read_bitpack (ib); + inline_failed = bp_unpack_enum (&bp, cgraph_inline_failed_t, CIF_N_REASONS); + stmt_id = bp_unpack_var_len_unsigned (&bp); + freq = (int) bp_unpack_var_len_unsigned (&bp); if (indirect) - edge = cgraph_create_indirect_edge (caller, NULL, 0, count, freq, nest); + edge = caller->create_indirect_edge (NULL, 0, count, freq); else - edge = cgraph_create_edge (caller, callee, NULL, count, freq, nest); + edge = caller->create_edge (callee, NULL, count, freq); edge->indirect_inlining_edge = bp_unpack_value (&bp, 1); + edge->speculative = bp_unpack_value (&bp, 1); edge->lto_stmt_uid = stmt_id; edge->inline_failed = inline_failed; edge->call_stmt_cannot_inline_p = bp_unpack_value (&bp, 1); edge->can_throw_external = bp_unpack_value (&bp, 1); + edge->in_polymorphic_cdtor = bp_unpack_value (&bp, 1); if (indirect) { if (bp_unpack_value (&bp, 1)) @@ -1263,160 +1516,139 @@ if (bp_unpack_value (&bp, 1)) ecf_flags |= ECF_RETURNS_TWICE; edge->indirect_info->ecf_flags = ecf_flags; + edge->indirect_info->common_target_id = streamer_read_hwi (ib); + if (edge->indirect_info->common_target_id) + edge->indirect_info->common_target_probability = streamer_read_hwi (ib); } } /* Read a cgraph from IB using the info in FILE_DATA. */ -static VEC(cgraph_node_ptr, heap) * +static vec<symtab_node *> input_cgraph_1 (struct lto_file_decl_data *file_data, struct lto_input_block *ib) { - enum LTO_cgraph_tags tag; - VEC(cgraph_node_ptr, heap) *nodes = NULL; - struct cgraph_node *node; + enum LTO_symtab_tags tag; + vec<symtab_node *> nodes = vNULL; + symtab_node *node; unsigned i; - unsigned HOST_WIDE_INT len; - tag = (enum LTO_cgraph_tags) lto_input_uleb128 (ib); + tag = streamer_read_enum (ib, LTO_symtab_tags, LTO_symtab_last_tag); + order_base = symtab->order; while (tag) { - if (tag == LTO_cgraph_edge) + if (tag == LTO_symtab_edge) input_edge (ib, nodes, false); - else if (tag == LTO_cgraph_indirect_edge) + else if (tag == LTO_symtab_indirect_edge) input_edge (ib, nodes, true); + else if (tag == LTO_symtab_variable) + { + node = input_varpool_node (file_data, ib); + nodes.safe_push (node); + lto_symtab_encoder_encode (file_data->symtab_node_encoder, node); + } else { - node = input_node (file_data, ib, tag,nodes); + node = input_node (file_data, ib, tag, nodes); if (node == NULL || node->decl == NULL_TREE) internal_error ("bytecode stream: found empty cgraph node"); - VEC_safe_push (cgraph_node_ptr, heap, nodes, node); - lto_cgraph_encoder_encode (file_data->cgraph_node_encoder, node); + nodes.safe_push (node); + lto_symtab_encoder_encode (file_data->symtab_node_encoder, node); } - tag = (enum LTO_cgraph_tags) lto_input_uleb128 (ib); + tag = streamer_read_enum (ib, LTO_symtab_tags, LTO_symtab_last_tag); } - /* Input toplevel asms. */ - len = lto_input_uleb128 (ib); - while (len) + lto_input_toplevel_asms (file_data, order_base); + + /* AUX pointers should be all non-zero for function nodes read from the stream. */ + if (flag_checking) { - char *str = (char *)xmalloc (len + 1); - for (i = 0; i < len; ++i) - str[i] = lto_input_1_unsigned (ib); - cgraph_add_asm_node (build_string (len, str)); - free (str); - - len = lto_input_uleb128 (ib); + FOR_EACH_VEC_ELT (nodes, i, node) + gcc_assert (node->aux || !is_a <cgraph_node *> (node)); } - /* AUX pointers should be all non-zero for nodes read from the stream. */ -#ifdef ENABLE_CHECKING - FOR_EACH_VEC_ELT (cgraph_node_ptr, nodes, i, node) - gcc_assert (node->aux); -#endif - FOR_EACH_VEC_ELT (cgraph_node_ptr, nodes, i, node) + FOR_EACH_VEC_ELT (nodes, i, node) { - int ref = (int) (intptr_t) node->global.inlined_to; + int ref; + if (cgraph_node *cnode = dyn_cast <cgraph_node *> (node)) + { + ref = (int) (intptr_t) cnode->global.inlined_to; + + /* We share declaration of builtins, so we may read same node twice. */ + if (!node->aux) + continue; + node->aux = NULL; + + /* Fixup inlined_to from reference to pointer. */ + if (ref != LCC_NOT_FOUND) + dyn_cast<cgraph_node *> (node)->global.inlined_to + = dyn_cast<cgraph_node *> (nodes[ref]); + else + cnode->global.inlined_to = NULL; + + /* Compute instrumented_version. */ + if (cnode->instrumentation_clone) + { + gcc_assert (cnode->orig_decl); - /* We share declaration of builtins, so we may read same node twice. */ - if (!node->aux) - continue; - node->aux = NULL; + cnode->instrumented_version = cgraph_node::get (cnode->orig_decl); + if (cnode->instrumented_version) + { + /* We may have multiple nodes for a single function which + will be merged later. To have a proper merge we need + to keep instrumentation_version reference between nodes + consistent: each instrumented_version reference should + have proper reverse reference. Thus don't break existing + instrumented_version reference if it already exists. */ + if (cnode->instrumented_version->instrumented_version) + cnode->instrumented_version = NULL; + else + cnode->instrumented_version->instrumented_version = cnode; + } - /* Fixup inlined_to from reference to pointer. */ - if (ref != LCC_NOT_FOUND) - node->global.inlined_to = VEC_index (cgraph_node_ptr, nodes, ref); - else - node->global.inlined_to = NULL; + /* Restore decl names reference except for wrapper functions. */ + if (!chkp_wrap_function (cnode->orig_decl)) + { + tree name = DECL_ASSEMBLER_NAME (cnode->decl); + IDENTIFIER_TRANSPARENT_ALIAS (name) = 1; + TREE_CHAIN (name) = DECL_ASSEMBLER_NAME (cnode->orig_decl); + } + } + } ref = (int) (intptr_t) node->same_comdat_group; /* Fixup same_comdat_group from reference to pointer. */ if (ref != LCC_NOT_FOUND) - node->same_comdat_group = VEC_index (cgraph_node_ptr, nodes, ref); + node->same_comdat_group = nodes[ref]; else node->same_comdat_group = NULL; } - FOR_EACH_VEC_ELT (cgraph_node_ptr, nodes, i, node) - node->aux = (void *)1; + FOR_EACH_VEC_ELT (nodes, i, node) + node->aux = is_a <cgraph_node *> (node) ? (void *)1 : NULL; return nodes; } -/* Read a varpool from IB using the info in FILE_DATA. */ - -static VEC(varpool_node_ptr, heap) * -input_varpool_1 (struct lto_file_decl_data *file_data, - struct lto_input_block *ib) -{ - unsigned HOST_WIDE_INT len; - VEC(varpool_node_ptr, heap) *varpool = NULL; - int i; - struct varpool_node *node; - - len = lto_input_uleb128 (ib); - while (len) - { - VEC_safe_push (varpool_node_ptr, heap, varpool, - input_varpool_node (file_data, ib)); - len--; - } -#ifdef ENABLE_CHECKING - FOR_EACH_VEC_ELT (varpool_node_ptr, varpool, i, node) - gcc_assert (!node->aux); -#endif - FOR_EACH_VEC_ELT (varpool_node_ptr, varpool, i, node) - { - int ref = (int) (intptr_t) node->same_comdat_group; - /* We share declaration of builtins, so we may read same node twice. */ - if (node->aux) - continue; - node->aux = (void *)1; - - /* Fixup same_comdat_group from reference to pointer. */ - if (ref != LCC_NOT_FOUND) - node->same_comdat_group = VEC_index (varpool_node_ptr, varpool, ref); - else - node->same_comdat_group = NULL; - } - FOR_EACH_VEC_ELT (varpool_node_ptr, varpool, i, node) - node->aux = NULL; - return varpool; -} - /* Input ipa_refs. */ static void input_refs (struct lto_input_block *ib, - VEC(cgraph_node_ptr, heap) *nodes, - VEC(varpool_node_ptr, heap) *varpool) + vec<symtab_node *> nodes) { int count; int idx; while (true) { - struct cgraph_node *node; - count = lto_input_uleb128 (ib); + symtab_node *node; + count = streamer_read_uhwi (ib); if (!count) break; - idx = lto_input_uleb128 (ib); - node = VEC_index (cgraph_node_ptr, nodes, idx); + idx = streamer_read_uhwi (ib); + node = nodes[idx]; while (count) { - input_ref (ib, node, NULL, nodes, varpool); - count--; - } - } - while (true) - { - struct varpool_node *node; - count = lto_input_uleb128 (ib); - if (!count) - break; - node = VEC_index (varpool_node_ptr, varpool, lto_input_uleb128 (ib)); - while (count) - { - input_ref (ib, NULL, node, nodes, varpool); + input_ref (ib, node, nodes); count--; } } @@ -1430,11 +1662,42 @@ input_profile_summary (struct lto_input_block *ib, struct lto_file_decl_data *file_data) { - unsigned int runs = lto_input_uleb128 (ib); + unsigned h_ix; + struct bitpack_d bp; + unsigned int runs = streamer_read_uhwi (ib); if (runs) { file_data->profile_info.runs = runs; - file_data->profile_info.sum_max = lto_input_uleb128 (ib); + file_data->profile_info.sum_max = streamer_read_gcov_count (ib); + file_data->profile_info.sum_all = streamer_read_gcov_count (ib); + + memset (file_data->profile_info.histogram, 0, + sizeof (gcov_bucket_type) * GCOV_HISTOGRAM_SIZE); + /* Input the bitpack of non-zero histogram indices. */ + bp = streamer_read_bitpack (ib); + /* Read in and unpack the full bitpack, flagging non-zero + histogram entries by setting the num_counters non-zero. */ + for (h_ix = 0; h_ix < GCOV_HISTOGRAM_SIZE; h_ix++) + { + file_data->profile_info.histogram[h_ix].num_counters + = bp_unpack_value (&bp, 1); + } + for (h_ix = 0; h_ix < GCOV_HISTOGRAM_SIZE; h_ix++) + { + if (!file_data->profile_info.histogram[h_ix].num_counters) + continue; + + file_data->profile_info.histogram[h_ix].num_counters + = streamer_read_gcov_count (ib); + file_data->profile_info.histogram[h_ix].min_value + = streamer_read_gcov_count (ib); + file_data->profile_info.histogram[h_ix].cum_value + = streamer_read_gcov_count (ib); + } + /* IPA-profile computes hot bb threshold based on cumulated + whole program profile. We need to stream it down to ltrans. */ + if (flag_ltrans) + set_hot_bb_threshold (streamer_read_gcov_count (ib)); } } @@ -1445,10 +1708,13 @@ merge_profile_summaries (struct lto_file_decl_data **file_data_vec) { struct lto_file_decl_data *file_data; - unsigned int j; + unsigned int j, h_ix; gcov_unsigned_t max_runs = 0; struct cgraph_node *node; struct cgraph_edge *edge; + gcov_type saved_sum_all = 0; + gcov_ctr_summary *saved_profile_info = 0; + int saved_scale = 0; /* Find unit with maximal number of runs. If we ever get serious about roundoff errors, we might also consider computing smallest common @@ -1472,6 +1738,8 @@ profile_info = <o_gcov_summary; lto_gcov_summary.runs = max_runs; lto_gcov_summary.sum_max = 0; + memset (lto_gcov_summary.histogram, 0, + sizeof (gcov_bucket_type) * GCOV_HISTOGRAM_SIZE); /* Rescale all units to the maximal number of runs. sum_max can not be easily merged, as we have no idea what files come from @@ -1479,16 +1747,55 @@ for (j = 0; (file_data = file_data_vec[j]) != NULL; j++) if (file_data->profile_info.runs) { - int scale = ((REG_BR_PROB_BASE * max_runs - + file_data->profile_info.runs / 2) - / file_data->profile_info.runs); - lto_gcov_summary.sum_max = MAX (lto_gcov_summary.sum_max, - (file_data->profile_info.sum_max - * scale - + REG_BR_PROB_BASE / 2) - / REG_BR_PROB_BASE); + int scale = GCOV_COMPUTE_SCALE (max_runs, + file_data->profile_info.runs); + lto_gcov_summary.sum_max + = MAX (lto_gcov_summary.sum_max, + apply_scale (file_data->profile_info.sum_max, scale)); + lto_gcov_summary.sum_all + = MAX (lto_gcov_summary.sum_all, + apply_scale (file_data->profile_info.sum_all, scale)); + /* Save a pointer to the profile_info with the largest + scaled sum_all and the scale for use in merging the + histogram. */ + if (!saved_profile_info + || lto_gcov_summary.sum_all > saved_sum_all) + { + saved_profile_info = &file_data->profile_info; + saved_sum_all = lto_gcov_summary.sum_all; + saved_scale = scale; + } } + gcc_assert (saved_profile_info); + + /* Scale up the histogram from the profile that had the largest + scaled sum_all above. */ + for (h_ix = 0; h_ix < GCOV_HISTOGRAM_SIZE; h_ix++) + { + /* Scale up the min value as we did the corresponding sum_all + above. Use that to find the new histogram index. */ + gcov_type scaled_min + = apply_scale (saved_profile_info->histogram[h_ix].min_value, + saved_scale); + /* The new index may be shared with another scaled histogram entry, + so we need to account for a non-zero histogram entry at new_ix. */ + unsigned new_ix = gcov_histo_index (scaled_min); + lto_gcov_summary.histogram[new_ix].min_value + = (lto_gcov_summary.histogram[new_ix].num_counters + ? MIN (lto_gcov_summary.histogram[new_ix].min_value, scaled_min) + : scaled_min); + /* Some of the scaled counter values would ostensibly need to be placed + into different (larger) histogram buckets, but we keep things simple + here and place the scaled cumulative counter value in the bucket + corresponding to the scaled minimum counter value. */ + lto_gcov_summary.histogram[new_ix].cum_value + += apply_scale (saved_profile_info->histogram[h_ix].cum_value, + saved_scale); + lto_gcov_summary.histogram[new_ix].num_counters + += saved_profile_info->histogram[h_ix].num_counters; + } + /* Watch roundoff errors. */ if (lto_gcov_summary.sum_max < max_runs) lto_gcov_summary.sum_max = max_runs; @@ -1500,35 +1807,32 @@ /* Now compute count_materialization_scale of each node. During LTRANS we already have values of count_materialization_scale computed, so just update them. */ - for (node = cgraph_nodes; node; node = node->next) - if (node->local.lto_file_data->profile_info.runs) + FOR_EACH_FUNCTION (node) + if (node->lto_file_data + && node->lto_file_data->profile_info.runs) { int scale; - scale = - ((node->count_materialization_scale * max_runs - + node->local.lto_file_data->profile_info.runs / 2) - / node->local.lto_file_data->profile_info.runs); + scale = RDIV (node->count_materialization_scale * max_runs, + node->lto_file_data->profile_info.runs); node->count_materialization_scale = scale; if (scale < 0) - fatal_error ("Profile information in %s corrupted", + fatal_error (input_location, "Profile information in %s corrupted", file_data->file_name); if (scale == REG_BR_PROB_BASE) continue; for (edge = node->callees; edge; edge = edge->next_callee) - edge->count = ((edge->count * scale + REG_BR_PROB_BASE / 2) - / REG_BR_PROB_BASE); - node->count = ((node->count * scale + REG_BR_PROB_BASE / 2) - / REG_BR_PROB_BASE); + edge->count = edge->count.apply_scale (scale, REG_BR_PROB_BASE); + node->count = node->count.apply_scale (scale, REG_BR_PROB_BASE); } } -/* Input and merge the cgraph from each of the .o files passed to +/* Input and merge the symtab from each of the .o files passed to lto1. */ void -input_cgraph (void) +input_symtab (void) { struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data (); struct lto_file_decl_data *file_data; @@ -1540,75 +1844,118 @@ const char *data; size_t len; struct lto_input_block *ib; - VEC(cgraph_node_ptr, heap) *nodes; - VEC(varpool_node_ptr, heap) *varpool; + vec<symtab_node *> nodes; - ib = lto_create_simple_input_block (file_data, LTO_section_cgraph, + ib = lto_create_simple_input_block (file_data, LTO_section_symtab_nodes, &data, &len); if (!ib) - fatal_error ("cannot find LTO cgraph in %s", file_data->file_name); + fatal_error (input_location, + "cannot find LTO cgraph in %s", file_data->file_name); input_profile_summary (ib, file_data); - file_data->cgraph_node_encoder = lto_cgraph_encoder_new (); + file_data->symtab_node_encoder = lto_symtab_encoder_new (true); nodes = input_cgraph_1 (file_data, ib); - lto_destroy_simple_input_block (file_data, LTO_section_cgraph, - ib, data, len); - - ib = lto_create_simple_input_block (file_data, LTO_section_varpool, - &data, &len); - if (!ib) - fatal_error ("cannot find LTO varpool in %s", file_data->file_name); - varpool = input_varpool_1 (file_data, ib); - lto_destroy_simple_input_block (file_data, LTO_section_varpool, + lto_destroy_simple_input_block (file_data, LTO_section_symtab_nodes, ib, data, len); ib = lto_create_simple_input_block (file_data, LTO_section_refs, &data, &len); if (!ib) - fatal_error("cannot find LTO section refs in %s", file_data->file_name); - input_refs (ib, nodes, varpool); + fatal_error (input_location, "cannot find LTO section refs in %s", + file_data->file_name); + input_refs (ib, nodes); lto_destroy_simple_input_block (file_data, LTO_section_refs, ib, data, len); if (flag_ltrans) input_cgraph_opt_summary (nodes); - VEC_free (cgraph_node_ptr, heap, nodes); - VEC_free (varpool_node_ptr, heap, varpool); + nodes.release (); } + merge_profile_summaries (file_data_vec); - + + if (!flag_auto_profile) + get_working_sets (); + /* Clear out the aux field that was used to store enough state to tell which nodes should be overwritten. */ - for (node = cgraph_nodes; node; node = node->next) + FOR_EACH_FUNCTION (node) { /* Some nodes may have been created by cgraph_node. This happens when the callgraph contains nested functions. If the node for the parent function was never emitted to the gimple file, cgraph_node will create a node for it when setting the context of the nested function. */ - if (node->local.lto_file_data) + if (node->lto_file_data) node->aux = NULL; } } +/* Input function/variable tables that will allow libgomp to look up offload + target code, and store them into OFFLOAD_FUNCS and OFFLOAD_VARS. */ + +void +input_offload_tables (bool do_force_output) +{ + struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data (); + struct lto_file_decl_data *file_data; + unsigned int j = 0; + + while ((file_data = file_data_vec[j++])) + { + const char *data; + size_t len; + struct lto_input_block *ib + = lto_create_simple_input_block (file_data, LTO_section_offload_table, + &data, &len); + if (!ib) + continue; + + enum LTO_symtab_tags tag + = streamer_read_enum (ib, LTO_symtab_tags, LTO_symtab_last_tag); + while (tag) + { + if (tag == LTO_symtab_unavail_node) + { + int decl_index = streamer_read_uhwi (ib); + tree fn_decl + = lto_file_decl_data_get_fn_decl (file_data, decl_index); + vec_safe_push (offload_funcs, fn_decl); + + /* Prevent IPA from removing fn_decl as unreachable, since there + may be no refs from the parent function to child_fn in offload + LTO mode. */ + if (do_force_output) + cgraph_node::get (fn_decl)->mark_force_output (); + } + else if (tag == LTO_symtab_variable) + { + int decl_index = streamer_read_uhwi (ib); + tree var_decl + = lto_file_decl_data_get_var_decl (file_data, decl_index); + vec_safe_push (offload_vars, var_decl); + + /* Prevent IPA from removing var_decl as unused, since there + may be no refs to var_decl in offload LTO mode. */ + if (do_force_output) + varpool_node::get (var_decl)->force_output = 1; + } + else + fatal_error (input_location, + "invalid offload table in %s", file_data->file_name); + + tag = streamer_read_enum (ib, LTO_symtab_tags, LTO_symtab_last_tag); + } + + lto_destroy_simple_input_block (file_data, LTO_section_offload_table, + ib, data, len); + } +} + /* True when we need optimization summary for NODE. */ static int -output_cgraph_opt_summary_p (struct cgraph_node *node, cgraph_node_set set) +output_cgraph_opt_summary_p (struct cgraph_node *node) { - struct cgraph_edge *e; - - if (cgraph_node_in_set_p (node, set)) - { - for (e = node->callees; e; e = e->next_callee) - if (e->indirect_info - && e->indirect_info->thunk_delta != 0) - return true; - - for (e = node->indirect_calls; e; e = e->next_callee) - if (e->indirect_info->thunk_delta != 0) - return true; - } - return (node->clone_of && (node->clone.tree_map || node->clone.args_to_skip @@ -1617,14 +1964,9 @@ /* Output optimization summary for EDGE to OB. */ static void -output_edge_opt_summary (struct output_block *ob, - struct cgraph_edge *edge) +output_edge_opt_summary (struct output_block *ob ATTRIBUTE_UNUSED, + struct cgraph_edge *edge ATTRIBUTE_UNUSED) { - if (edge->indirect_info) - lto_output_sleb128_stream (ob->main_stream, - edge->indirect_info->thunk_delta); - else - lto_output_sleb128_stream (ob->main_stream, 0); } /* Output optimization summary for NODE to OB. */ @@ -1632,7 +1974,7 @@ static void output_node_opt_summary (struct output_block *ob, struct cgraph_node *node, - cgraph_node_set set) + lto_symtab_encoder_t encoder) { unsigned int index; bitmap_iterator bi; @@ -1641,37 +1983,38 @@ int i; struct cgraph_edge *e; - lto_output_uleb128_stream (ob->main_stream, - bitmap_count_bits (node->clone.args_to_skip)); - EXECUTE_IF_SET_IN_BITMAP (node->clone.args_to_skip, 0, index, bi) - lto_output_uleb128_stream (ob->main_stream, index); - lto_output_uleb128_stream (ob->main_stream, - bitmap_count_bits (node->clone.combined_args_to_skip)); - EXECUTE_IF_SET_IN_BITMAP (node->clone.combined_args_to_skip, 0, index, bi) - lto_output_uleb128_stream (ob->main_stream, index); - lto_output_uleb128_stream (ob->main_stream, - VEC_length (ipa_replace_map_p, node->clone.tree_map)); - FOR_EACH_VEC_ELT (ipa_replace_map_p, node->clone.tree_map, i, map) + if (node->clone.args_to_skip) + { + streamer_write_uhwi (ob, bitmap_count_bits (node->clone.args_to_skip)); + EXECUTE_IF_SET_IN_BITMAP (node->clone.args_to_skip, 0, index, bi) + streamer_write_uhwi (ob, index); + } + else + streamer_write_uhwi (ob, 0); + if (node->clone.combined_args_to_skip) { - int parm_num; - tree parm; - - for (parm_num = 0, parm = DECL_ARGUMENTS (node->decl); parm; - parm = DECL_CHAIN (parm), parm_num++) - if (map->old_tree == parm) - break; + streamer_write_uhwi (ob, bitmap_count_bits (node->clone.combined_args_to_skip)); + EXECUTE_IF_SET_IN_BITMAP (node->clone.combined_args_to_skip, 0, index, bi) + streamer_write_uhwi (ob, index); + } + else + streamer_write_uhwi (ob, 0); + streamer_write_uhwi (ob, vec_safe_length (node->clone.tree_map)); + FOR_EACH_VEC_SAFE_ELT (node->clone.tree_map, i, map) + { /* At the moment we assume all old trees to be PARM_DECLs, because we have no mechanism to store function local declarations into summaries. */ - gcc_assert (parm); - lto_output_uleb128_stream (ob->main_stream, parm_num); - lto_output_tree (ob, map->new_tree, true); + gcc_assert (!map->old_tree); + streamer_write_uhwi (ob, map->parm_num); + gcc_assert (EXPR_LOCATION (map->new_tree) == UNKNOWN_LOCATION); + stream_write_tree (ob, map->new_tree, true); bp = bitpack_create (ob->main_stream); bp_pack_value (&bp, map->replace_p, 1); bp_pack_value (&bp, map->ref_p, 1); - lto_output_bitpack (&bp); + streamer_write_bitpack (&bp); } - if (cgraph_node_in_set_p (node, set)) + if (lto_symtab_encoder_in_partition_p (encoder, node)) { for (e = node->callees; e; e = e->next_callee) output_edge_opt_summary (ob, e); @@ -1684,29 +2027,32 @@ At the moment it is the clone info structure. */ static void -output_cgraph_opt_summary (cgraph_node_set set) +output_cgraph_opt_summary (void) { - struct cgraph_node *node; int i, n_nodes; - lto_cgraph_encoder_t encoder; + lto_symtab_encoder_t encoder; struct output_block *ob = create_output_block (LTO_section_cgraph_opt_sum); unsigned count = 0; - ob->cgraph_node = NULL; - encoder = ob->decl_state->cgraph_node_encoder; - n_nodes = lto_cgraph_encoder_size (encoder); - for (i = 0; i < n_nodes; i++) - if (output_cgraph_opt_summary_p (lto_cgraph_encoder_deref (encoder, i), - set)) - count++; - lto_output_uleb128_stream (ob->main_stream, count); + ob->symbol = NULL; + encoder = ob->decl_state->symtab_node_encoder; + n_nodes = lto_symtab_encoder_size (encoder); for (i = 0; i < n_nodes; i++) { - node = lto_cgraph_encoder_deref (encoder, i); - if (output_cgraph_opt_summary_p (node, set)) + symtab_node *node = lto_symtab_encoder_deref (encoder, i); + cgraph_node *cnode = dyn_cast <cgraph_node *> (node); + if (cnode && output_cgraph_opt_summary_p (cnode)) + count++; + } + streamer_write_uhwi (ob, count); + for (i = 0; i < n_nodes; i++) + { + symtab_node *node = lto_symtab_encoder_deref (encoder, i); + cgraph_node *cnode = dyn_cast <cgraph_node *> (node); + if (cnode && output_cgraph_opt_summary_p (cnode)) { - lto_output_uleb128_stream (ob->main_stream, i); - output_node_opt_summary (ob, node, set); + streamer_write_uhwi (ob, i); + output_node_opt_summary (ob, cnode, encoder); } } produce_asm (ob, NULL); @@ -1716,17 +2062,9 @@ /* Input optimisation summary of EDGE. */ static void -input_edge_opt_summary (struct cgraph_edge *edge, - struct lto_input_block *ib_main) +input_edge_opt_summary (struct cgraph_edge *edge ATTRIBUTE_UNUSED, + struct lto_input_block *ib_main ATTRIBUTE_UNUSED) { - HOST_WIDE_INT thunk_delta; - thunk_delta = lto_input_sleb128 (ib_main); - if (thunk_delta != 0) - { - gcc_assert (!edge->indirect_info); - edge->indirect_info = cgraph_allocate_init_indirect_info (); - edge->indirect_info->thunk_delta = thunk_delta; - } } /* Input optimisation summary of NODE. */ @@ -1742,37 +2080,32 @@ struct bitpack_d bp; struct cgraph_edge *e; - count = lto_input_uleb128 (ib_main); + count = streamer_read_uhwi (ib_main); if (count) node->clone.args_to_skip = BITMAP_GGC_ALLOC (); for (i = 0; i < count; i++) { - bit = lto_input_uleb128 (ib_main); + bit = streamer_read_uhwi (ib_main); bitmap_set_bit (node->clone.args_to_skip, bit); } - count = lto_input_uleb128 (ib_main); + count = streamer_read_uhwi (ib_main); if (count) node->clone.combined_args_to_skip = BITMAP_GGC_ALLOC (); for (i = 0; i < count; i++) { - bit = lto_input_uleb128 (ib_main); + bit = streamer_read_uhwi (ib_main); bitmap_set_bit (node->clone.combined_args_to_skip, bit); } - count = lto_input_uleb128 (ib_main); + count = streamer_read_uhwi (ib_main); for (i = 0; i < count; i++) { - int parm_num; - tree parm; - struct ipa_replace_map *map = ggc_alloc_ipa_replace_map (); + struct ipa_replace_map *map = ggc_alloc<ipa_replace_map> (); - VEC_safe_push (ipa_replace_map_p, gc, node->clone.tree_map, map); - for (parm_num = 0, parm = DECL_ARGUMENTS (node->decl); parm_num; - parm = DECL_CHAIN (parm)) - parm_num --; - map->parm_num = lto_input_uleb128 (ib_main); + vec_safe_push (node->clone.tree_map, map); + map->parm_num = streamer_read_uhwi (ib_main); map->old_tree = NULL; - map->new_tree = lto_input_tree (ib_main, data_in); - bp = lto_input_bitpack (ib_main); + map->new_tree = stream_read_tree (ib_main, data_in); + bp = streamer_read_bitpack (ib_main); map->replace_p = bp_unpack_value (&bp, 1); map->ref_p = bp_unpack_value (&bp, 1); } @@ -1786,31 +2119,30 @@ static void input_cgraph_opt_section (struct lto_file_decl_data *file_data, - const char *data, size_t len, VEC (cgraph_node_ptr, - heap) * nodes) + const char *data, size_t len, + vec<symtab_node *> nodes) { const struct lto_function_header *header = (const struct lto_function_header *) data; - const int32_t cfg_offset = sizeof (struct lto_function_header); - const int32_t main_offset = cfg_offset + header->cfg_size; - const int32_t string_offset = main_offset + header->main_size; + const int cfg_offset = sizeof (struct lto_function_header); + const int main_offset = cfg_offset + header->cfg_size; + const int string_offset = main_offset + header->main_size; struct data_in *data_in; - struct lto_input_block ib_main; unsigned int i; unsigned int count; - LTO_INIT_INPUT_BLOCK (ib_main, (const char *) data + main_offset, 0, - header->main_size); + lto_input_block ib_main ((const char *) data + main_offset, + header->main_size, file_data->mode_table); data_in = lto_data_in_create (file_data, (const char *) data + string_offset, - header->string_size, NULL); - count = lto_input_uleb128 (&ib_main); + header->string_size, vNULL); + count = streamer_read_uhwi (&ib_main); for (i = 0; i < count; i++) { - int ref = lto_input_uleb128 (&ib_main); - input_node_opt_summary (VEC_index (cgraph_node_ptr, nodes, ref), + int ref = streamer_read_uhwi (&ib_main); + input_node_opt_summary (dyn_cast<cgraph_node *> (nodes[ref]), &ib_main, data_in); } lto_free_section_data (file_data, LTO_section_cgraph_opt_sum, NULL, data, @@ -1821,7 +2153,7 @@ /* Input optimization summary of cgraph. */ static void -input_cgraph_opt_summary (VEC (cgraph_node_ptr, heap) * nodes) +input_cgraph_opt_summary (vec<symtab_node *> nodes) { struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data (); struct lto_file_decl_data *file_data;