Implement sample-scoped ELF and DWARF loading, line and inline indexes,
symbol identity lookup, and address symbolization. Only compilation units
touched by profile samples are parsed, limiting memory and load time on
large binaries.
afdo_tools/ChangeLog:
* src/dwarf_reader.cc: New file.
* src/dwarf_reader_impl.h: New file.
Signed-off-by: Kugan Vivekanandarajah <[email protected]>
Co-authored-by: Dhruv Chawla <[email protected]>
Co-authored-by: Rahul Shinde <[email protected]>
---
afdo_tools/src/dwarf_reader.cc | 2545 ++++++++++++++++++++++++++++
afdo_tools/src/dwarf_reader_impl.h | 359 ++++
2 files changed, 2904 insertions(+)
create mode 100644 afdo_tools/src/dwarf_reader.cc
create mode 100644 afdo_tools/src/dwarf_reader_impl.h
diff --git a/afdo_tools/src/dwarf_reader.cc b/afdo_tools/src/dwarf_reader.cc
new file mode 100644
index 00000000000..ecf1deb986d
--- /dev/null
+++ b/afdo_tools/src/dwarf_reader.cc
@@ -0,0 +1,2545 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#include "dwarf_reader_impl.h"
+#include "symbol_util.h"
+#include "afdo_assert.h"
+
+#include <algorithm>
+#include <atomic>
+#include <climits>
+#include <cstdint>
+#include <cstdio>
+#include <dwarf.h>
+#include <elf.h>
+#include <elfutils/libdw.h>
+#include <gelf.h>
+#include <fcntl.h>
+#include <iterator>
+#include <libelf.h>
+#include <limits>
+#include <thread>
+#include <unordered_set>
+#include <unistd.h>
+
+namespace dwarf_debug {
+namespace {
+
+/* DW_AT_GNU_discriminator (GCC vendor attr; not in elfutils dwarf.h). */
+constexpr Dwarf_Half DW_AT_CALL_DISCRIMINATOR_EXT = 0x2136;
+
+/* Retain ELF binding only while choosing the owner of aliased ranges. */
+struct elf_func_sym
+{
+ function_metadata meta;
+ unsigned char bind = 0;
+};
+
+/* Sort merged subprogram rows for binary search; drop exact duplicates
+ only. */
+void
+sort_dedupe_function_metadata (std::vector<function_metadata> &funcs)
+{
+ if (funcs.empty ())
+ return;
+ std::sort (funcs.begin (),
+ funcs.end (),
+ [] (const function_metadata &a, const function_metadata &b)
+ {
+ if (a.start_addr != b.start_addr)
+ return a.start_addr < b.start_addr;
+ if (a.end_addr != b.end_addr)
+ return a.end_addr < b.end_addr;
+ /* By name text: interned_string is a pointer, so ordering it
+ directly would follow intern order and vary between runs of
+ the parallel CU parse. */
+ return string_value (a.name) < string_value (b.name);
+ });
+ funcs.erase (
+ std::unique (funcs.begin (),
+ funcs.end (),
+ [] (const function_metadata &a, const function_metadata &b)
+ {
+ return a.start_addr == b.start_addr
+ && a.end_addr == b.end_addr && a.name == b.name;
+ }),
+ funcs.end ());
+}
+
+/* True when NAME ends in a GCC clone suffix that symbol_base_name strips. */
+bool
+has_gcc_generated_suffix (const std::string &name)
+{
+ return gcc_symbol::symbol_base_name (name) != name;
+}
+
+/* Normalize path separators and redundant "." components. */
+std::string
+canonical_source_path (std::string path)
+{
+ if (path.empty ())
+ return path;
+ for (char &c : path)
+ if (c == '\\')
+ c = '/';
+ while (path.size () >= 2 && path[0] == '.' && path[1] == '/')
+ path.erase (0, 2);
+ for (;;)
+ {
+ const size_t pos = path.find ("/./");
+ if (pos == std::string::npos)
+ break;
+ path.erase (pos, 2);
+ }
+ for (;;)
+ {
+ const size_t pos = path.find ("//");
+ if (pos == std::string::npos)
+ break;
+ path.erase (pos, 1);
+ }
+ return path;
+}
+
+/* Build INDEX for RANGES, which must be sorted by start. Each leaf stores
+ one range's end, and each internal node stores the maximum end below it. */
+template <typename Range>
+void
+build_range_max_tree (const std::vector<Range> &ranges,
+ range_max_tree_index &index)
+{
+ index.leaf_base = 1;
+ while (index.leaf_base < ranges.size ())
+ index.leaf_base *= 2;
+ index.max_ends.assign (index.leaf_base * 2, 0);
+ for (size_t i = 0; i < ranges.size (); ++i)
+ index.max_ends[index.leaf_base + i] = ranges[i].end;
+ for (size_t i = index.leaf_base; i-- > 1;)
+ index.max_ends[i]
+ = std::max (index.max_ends[i * 2], index.max_ends[i * 2 + 1]);
+}
+
+/* Visit candidate leaf indices below NODE. Ignore indices at or beyond
+ RANGE_LIMIT and subtrees whose ranges all end at or before QUERY_START.
+ NODE covers indices from NODE_BEGIN up to but not including NODE_END. */
+template <typename Visitor>
+void
+visit_range_max_tree (const range_max_tree_index &index, size_t node,
+ size_t node_begin, size_t node_end, size_t range_limit,
+ uint64_t query_start, Visitor &visitor)
+{
+ if (node_begin >= range_limit || index.max_ends[node] <= query_start)
+ return;
+ if (node >= index.leaf_base)
+ {
+ visitor (node - index.leaf_base);
+ return;
+ }
+ const size_t middle = node_begin + (node_end - node_begin) / 2;
+ visit_range_max_tree (
+ index, node * 2, node_begin, middle, range_limit, query_start, visitor);
+ visit_range_max_tree (
+ index, node * 2 + 1, middle, node_end, range_limit, query_start, visitor);
+}
+
+/* Call VISITOR with the index of every range overlapping the half-open range
+ from START to END. RANGES must be sorted by start and INDEX must have been
+ built from it. */
+template <typename Range, typename Visitor>
+void
+visit_overlapping_ranges (const std::vector<Range> &ranges,
+ const range_max_tree_index &index, uint64_t start,
+ uint64_t end, Visitor visitor)
+{
+ if (start >= end || ranges.empty ())
+ return;
+ const auto limit
+ = std::lower_bound (ranges.begin (),
+ ranges.end (),
+ end,
+ [] (const Range &range, uint64_t address)
+ {
+ return range.start < address;
+ });
+ const size_t range_limit = static_cast<size_t> (limit - ranges.begin ());
+ visit_range_max_tree (
+ index, 1, 0, index.leaf_base, range_limit, start, visitor);
+}
+
+/* A DWARF file-table entry usable as a profile source file. GCC also emits
+ pseudo-files (<built-in>, <artificial>) that name no source. */
+bool
+dwarf_source_file_is_usable (const char *file)
+{
+ return file != nullptr && file[0] != '\0' && file[0] != '<';
+}
+
+/* ELF symtab entries treated as correlatable function ranges: STT_FUNC,
+ trailing .cold blocks, and .__part.N splits; reject @plt and size 0. */
+bool
+symtab_accepts_function_symbol (const char *name, uint64_t size, int sym_type)
+{
+ if (name == nullptr || name[0] == '\0' || size == 0)
+ return false;
+
+ std::string sv (name);
+ if (gcc_symbol::has_suffix (sv, "@plt"))
+ return false;
+ if (sym_type == STT_FUNC || gcc_symbol::has_suffix (sv, ".cold"))
+ return true;
+
+ while (!sv.empty ()
+ && std::isdigit (static_cast<unsigned char> (sv.back ())))
+ sv.pop_back ();
+ return gcc_symbol::has_suffix (sv, ".__part.");
+}
+
+/* True for a DW_AT_declaration DIE, that is, a declaration without a body. */
+bool
+die_is_declaration (Dwarf_Die *die)
+{
+ Dwarf_Attribute decl_mem;
+ Dwarf_Attribute *decl_attr = dwarf_attr (die, DW_AT_declaration, &decl_mem);
+ if (decl_attr == nullptr)
+ return false;
+ bool flag = false;
+ return dwarf_formflag (decl_attr, &flag) == 0 && flag;
+}
+
+/* Resolve gcov symbol name: prefer linkage name, else DW_AT_name via
+ spec/abstract-origin chain. */
+const char *
+resolve_die_symbol_name (Dwarf_Die *die)
+{
+ if (die == nullptr)
+ return nullptr;
+
+ const char *first_diename = nullptr;
+ Dwarf_Die cur = *die;
+ /* Cycle guard: a spec/abstract-origin reference must never revisit a DIE
+ we already walked (malformed/looping DWARF). Well-formed chains are
+ 1-3 deep, so this set stays tiny. */
+ std::vector<Dwarf_Off> seen;
+ for (;;)
+ {
+ const Dwarf_Off off = dwarf_dieoffset (&cur);
+ if (std::find (seen.begin (), seen.end (), off) != seen.end ())
+ break;
+ seen.push_back (off);
+
+ /* Linkage name is the gcov key; the first one on the chain is the
+ one. */
+ Dwarf_Attribute link_mem;
+ Dwarf_Attribute *const link_attr
+ = dwarf_attr (&cur, DW_AT_linkage_name, &link_mem);
+ const char *const link
+ = link_attr == nullptr ? nullptr : dwarf_formstring (link_attr);
+ if (link != nullptr && link[0] != '\0')
+ return link;
+
+ const char *diename = dwarf_diename (&cur);
+ if (diename != nullptr && diename[0] != '\0' && first_diename == nullptr)
+ first_diename = diename;
+
+ /* Follow DW_AT_specification, else DW_AT_abstract_origin, to the next
+ DIE in the chain (dwarf_formref_die rewrites cur); stop when
+ neither. */
+ Dwarf_Attribute attr_mem;
+ const bool advanced
+ = (dwarf_attr (&cur, DW_AT_specification, &attr_mem) != nullptr
+ && dwarf_formref_die (&attr_mem, &cur) != nullptr)
+ || (dwarf_attr (&cur, DW_AT_abstract_origin, &attr_mem) != nullptr
+ && dwarf_formref_die (&attr_mem, &cur) != nullptr);
+ if (!advanced)
+ break;
+ }
+
+ return first_diename;
+}
+
+/* True if every child PC range lies inside some parent range. */
+bool
+pc_ranges_subset_of (const pc_range_list &child, const pc_range_list &parent)
+{
+ if (child.empty ())
+ return false;
+ return std::all_of (child.begin (), child.end (), [&] (const auto &cr)
+ {
+ const uint64_t clo = cr.first;
+ const uint64_t chi = cr.second;
+ if (chi <= clo)
+ return true;
+ return std::any_of (parent.begin (), parent.end (), [&] (const auto &pr)
+ {
+ return pr.first <= clo && chi <= pr.second;
+ });
+ });
+}
+
+/* Unsigned value of ATTR_NAME on DIE, or 0 if absent or not udata-form. */
+Dwarf_Word
+die_udata_attr (Dwarf_Die *die, Dwarf_Half attr_name)
+{
+ if (die == nullptr)
+ return 0;
+ Dwarf_Attribute attr_mem;
+ Dwarf_Attribute *attr = dwarf_attr (die, attr_name, &attr_mem);
+ Dwarf_Word value = 0;
+ if (attr == nullptr || dwarf_formudata (attr, &value) != 0)
+ return 0;
+ return value;
+}
+
+/* CU path for DW_AT_call_file, or null. libdw's table is already indexed by
+ the raw attribute value (DWARF 5 from 0; earlier versions keep a dummy
+ row 0). */
+const char *
+die_call_file_path (Dwarf_Die *die)
+{
+ Dwarf_Attribute attr_mem;
+ if (die == nullptr || dwarf_attr (die, DW_AT_call_file, &attr_mem) ==
nullptr)
+ return nullptr;
+ Dwarf_Word file_idx = 0;
+ if (dwarf_formudata (&attr_mem, &file_idx) != 0)
+ return nullptr;
+ Dwarf_Die cu_mem;
+ Dwarf_Die *cu = dwarf_diecu (die, &cu_mem, nullptr, nullptr);
+ Dwarf_Files *files = nullptr;
+ size_t nfiles = 0;
+ if (cu == nullptr || dwarf_getsrcfiles (cu, &files, &nfiles) != 0
+ || files == nullptr || file_idx >= nfiles)
+ return nullptr;
+ const char *path
+ = dwarf_filesrc (files, static_cast<size_t> (file_idx), nullptr, nullptr);
+ if (path == nullptr || path[0] == '\0')
+ return nullptr;
+ return path;
+}
+
+/* Decl line/file from the DIE, or its enclosing class/struct for artificial
+ members (e.g. lambda operator()) that omit DW_AT_decl_*. Prefer the
+ enclosing-type index; dwarf_getscopes_die is last resort. */
+die_decl_site
+resolve_die_decl_site (
+ Dwarf_Die *die,
+ const std::unordered_map<Dwarf_Off, die_decl_site> *enclosing_by_die)
+{
+ die_decl_site info;
+ if (die == nullptr)
+ return info;
+
+ int ln = 0;
+ if (dwarf_decl_line (die, &ln) == 0 && ln > 0)
+ info.line_number = static_cast<uint32_t> (ln);
+ /* <artificial> / <built-in> are not source files; skip them so origin
+ and enclosing-type fallbacks can still run. */
+ const char *file = dwarf_decl_file (die);
+ if (dwarf_source_file_is_usable (file))
+ info.file_path = file;
+ if (info.line_number != 0 && info.file_path != nullptr)
+ return info;
+
+ /* Follow the semantic declaration chain before asking for lexical scopes.
+ An inlined DIE is nested in its caller, not in the callee's declaring
+ type. Prefer abstract_origin at each step, as libdw integration does. */
+ Dwarf_Die origin = *die;
+ bool advanced = false;
+ /* True if the start DIE has origin/spec. Its lexical parent is the
+ caller, even when the reference does not resolve. */
+ bool start_has_semantic_link = false;
+ std::vector<Dwarf_Off> seen;
+ for (;;)
+ {
+ const Dwarf_Off off = dwarf_dieoffset (&origin);
+ if (std::find (seen.begin (), seen.end (), off) != seen.end ())
+ break;
+ seen.push_back (off);
+
+ if (advanced)
+ {
+ /* Integration can stop on a pseudo-file; a later DIE may have a
+ real path. */
+ if (info.file_path == nullptr)
+ {
+ file = dwarf_decl_file (&origin);
+ if (dwarf_source_file_is_usable (file))
+ info.file_path = file;
+ }
+ if (enclosing_by_die != nullptr)
+ {
+ const auto it = enclosing_by_die->find (off);
+ if (it != enclosing_by_die->end ())
+ {
+ if (info.file_path == nullptr
+ && dwarf_source_file_is_usable (it->second.file_path))
+ info.file_path = it->second.file_path;
+ if (info.line_number == 0)
+ info.line_number = it->second.line_number;
+ }
+ }
+ if (info.line_number != 0 && info.file_path != nullptr)
+ return info;
+ }
+
+ Dwarf_Attribute attr_mem;
+ Dwarf_Attribute *attr
+ = dwarf_attr (&origin, DW_AT_abstract_origin, &attr_mem);
+ if (attr == nullptr)
+ attr = dwarf_attr (&origin, DW_AT_specification, &attr_mem);
+ if (attr == nullptr)
+ break;
+ if (!advanced)
+ start_has_semantic_link = true;
+
+ Dwarf_Die next;
+ if (dwarf_formref_die (attr, &next) == nullptr)
+ break;
+ origin = next;
+ advanced = true;
+ }
+
+ /* With no semantic indirection, the DIE's own lexical parent is relevant.
+ Otherwise the loop above deliberately skipped the caller-side entry. */
+ if (!start_has_semantic_link && enclosing_by_die != nullptr)
+ {
+ const auto it = enclosing_by_die->find (dwarf_dieoffset (&origin));
+ if (it != enclosing_by_die->end ())
+ {
+ if (info.file_path == nullptr
+ && dwarf_source_file_is_usable (it->second.file_path))
+ info.file_path = it->second.file_path;
+ if (info.line_number == 0)
+ info.line_number = it->second.line_number;
+ if (info.line_number != 0 && info.file_path != nullptr)
+ return info;
+ }
+ }
+
+ /* Broken origin/spec: ORIGIN is still the caller-nested start DIE. */
+ if (start_has_semantic_link && !advanced)
+ return info;
+
+ Dwarf_Die *scopes = nullptr;
+ const int nscopes = dwarf_getscopes_die (&origin, &scopes);
+ /* scopes[0] is ORIGIN itself; enclosing scopes begin at index 1. */
+ for (int i = 1; i < nscopes; ++i)
+ {
+ const int tag = dwarf_tag (&scopes[i]);
+ if (tag != DW_TAG_class_type && tag != DW_TAG_structure_type
+ && tag != DW_TAG_union_type)
+ continue;
+ if (info.line_number == 0)
+ {
+ ln = 0;
+ if (dwarf_decl_line (&scopes[i], &ln) == 0 && ln > 0)
+ info.line_number = static_cast<uint32_t> (ln);
+ }
+ if (info.file_path == nullptr)
+ {
+ file = dwarf_decl_file (&scopes[i]);
+ if (dwarf_source_file_is_usable (file))
+ info.file_path = file;
+ }
+ break;
+ }
+ free (scopes);
+ return info;
+}
+
+/* Open a libdw session on BINARY_PATH; each thread needs its own, as handles
+ are not thread-safe. */
+bool
+open_libdw_session (const std::string &binary_path, libdw_file_session
&session)
+{
+ session.fd = open (binary_path.c_str (), O_RDONLY);
+ if (session.fd < 0)
+ return false;
+ session.dwarf = dwarf_begin (session.fd, DWARF_C_READ);
+ if (!session.dwarf)
+ {
+ close (session.fd);
+ session.fd = -1;
+ return false;
+ }
+ return true;
+}
+
+/* Release a libdw session; safe on a partial or already-closed open. */
+void
+close_libdw_session (libdw_file_session &session)
+{
+ if (session.dwarf)
+ {
+ dwarf_end (session.dwarf);
+ session.dwarf = nullptr;
+ }
+ if (session.fd >= 0)
+ {
+ close (session.fd);
+ session.fd = -1;
+ }
+}
+
+enum class pc_range_decode_status
+{
+ no_extent,
+ valid,
+ invalid
+};
+
+/* Decode and merge the PC ranges that libdw reports for DIE. Preserve the
+ first decoded range start in FIRST_START before sorting. */
+pc_range_decode_status
+collect_die_pc_ranges (Dwarf_Die *d, pc_range_list &out,
+ uint64_t *first_start = nullptr)
+{
+ out.clear ();
+ if (first_start != nullptr)
+ *first_start = 0;
+ if (d == nullptr)
+ return pc_range_decode_status::invalid;
+
+ const bool declared_pc_extent
+ = (dwarf_hasattr (d, DW_AT_low_pc)
+ || dwarf_hasattr (d, DW_AT_high_pc)
+ || dwarf_hasattr (d, DW_AT_ranges));
+
+ Dwarf_Addr base = 0, start = 0, end = 0;
+ ptrdiff_t offset = 0;
+ bool decoded_range = false;
+ while (true)
+ {
+ const ptrdiff_t next
+ = dwarf_ranges (d, offset, &base, &start, &end);
+ if (next < 0)
+ {
+ out.clear ();
+ return pc_range_decode_status::invalid;
+ }
+ if (next == 0)
+ break;
+ const ptrdiff_t next_off = next;
+ if (next_off <= offset)
+ {
+ out.clear ();
+ return pc_range_decode_status::invalid;
+ }
+ offset = next_off;
+ if (end > start)
+ {
+ if (!decoded_range && first_start != nullptr)
+ *first_start = static_cast<uint64_t> (start);
+ decoded_range = true;
+ out.emplace_back (
+ static_cast<uint64_t> (start), static_cast<uint64_t> (end));
+ }
+ }
+
+ std::sort (out.begin (), out.end ());
+ size_t merged = 0;
+ for (const auto &range : out)
+ {
+ if (merged != 0 && range.first <= out[merged - 1].second)
+ out[merged - 1].second
+ = std::max (out[merged - 1].second, range.second);
+ else
+ out[merged++] = range;
+ }
+ out.resize (merged);
+ if (!declared_pc_extent && !decoded_range)
+ return pc_range_decode_status::no_extent;
+ return pc_range_decode_status::valid;
+}
+
+/* Return DIE's entry PC when it lies in RANGES, otherwise their first decoded
+ address. DWARF 5 permits DW_AT_entry_pc to be an address or a base-relative
+ unsigned constant. */
+uint64_t
+subprogram_entry_pc (Dwarf_Die *die, uint64_t first_start,
+ const pc_range_list &ranges)
+{
+ const auto contained = [&ranges] (uint64_t pc)
+ {
+ return std::any_of (ranges.begin (), ranges.end (),
+ [pc] (const std::pair<uint64_t, uint64_t> &range)
+ {
+ return range.first <= pc && pc < range.second;
+ });
+ };
+
+ Dwarf_Attribute attr_mem;
+ Dwarf_Attribute *const attr
+ = dwarf_attr (die, DW_AT_entry_pc, &attr_mem);
+ if (attr != nullptr)
+ {
+ /* Address vs offset is the form, not whether the value is in RANGES.
+ dwarf_formudata accepts addrx and returns the .debug_addr index. */
+ const unsigned int form = dwarf_whatform (attr);
+ const bool constant_form
+ = (form == DW_FORM_data1 || form == DW_FORM_data2
+ || form == DW_FORM_data4 || form == DW_FORM_data8
+ || form == DW_FORM_udata || form == DW_FORM_sdata
+ || form == DW_FORM_implicit_const);
+
+ if (!constant_form)
+ {
+ Dwarf_Addr address = 0;
+ if (dwarf_formaddr (attr, &address) == 0
+ && contained (static_cast<uint64_t> (address)))
+ return static_cast<uint64_t> (address);
+ }
+ else
+ {
+ Dwarf_Word offset = 0;
+ Dwarf_Addr lowpc = 0;
+ const uint64_t base
+ = dwarf_lowpc (die, &lowpc) == 0
+ ? static_cast<uint64_t> (lowpc)
+ : first_start;
+ if (dwarf_formudata (attr, &offset) == 0
+ && offset <= UINT64_MAX - base && contained (base + offset))
+ return base + offset;
+ }
+ }
+
+ if (contained (first_start))
+ return first_start;
+ return ranges.front ().first;
+}
+
+} /* anonymous namespace. */
+
+/* Interned STR if already present, else null; never inserts. */
+interned_string
+StringPool::lookup (const std::string &str) const
+{
+ if (str.empty ())
+ return nullptr;
+
+ const size_t shard = StringPool::shard (str);
+ std::lock_guard<std::mutex> lock (shard_mus_[shard]);
+ const intern_string_set &shard_set = shards_[shard];
+ const intern_string_set::const_iterator it = shard_set.find (str);
+ return it != shard_set.end () ? &*it : nullptr;
+}
+
+/* Intern STR, returning a pointer that stays valid for the pool's life. */
+interned_string
+StringPool::intern (const std::string &str)
+{
+ if (str.empty ())
+ return nullptr;
+
+ const size_t shard = StringPool::shard (str);
+ std::lock_guard<std::mutex> lock (shard_mus_[shard]);
+ intern_string_set &shard_set = shards_[shard];
+ const intern_string_set::const_iterator it = shard_set.find (str);
+ if (it != shard_set.end ())
+ return &*it;
+ return &*shard_set.insert (str).first;
+}
+
+/* Move this worker's cached symbolizations into DEST. */
+void
+correlate_session::drain_symbolize_snapshot_into (correlate_symbolize_map
&dest)
+{
+ for (auto &kv : symbolize_cache_)
+ dest[kv.first] = std::move (kv.second);
+ symbolize_cache_.clear ();
+}
+
+/* Set the CU parse worker count, 0 selecting the default; must be called
+ before load(). */
+void
+dwarf_reader_impl::set_parse_threads (unsigned n)
+{
+ if (mappings_loaded)
+ {
+ fprintf (stderr,
+ _("Warning: set_parse_threads ignored after DWARF load "
+ "(call before load())\n"));
+ return;
+ }
+ parse_threads_ = n != 0 ? n : CU_PARSE_DEFAULT_THREADS;
+}
+
+/* Symtab function whose [start, end) covers ADDRESS, or null. */
+const function_metadata *
+dwarf_reader_impl::containing_function_at_pc (uint64_t address) const
+{
+ afdo_assert (mappings_loaded);
+ afdo_assert (!symtab_functions_.empty ());
+ return function_containing_address (symtab_functions_, address);
+}
+
+/* Construct over BINARY_PATH, narrowing the DWARF scope to SAMPLED_RANGES. */
+dwarf_reader_impl::dwarf_reader_impl (std::string binary_path,
+ pc_range_list sampled_ranges)
+ : binary_path_ (std::move (binary_path))
+{
+ /* Split sample scope into merged intervals. */
+ pc_range_list ranges;
+ ranges.reserve (sampled_ranges.size ());
+ for (const auto &sampled_range : sampled_ranges)
+ {
+ const uint64_t start = sampled_range.first;
+ const uint64_t end = sampled_range.second;
+ if (end <= start)
+ continue;
+ ranges.emplace_back (start, end);
+ }
+
+ std::sort (ranges.begin (), ranges.end ());
+ for (const auto &r : ranges)
+ {
+ if (!profile_sample_ranges_.empty ()
+ && r.first <= profile_sample_ranges_.back ().second)
+ profile_sample_ranges_.back ().second
+ = std::max (profile_sample_ranges_.back ().second, r.second);
+ else
+ profile_sample_ranges_.push_back (r);
+ }
+}
+
+/* Only the libdw session needs releasing; the cached indexes go with normal
+ member destruction, as the reader is one-shot (load() asserts
+ !mappings_loaded) and so has no reuse path needing a cache reset. */
+dwarf_reader_impl::~dwarf_reader_impl ()
+{
+ close_libdw_session (dw_);
+}
+
+/* Run the whole load pipeline: open libdw, index the symtab, parse the
+ profile-touch CUs, then build the address, name and span indexes. */
+bool
+dwarf_reader_impl::load ()
+{
+ if (profile_sample_ranges_.empty ())
+ {
+ fprintf (stderr,
+ _("Error: DWARF load requires a non-empty profile sample "
+ "scope\n"));
+ return false;
+ }
+
+ afdo_assert (!mappings_loaded);
+
+ if (!open_binary_dwarf ())
+ return false;
+
+ /* ELF symtab index (needed during CU parse). */
+ build_symtab_func_index ();
+ if (symtab_functions_.empty ())
+ {
+ fprintf (stderr,
+ _("Error: ELF symtab has no correlatable function symbols "
+ "for profile bucketing.\n"));
+ close_libdw_session (dw_);
+ return false;
+ }
+
+ build_global_die_metadata ();
+ if (!build_address_range_index ())
+ {
+ close_libdw_session (dw_);
+ return false;
+ }
+
+ /* Dedupe DWARF subprograms and build name index. */
+ sort_dedupe_function_metadata (dwarf_functions_);
+ build_dwarf_name_indexes ();
+ afdo_assert (mappings_loaded);
+
+ /* Fill missing STT_FUNC file/line from entry-PC symbolize (on demand). */
+ resolve_touch_symtab_export_anchors ();
+
+ load_summary_.touch_cus = profile_cu_offs_.size ();
+ load_summary_.dwarf_functions = dwarf_functions_.size ();
+ load_summary_.symtab_functions = symtab_functions_.size ();
+ load_summary_.line_segments = address_ranges_.size ();
+ /* Unique profile-touch PCs (range endpoints); symbolize is on demand. */
+ load_summary_.touch_pcs = profile_sample_ranges_.size ();
+ for (const auto &r : profile_sample_ranges_)
+ if (r.second > r.first + 1)
+ ++load_summary_.touch_pcs;
+ /* Both are load scratch; the query API below never consults them. */
+ profile_cu_offs_.clear ();
+ profile_cu_offs_.shrink_to_fit ();
+ cu_end_by_off_.clear ();
+
+ close_libdw_session (dw_);
+ return true;
+}
+
+/* Deepest inline frame stack covering PC, or null if PC is not inlined. */
+const std::vector<inline_frame> *
+dwarf_reader_impl::inline_stack_at_pc (uint64_t pc) const
+{
+ if (inline_stack_spans_.empty ())
+ return nullptr;
+
+ /* Prefer more frames, then the tighter range. */
+ const inline_stack_span *best = nullptr;
+ visit_overlapping_ranges (
+ inline_stack_spans_, inline_stack_span_index_, pc, pc + 1,
+ [&] (size_t i)
+ {
+ const inline_stack_span &span = inline_stack_spans_[i];
+ if (!span.stack)
+ return;
+ if (best == nullptr || span.stack->size () > best->stack->size ()
+ || (span.stack->size () == best->stack->size ()
+ && span.end - span.start < best->end - best->start))
+ best = &span;
+ });
+ return best != nullptr ? best->stack.get () : nullptr;
+}
+
+/* One PC per distinct line-table span in [START, END); see dwarf_reader.h. */
+void
+dwarf_reader_impl::collect_line_span_seed_pcs (uint64_t start, uint64_t end,
+ std::vector<uint64_t> &out) const
+{
+ out.clear ();
+ std::unordered_set<uint64_t> seen;
+ seen.reserve (64);
+ visit_overlapping_ranges
+ (address_ranges_, address_range_index_, start, end, [&] (size_t i)
+ {
+ const address_range_entry &entry = address_ranges_[i];
+ const uint64_t seed_pc
+ = entry.line_table_pc != 0 ? entry.line_table_pc : entry.start;
+ if (seed_pc >= start && seed_pc < end && seen.insert (seed_pc).second)
+ out.push_back (seed_pc);
+ });
+}
+
+/* True when two adjacent segments may be coalesced into one span. */
+bool
+dwarf_reader_impl::same_line_table_coalesce_group (const address_range_entry
&a,
+ const address_range_entry &b)
+{
+ /* Each .debug_line Address column starts a distinct variant group; do not
+ merge across groups even when file:line:disc match (duplicate PCs). */
+ return a.line_table_pc == b.line_table_pc;
+}
+
+/* The run of DWARF functions whose entry PC is exactly ADDR. */
+dwarf_reader_impl::function_range
+dwarf_reader_impl::dwarf_functions_at_entry (uint64_t addr) const
+{
+ if (addr == 0)
+ return {dwarf_functions_.end (), dwarf_functions_.end ()};
+ struct by_start_addr
+ {
+ bool operator() (const function_metadata &f, uint64_t a) const noexcept
+ {
+ return f.start_addr < a;
+ }
+ bool operator() (uint64_t a, const function_metadata &f) const noexcept
+ {
+ return a < f.start_addr;
+ }
+ };
+ return std::equal_range
+ (dwarf_functions_.begin (), dwarf_functions_.end (), addr,
by_start_addr{});
+}
+
+/* DWARF definition named NAME entering at START_ADDR, or null. */
+const function_metadata *
+dwarf_reader_impl::find_function_at_entry (const std::string &name,
+ uint64_t start_addr) const
+{
+ if (name.empty ())
+ return nullptr;
+ const function_range entry = dwarf_functions_at_entry (start_addr);
+ for (auto it = entry.first; it != entry.second; ++it)
+ if (it->end_addr > it->start_addr && string_value (it->name) == name)
+ return &*it;
+ return nullptr;
+}
+
+/* INDEX entry for NAME, retrying under its GCC-suffix base name. */
+const std::vector<size_t> *
+dwarf_reader_impl::find_name_index (const name_index_map &index,
+ const std::string &name) const
+{
+ if (name.empty ())
+ return nullptr;
+ auto it = index.find (string_pool_.lookup (name));
+ if (it == index.end ())
+ {
+ /* Only a dotted name can carry a GCC suffix, and symbol_base_name
+ copies its argument, so screen the common case out first. */
+ if (name.find ('.') == std::string::npos)
+ return nullptr;
+ const std::string base = gcc_symbol::symbol_base_name (name);
+ if (base == name)
+ return nullptr;
+ it = index.find (string_pool_.lookup (base));
+ if (it == index.end ())
+ return nullptr;
+ }
+ return &it->second;
+}
+
+/* Source identity of the addressless ICF alias NAME; null when nothing
+ matches or the candidates disagree on file or line. */
+const function_metadata *
+dwarf_reader_impl::find_unique_alias_identity (const std::string &name) const
+{
+ const std::vector<size_t> *const indices
+ = find_name_index (dwarf_alias_name_index_, name);
+ if (indices == nullptr)
+ return nullptr;
+
+ const function_metadata *result = nullptr;
+ for (size_t idx : *indices)
+ {
+ if (idx >= dwarf_alias_identities_.size ())
+ continue;
+ const function_metadata &candidate = dwarf_alias_identities_[idx];
+ if (!gcc_symbol::name_equivalent (string_value (candidate.name), name))
+ continue;
+ if (result == nullptr)
+ {
+ result = &candidate;
+ continue;
+ }
+ const bool different_file
+ = !string_empty (result->file_path)
+ && !string_empty (candidate.file_path)
+ && !source_path::equivalent (string_value (result->file_path),
+ string_value (candidate.file_path));
+ const bool different_line = result->start_line != 0
+ && candidate.start_line != 0
+ && result->start_line != candidate.start_line;
+ if (different_file || different_line)
+ return nullptr;
+ if ((string_empty (result->file_path)
+ && !string_empty (candidate.file_path))
+ || (result->start_line == 0 && candidate.start_line != 0))
+ result = &candidate;
+ }
+ return result;
+}
+
+/* Declaration line for NAME at START_ADDR, given the DWARF definition
+ AT_ENTRY and alias identity IDENTITY already resolved for it (either may
+ be null). Split from the lookup below so callers that need those two
+ pointers anyway do not pay for them twice. */
+uint32_t
+dwarf_reader_impl::parse_time_decl_line
+ (const std::string &name, uint64_t start_addr,
+ const function_metadata *at_entry,
+ const function_metadata *identity) const
+{
+ if (at_entry != nullptr)
+ return at_entry->start_line;
+ if (identity != nullptr && identity->start_line != 0)
+ return identity->start_line;
+
+ if (name.empty ())
+ return 0;
+ const bool is_clone = has_gcc_generated_suffix (name);
+ const function_range entry = dwarf_functions_at_entry (start_addr);
+ uint32_t unnamed = 0;
+ bool unnamed_ambiguous = false;
+ for (auto it = entry.first; it != entry.second; ++it)
+ {
+ if (it->end_addr <= it->start_addr)
+ continue;
+ if (is_clone
+ && gcc_symbol::name_equivalent (string_value (it->name), name))
+ return it->start_line;
+ if (it->start_line == 0)
+ continue;
+ if (unnamed == 0)
+ unnamed = it->start_line;
+ else if (unnamed != it->start_line)
+ {
+ unnamed_ambiguous = true;
+ unnamed = std::min (unnamed, it->start_line);
+ }
+ }
+
+ /* No name match: use the entry's decl line when symtab/DWARF spell the same
+ body differently (clones, template mangling). Ambiguous decl lines at one
+ address mean ICF folded unrelated bodies; only clones may fall back. */
+ if (unnamed_ambiguous && !is_clone)
+ return 0;
+ return unnamed;
+}
+
+/* Declaring file and line for the function NAME entering at ADDR. */
+bool
+dwarf_reader_impl::function_profile_root_at_addr (
+ uint64_t addr, const std::string &name, source_location &out) const
+{
+ afdo_assert (mappings_loaded);
+ if (addr == 0)
+ return false;
+
+ const function_metadata *const at_entry = find_function_at_entry (name,
addr);
+ const function_metadata *const identity
+ = at_entry == nullptr ? find_unique_alias_identity (name) : nullptr;
+
+ out = source_location ();
+ if (!name.empty ())
+ out.line_number = parse_time_decl_line (name, addr, at_entry, identity);
+
+ /* An ICF alias left addressless by folding is the one case where the file
+ has to come from the name: the code at ADDR is the primary's, so both the
+ line table and the inline stack would name the primary's file. A
+ same-body alias (a C1/C2 constructor pair, a .localalias) does have a
+ definition at ADDR and shares the primary's file. */
+ if (identity != nullptr)
+ out.file_path = identity->file_path;
+
+ /* Profile-root file: outermost inline call file, else line-table row. */
+ source_location primary;
+ interned_string entry_file = nullptr;
+ if (string_empty (out.file_path))
+ {
+ if (lookup_primary_source_location_at_address (addr, primary))
+ entry_file = primary.file_path;
+
+ const std::vector<inline_frame> *inline_stack = inline_stack_at_pc
(addr);
+ if (inline_stack != nullptr && !inline_stack->empty ()
+ && !string_empty (inline_stack->front ().caller_file_path))
+ entry_file = inline_stack->front ().caller_file_path;
+
+ if (!string_empty (entry_file))
+ out.file_path = entry_file;
+ }
+
+ if (out.line_number == 0 && string_empty (out.file_path))
+ return false;
+ return true;
+}
+
+namespace {
+
+/* The declaring file and line agreed on by a set of equivalently-named
+ definitions, or nothing when they disagree. */
+struct decl_location_candidates
+{
+ interned_string file_path = nullptr;
+ uint32_t line_number = 0;
+ bool file_ambiguous = false;
+ bool line_ambiguous = false;
+
+ /* Fold FUNCTION in, marking the file or line ambiguous once two candidates
+ disagree. */
+ void add (const function_metadata &function)
+ {
+ if (!string_empty (function.file_path))
+ {
+ if (file_path == nullptr)
+ file_path = function.file_path;
+ else if (!source_path::equivalent
+ (string_value (file_path),
+ string_value (function.file_path)))
+ file_ambiguous = true;
+ }
+ if (function.start_line != 0)
+ {
+ if (line_number == 0)
+ line_number = function.start_line;
+ else if (line_number != function.start_line)
+ line_ambiguous = true;
+ }
+ }
+
+ source_location location () const
+ {
+ source_location result;
+ if (!file_ambiguous && !string_empty (file_path))
+ result.file_path = file_path;
+ if (!line_ambiguous)
+ result.line_number = line_number;
+ return result;
+ }
+};
+
+} /* anonymous namespace. */
+
+/* Declaring file and line of an inlined callee, filling in whatever the
+ frame itself does not carry by matching equivalently-named definitions. */
+source_location
+dwarf_reader_impl::resolve_inline_frame_decl_location (
+ const inline_frame &frame, const std::string &caller_file) const
+{
+ source_location location;
+ const std::string &name = string_value (frame.function_name);
+ const std::string &call_file = string_value (frame.caller_file_path);
+ location.file_path = frame.file_path;
+ location.line_number = frame.line_number;
+ if ((!string_empty (location.file_path) && location.line_number != 0)
+ || name.empty ())
+ return location;
+
+ const std::string &file_hint
+ = !string_empty (location.file_path)
+ ? string_value (location.file_path)
+ : (!call_file.empty () ? call_file : caller_file);
+ decl_location_candidates all;
+ decl_location_candidates contextual;
+
+ /* Every equivalently-named definition feeds ALL; those whose file matches
+ FILE_HINT also feed the preferred CONTEXTUAL set. */
+ const auto collect = [&] (const name_index_map &index,
+ const std::vector<function_metadata> &functions)
+ {
+ const std::vector<size_t> *const indices = find_name_index (index, name);
+ if (indices == nullptr)
+ return;
+ for (size_t idx : *indices)
+ {
+ if (idx >= functions.size ())
+ continue;
+ const function_metadata &function = functions[idx];
+ if (!gcc_symbol::name_equivalent (string_value (function.name), name))
+ continue;
+ all.add (function);
+ if (!file_hint.empty () && !string_empty (function.file_path)
+ && source_path::equivalent
+ (string_value (function.file_path), file_hint))
+ contextual.add (function);
+ }
+ };
+ collect (dwarf_func_name_index_, dwarf_functions_);
+ collect (symtab_equiv_index_, symtab_functions_);
+
+ source_location candidate = contextual.location ();
+ const source_location unique = all.location ();
+ if (string_empty (candidate.file_path))
+ candidate.file_path = unique.file_path;
+ if (candidate.line_number == 0)
+ candidate.line_number = unique.line_number;
+ if (string_empty (location.file_path))
+ location.file_path = candidate.file_path;
+ if (location.line_number == 0)
+ location.line_number = candidate.line_number;
+ return location;
+}
+
+/* All candidate locations for FILE_OFFSET, OUT[0] being the primary one and
+ DWARF_LINE_INDEX the row an addr2line-style consumer would report. */
+bool
+dwarf_reader_impl::build_correlate_locations_at_pc (
+ uint64_t file_offset, const function_metadata &func,
+ std::vector<source_location> &out, size_t &dwarf_line_index) const
+{
+ out.clear ();
+ dwarf_line_index = 0;
+
+ source_location primary;
+ if (!lookup_primary_source_location_at_address (file_offset, primary))
+ {
+ /* Line-table gap, for example a DW_AT_ranges hole: fall back to the
+ subprogram decl. */
+ source_location root;
+ if (!function_profile_root_at_addr
+ (file_offset, string_value (func.name), root))
+ return false;
+ if (!string_empty (root.file_path))
+ primary.file_path = root.file_path;
+ if (root.line_number != 0)
+ primary.line_number = root.line_number;
+ if (primary.line_number == 0 && string_empty (primary.file_path))
+ return false;
+ }
+ out.push_back (primary);
+
+ /* Collect exact-PC is_stmt rows; a unique one becomes OUT[0]. */
+ auto range_it
+ = std::lower_bound
+ (address_ranges_.begin (),
+ address_ranges_.end (),
+ file_offset,
+ [] (const address_range_entry &entry, uint64_t address)
+ {
+ return entry.start < address;
+ });
+ source_location stmt_location;
+ bool have_stmt_location = false;
+ bool ambiguous_stmt_location = false;
+ for (; range_it != address_ranges_.end ()
+ && range_it->start == file_offset;
+ ++range_it)
+ {
+ const address_range_entry &entry = *range_it;
+ if (!entry.line_table_is_stmt || entry.line_table_pc != file_offset)
+ continue;
+ if (entry.loc.line_number == 0 || string_empty (entry.loc.file_path))
+ continue;
+ if (!have_stmt_location)
+ {
+ stmt_location = entry.loc;
+ have_stmt_location = true;
+ }
+ else if (!source_location_equal (stmt_location, entry.loc))
+ ambiguous_stmt_location = true;
+ if (entry.loc.file_path == primary.file_path
+ && entry.loc.line_number == primary.line_number
+ && entry.loc.discriminator == primary.discriminator)
+ continue;
+ out.push_back (entry.loc);
+ }
+ if (have_stmt_location && !ambiguous_stmt_location
+ && !source_location_equal (out[0], stmt_location))
+ for (size_t i = 1; i < out.size (); ++i)
+ if (source_location_equal (out[i], stmt_location))
+ {
+ std::swap (out[0], out[i]);
+ dwarf_line_index = i;
+ break;
+ }
+ return true;
+}
+
+/* Function, source locations and inline stack at FILE_OFFSET. Results live
+ in SESSION's per-thread cache and stay valid until it is reset. */
+symbolized_result
+dwarf_reader_impl::symbolize (
+ uint64_t file_offset, correlate_session &session,
+ const function_metadata &containing_function) const
+{
+ afdo_assert (mappings_loaded);
+ symbolized_result result;
+
+ const auto cache_it = session.symbolize_cache_.find (file_offset);
+ if (cache_it != session.symbolize_cache_.end ())
+ {
+ result.function = &containing_function;
+ result.locations = &cache_it->second.locations;
+ result.inline_stack = cache_it->second.inline_stack;
+ return result;
+ }
+
+ const function_metadata *const func = &containing_function;
+ if (string_empty (func->name))
+ return result;
+
+ std::vector<source_location> locs;
+ size_t dwarf_line_index = 0;
+ if (!build_correlate_locations_at_pc
+ (file_offset, *func, locs, dwarf_line_index))
+ return result;
+
+ const std::vector<inline_frame> *const inline_stack
+ = inline_stack_at_pc (file_offset);
+ correlate_symbolize_entry entry;
+ entry.function_name = func->name;
+ entry.locations = std::move (locs);
+ entry.dwarf_line_index = dwarf_line_index;
+ entry.inline_stack = inline_stack;
+ const auto inserted
+ = session.symbolize_cache_.emplace (file_offset, std::move (entry));
+
+ result.function = func;
+ result.locations = &inserted.first->second.locations;
+ result.inline_stack = inline_stack;
+ return result;
+}
+
+/* Open libdw once for sample-scoped DWARF load. An absent, empty, or
+ non-overlapping .debug_aranges section is not a load failure. */
+bool
+dwarf_reader_impl::open_binary_dwarf ()
+{
+ if (dw_.dwarf != nullptr)
+ return true;
+
+ dwarf_functions_.clear ();
+ dwarf_alias_identities_.clear ();
+ symtab_functions_.clear ();
+ symtab_aliases_.clear ();
+ if (!open_libdw_session (binary_path_, dw_))
+ return false;
+
+ Dwarf_Aranges *aranges = nullptr;
+ size_t naranges = 0;
+ const bool have_aranges
+ = dwarf_getaranges (dw_.dwarf, &aranges, &naranges) == 0 && naranges != 0
+ && build_profile_cu_index (aranges, naranges)
+ && !profile_cu_offs_.empty ();
+ if (have_aranges)
+ return true;
+
+ /* .debug_aranges cannot name the sample-touch CUs, so index every one of
+ them; their last-row bounds fall back to function ends. */
+ cu_end_by_off_.clear ();
+ profile_cu_offs_ = all_cu_die_offsets ();
+ for (const Dwarf_Off cu_off : profile_cu_offs_)
+ cu_end_by_off_.emplace (cu_off, 0);
+ if (profile_cu_offs_.empty ())
+ {
+ fprintf (stderr,
+ _("Error: no DWARF compilation units in %s\n"),
+ binary_path_.c_str ());
+ close_libdw_session (dw_);
+ return false;
+ }
+ return true;
+}
+
+/* True when [LO, HI) meets any sampled range. */
+bool
+dwarf_reader_impl::range_overlaps_sampled (uint64_t lo, uint64_t hi) const
+{
+ if (profile_sample_ranges_.empty () || hi <= lo)
+ return false;
+ /* First interval whose exclusive end is past lo; it overlaps [lo,hi)
+ iff it also begins before hi. */
+ auto it = std::upper_bound
+ (profile_sample_ranges_.begin (),
+ profile_sample_ranges_.end (),
+ lo,
+ [] (uint64_t v, const std::pair<uint64_t, uint64_t> &r)
+ {
+ return v < r.second;
+ });
+ return it != profile_sample_ranges_.end () && it->first < hi;
+}
+
+/* True when any part of FUNC was sampled. */
+bool
+dwarf_reader_impl::function_overlaps_sampled
+ (const function_metadata &func) const
+{
+ if (func.end_addr <= func.start_addr)
+ return false;
+ return range_overlaps_sampled (func.start_addr, func.end_addr);
+}
+
+/* True when the line rows covering [RANGE_START, RANGE_END_EXCL) are worth
+ keeping: either the range itself was sampled, or it belongs to a
+ CU_LOCAL_FUNCTIONS entry that was, so a sampled function keeps its whole
+ line table including cold rows. */
+bool
+dwarf_reader_impl::address_range_overlaps_sampled
+ (uint64_t range_start, uint64_t range_end_excl,
+ const std::vector<function_metadata> &cu_local_functions) const
+{
+ if (range_end_excl <= range_start)
+ return false;
+ if (range_overlaps_sampled (range_start, range_end_excl))
+ return true;
+ /* Only the CU-local subprogram list is consulted; line rows belong to this
+ CU's DIE tree. */
+ for (const function_metadata &func : cu_local_functions)
+ {
+ if (func.end_addr <= func.start_addr)
+ continue;
+ if (func.start_addr < range_end_excl && range_start < func.end_addr)
+ return true;
+ }
+ return false;
+}
+
+/* Build sample-scoped CU index from .debug_aranges.
+
+ Output:
+ - cu_end_by_off_: highest arange end per CU, bounding its last line row
+ - profile_cu_offs_: deduped CU offsets to parse in build_address_range_index
+
+ Returns true when at least one arange overlapped the samples. */
+bool
+dwarf_reader_impl::build_profile_cu_index (Dwarf_Aranges *aranges,
+ size_t naranges)
+{
+ cu_end_by_off_.clear ();
+ profile_cu_offs_.clear ();
+ cu_end_by_off_.reserve (naranges);
+ profile_cu_offs_.reserve (naranges);
+ for (size_t i = 0; i < naranges; ++i)
+ {
+ Dwarf_Arange *const ar = dwarf_onearange (aranges, i);
+ if (ar == nullptr)
+ continue;
+ Dwarf_Addr ar_addr = 0;
+ Dwarf_Word ar_len = 0;
+ Dwarf_Off cu_off = 0;
+ if (dwarf_getarangeinfo (ar, &ar_addr, &ar_len, &cu_off) != 0
+ || ar_len == 0)
+ continue;
+ const uint64_t start = static_cast<uint64_t> (ar_addr);
+ const uint64_t end = start + static_cast<uint64_t> (ar_len);
+ if (end <= start)
+ continue;
+ if (!range_overlaps_sampled (start, end))
+ continue;
+ const auto inserted = cu_end_by_off_.emplace (cu_off, end);
+ if (inserted.second)
+ profile_cu_offs_.push_back (cu_off);
+ else if (end > inserted.first->second)
+ inserted.first->second = end;
+ }
+
+ return !profile_cu_offs_.empty ();
+}
+
+/* Every CU DIE offset in the binary, in .debug_info order. */
+std::vector<Dwarf_Off>
+dwarf_reader_impl::all_cu_die_offsets () const
+{
+ std::vector<Dwarf_Off> cu_dies;
+ for (Dwarf_Off off = 0;;)
+ {
+ Dwarf_Off next_off = 0;
+ size_t header_size = 0;
+ if (dwarf_nextcu (
+ dw_.dwarf, off, &next_off, &header_size, nullptr, nullptr, nullptr)
+ != 0)
+ break;
+ cu_dies.push_back (off + header_size);
+ if (next_off <= off)
+ break;
+ off = next_off;
+ }
+ return cu_dies;
+}
+
+/* Per-CU line table and DIE tree entry points. */
+
+/* Walk each DWARF line-row and decode the running line-table state. GCC often
+ emits several rows at the same PC (different file/line/disc); keep every raw
+ row so emit_cu_line_ranges can emit all variants instead of collapsing to a
+ single "best" row. */
+void
+dwarf_reader_impl::decode_cu_line_rows (Dwarf_Lines *lines, size_t nlines,
+ cu_lines &out) const
+{
+ out.line_rows.reserve (nlines);
+ /* Accumulators mirror the libdw out-param types (dwarf_lineno: int,
+ dwarf_linediscriminator: unsigned int); widen to the uint32_t row fields
+ at assignment only. */
+ int run_line = 0;
+ unsigned int run_disc = 0;
+ interned_string active_file_path = nullptr;
+ const char *last_src_file = nullptr;
+ std::unordered_map<const char *, interned_string> file_view_cache;
+
+ for (size_t i = 0; i < nlines; ++i)
+ {
+ Dwarf_Line *const srcl = dwarf_onesrcline (lines, i);
+ if (srcl == nullptr)
+ continue;
+
+ Dwarf_Addr row_addr = 0;
+ if (dwarf_lineaddr (srcl, &row_addr) != 0)
+ continue;
+
+ cu_line_row row;
+ row.addr = static_cast<uint64_t> (row_addr);
+
+ /* DW_LNE_end_sequence marks the address one past the last instruction of
+ the sequence and carries no source location. Keep the row so it
+ terminates the preceding segment, but give it no file/line: the bytes
+ from here to the next sequence belong to other objects (crt startup,
+ assembler stubs) and must not inherit this CU's last line. */
+ bool is_end_sequence = false;
+ if (dwarf_lineendsequence (srcl, &is_end_sequence) == 0
+ && is_end_sequence)
+ {
+ out.line_rows.push_back (row);
+ active_file_path = {};
+ last_src_file = nullptr;
+ run_line = 0;
+ run_disc = 0;
+ continue;
+ }
+
+ int ln = 0;
+ if (dwarf_lineno (srcl, &ln) == 0 && ln > 0)
+ run_line = ln;
+ if (run_line > 0)
+ row.run_line = static_cast<uint32_t> (run_line);
+
+ unsigned int row_disc = 0;
+ if (dwarf_linediscriminator (srcl, &row_disc) == 0)
+ run_disc = row_disc;
+ row.disc = run_disc;
+
+ bool is_stmt = false;
+ if (dwarf_linebeginstatement (srcl, &is_stmt) == 0)
+ row.is_stmt = is_stmt;
+
+ const char *const src_file = dwarf_linesrc (srcl, nullptr, nullptr);
+ if (src_file != nullptr && src_file[0] != '\0')
+ {
+ if (src_file != last_src_file)
+ {
+ const auto cached = file_view_cache.find (src_file);
+ if (cached != file_view_cache.end ())
+ active_file_path = cached->second;
+ else
+ {
+ active_file_path = string_pool_.intern (src_file);
+ if (!string_empty (active_file_path))
+ file_view_cache.emplace (src_file, active_file_path);
+ }
+ last_src_file = src_file;
+ }
+ }
+
+ if (!string_empty (active_file_path))
+ row.file_path = active_file_path;
+
+ out.line_rows.push_back (row);
+ }
+}
+
+/* Turn a CU's line-table rows into address ranges, ending each row at the
+ next row's address or, for the last one, at the enclosing function's end.
*/
+void
+dwarf_reader_impl::emit_cu_line_ranges (
+ cu_lines &out, uint64_t cu_end_excl,
+ const std::vector<function_metadata> &cu_local_functions) const
+{
+ for (size_t i = 0; i < out.line_rows.size ();)
+ {
+ const uint64_t group_addr = out.line_rows[i].addr;
+ size_t next_i = i;
+ while (next_i < out.line_rows.size ()
+ && out.line_rows[next_i].addr == group_addr)
+ ++next_i;
+
+ uint64_t end = 0;
+ if (next_i < out.line_rows.size ())
+ end = out.line_rows[next_i].addr;
+ else
+ {
+ if (cu_end_excl > group_addr)
+ end = cu_end_excl;
+ for (const function_metadata &func : cu_local_functions)
+ if (func.start_addr <= group_addr && func.end_addr > group_addr
+ && (end == 0 || func.end_addr < end))
+ end = func.end_addr;
+ }
+
+ if (end <= group_addr
+ || !address_range_overlaps_sampled (
+ group_addr, end, cu_local_functions))
+ {
+ i = next_i;
+ continue;
+ }
+
+ /* Collect distinct (file,line,disc) variants at one PC. */
+ struct line_variant_key
+ {
+ uint32_t line = 0;
+ uint32_t disc = 0;
+ interned_string file = nullptr;
+ uint32_t line_row_seq = 0;
+ bool is_stmt = false;
+ };
+ std::vector<line_variant_key> variants;
+ variants.reserve (next_i - i);
+ for (size_t j = i; j < next_i; ++j)
+ {
+ const cu_line_row &candidate = out.line_rows[j];
+ if (candidate.run_line == 0 && candidate.disc == 0
+ && string_empty (candidate.file_path))
+ continue;
+ line_variant_key key;
+ key.line = candidate.run_line;
+ key.disc = candidate.disc;
+ key.file = candidate.file_path;
+ key.line_row_seq = static_cast<uint32_t> (j);
+ key.is_stmt = candidate.is_stmt;
+ auto existing = std::find_if (variants.begin (),
+ variants.end (),
+ [&] (const line_variant_key &v)
+ {
+ return v.line == key.line
+ && v.file == key.file
+ && v.disc == key.disc;
+ });
+ if (existing == variants.end ())
+ variants.push_back (key);
+ else
+ {
+ existing->is_stmt = existing->is_stmt || key.is_stmt;
+ if (j >= existing->line_row_seq)
+ existing->line_row_seq = static_cast<uint32_t> (j);
+ }
+ }
+
+ const cu_line_row &last_row_at_pc = out.line_rows[next_i - 1];
+ const uint32_t last_row_line = last_row_at_pc.run_line;
+
+ for (const line_variant_key &variant : variants)
+ {
+ address_range_entry entry;
+ entry.start = group_addr;
+ entry.end = end;
+ entry.loc.file_path = variant.file;
+ entry.loc.line_number = variant.line;
+ entry.loc.discriminator = variant.disc;
+ entry.line_row_seq = variant.line_row_seq;
+ entry.line_table_is_stmt = variant.is_stmt;
+ entry.line_table_pc = group_addr;
+ entry.line_table_primary = variant.line == last_row_line
+ && variant.file == last_row_at_pc.file_path
+ && variant.disc == last_row_at_pc.disc;
+
+ if (!out.ranges.empty ())
+ {
+ const address_range_entry &prev = out.ranges.back ();
+ if (prev.start == entry.start && prev.end == entry.end
+ && source_location_equal (prev.loc, entry.loc))
+ continue;
+ }
+ out.ranges.push_back (std::move (entry));
+ }
+ i = next_i;
+ }
+}
+
+/* One subprogram DIE yields zero or more function_metadata rows. */
+void
+dwarf_reader_impl::record_subprogram_metadata (
+ Dwarf_Die *die, std::vector<function_metadata> &symbols_out) const
+{
+ if (die == nullptr || die_is_declaration (die))
+ return;
+ afdo_assert (dwarf_tag (die) == DW_TAG_subprogram);
+
+ pc_range_list pc_ranges;
+ uint64_t first_range_start = 0;
+ const pc_range_decode_status range_status
+ = collect_die_pc_ranges (die, pc_ranges, &first_range_start);
+ if (range_status != pc_range_decode_status::valid || pc_ranges.empty ())
+ return;
+
+ /* Supported ELF binaries do not place emitted code at address zero;
+ linkers use it for discarded COMDAT range relocations. */
+ pc_ranges.erase (
+ std::remove_if (pc_ranges.begin (), pc_ranges.end (),
+ [] (const std::pair<uint64_t, uint64_t> &range)
+ {
+ return range.first == 0;
+ }),
+ pc_ranges.end ());
+ if (pc_ranges.empty ())
+ return;
+
+ const char *const linkage_name = resolve_die_symbol_name (die);
+ if (linkage_name == nullptr || linkage_name[0] == '\0')
+ return;
+
+ uint32_t start_line = 0;
+ interned_string decl_file = nullptr;
+ const die_decl_site decl
+ = resolve_die_decl_site (die, &enclosing_type_by_die_);
+ if (decl.line_number != 0)
+ start_line = decl.line_number;
+ if (dwarf_source_file_is_usable (decl.file_path))
+ decl_file = string_pool_.intern (canonical_source_path (decl.file_path));
+
+ /* Attribute the function to the file where its body actually lives. GCC
+ keys a function by DECL_SOURCE_FILE (the definition file), so a
+ DW_AT_decl_file inherited from a header via DW_AT_specification would be
+ wrong here. For a function genuinely defined in a header
+ (inline/template) the entry-PC row is that header, which is correct. Fall
+ back to the definition DIE's own decl_file only when the line table has no
+ row at the entry PC. */
+ const uint64_t entry_pc
+ = subprogram_entry_pc (die, first_range_start, pc_ranges);
+ interned_string file_path = decl_file;
+ Dwarf_Die cu_mem;
+ Dwarf_Die *const cudie = dwarf_diecu (die, &cu_mem, nullptr, nullptr);
+ Dwarf_Line *const entry_row
+ = cudie == nullptr
+ ? nullptr
+ : dwarf_getsrc_die (cudie, static_cast<Dwarf_Addr> (entry_pc));
+ const char *const def_file = entry_row == nullptr
+ ? nullptr
+ : dwarf_linesrc (entry_row, nullptr, nullptr);
+ if (dwarf_source_file_is_usable (def_file))
+ file_path = string_pool_.intern (canonical_source_path (def_file));
+
+ for (const std::pair<uint64_t, uint64_t> &range : pc_ranges)
+ {
+ const uint64_t start = range.first;
+ const uint64_t end = range.second;
+ if (end <= start)
+ continue;
+ if (!range_overlaps_sampled (start, end))
+ continue;
+ symbols_out.emplace_back (
+ string_pool_.intern (linkage_name), file_path, start, end, start_line);
+ }
+}
+
+/* One inlined_subroutine DIE: preserve its effective range constraint and,
+ when its extent and name are usable, emit spans and push its frame.
+ Returns true when the caller must pop a frame after visiting children;
+ the caller always pops the range constraint. */
+bool
+dwarf_reader_impl::visit_inlined_subroutine_die
+ (Dwarf_Die &die, cu_die_parser &parser,
+ std::vector<inline_stack_span> &spans_out) const
+{
+ const pc_range_decode_status range_status
+ = collect_die_pc_ranges (&die, parser.pc_ranges);
+ const bool declared_extent
+ = range_status != pc_range_decode_status::no_extent;
+ const pc_range_list *const parent_ranges
+ = parser.inline_ranges_stack.empty ()
+ ? nullptr
+ : &parser.inline_ranges_stack.back ();
+
+ bool valid_extent
+ = !declared_extent
+ || (range_status == pc_range_decode_status::valid
+ && !parser.pc_ranges.empty ());
+ if (valid_extent && parent_ranges != nullptr && !parent_ranges->empty ()
+ && !parser.pc_ranges.empty ()
+ && !pc_ranges_subset_of (parser.pc_ranges, *parent_ranges))
+ {
+ parser.pc_ranges.clear ();
+ valid_extent = false;
+ }
+
+ /* Missing or rejected extents inherit the nearest valid ancestor
+ constraint. A usable extent narrows that constraint for descendants. */
+ pc_range_list effective_ranges;
+ if (!parser.pc_ranges.empty ())
+ effective_ranges = parser.pc_ranges;
+ else if (parent_ranges != nullptr)
+ effective_ranges = *parent_ranges;
+ parser.inline_ranges_stack.push_back (std::move (effective_ranges));
+
+ if (!valid_extent)
+ return false;
+
+ inline_frame frame;
+ frame.function_name = string_pool_.intern (resolve_die_symbol_name (&die));
+ if (string_empty (frame.function_name))
+ return false;
+ const die_decl_site decl
+ = resolve_die_decl_site (&die, &enclosing_type_by_die_);
+ if (dwarf_source_file_is_usable (decl.file_path))
+ frame.file_path
+ = string_pool_.intern (canonical_source_path (decl.file_path));
+ frame.line_number = decl.line_number;
+
+ /* DW_AT_call_* name the call site; no call_line means the DIE has none. */
+ const Dwarf_Word call_line = die_udata_attr (&die, DW_AT_call_line);
+ if (call_line != 0)
+ {
+ frame.call_line = static_cast<uint32_t> (call_line);
+ frame.discriminator = static_cast<uint32_t>
+ (die_udata_attr (&die, DW_AT_CALL_DISCRIMINATOR_EXT));
+ const char *const call_file = die_call_file_path (&die);
+ if (dwarf_source_file_is_usable (call_file))
+ frame.caller_file_path
+ = string_pool_.intern (canonical_source_path (call_file));
+ }
+ /* Keep recursive self-inlines; the DFS visits each DIE once. */
+ parser.inline_stack.push_back (frame);
+
+ if (!parser.pc_ranges.empty ())
+ {
+ auto stack = std::make_shared<const std::vector<inline_frame>> (
+ parser.inline_stack);
+ for (const auto &range : parser.pc_ranges)
+ spans_out.push_back ({range.first, range.second, stack});
+ }
+
+ return true;
+}
+
+/* Collect metadata that requires an all-CU walk: enclosing type declarations
+ and source identities for addressless ICF aliases. */
+void
+dwarf_reader_impl::record_global_die_metadata (Dwarf_Die *die,
+ global_die_walk &walk) const
+{
+ if (die == nullptr)
+ return;
+
+ const int tag = dwarf_tag (die);
+ const bool is_type
+ = (tag == DW_TAG_class_type
+ || tag == DW_TAG_structure_type
+ || tag == DW_TAG_union_type);
+ if (is_type)
+ {
+ int ln = 0;
+ die_decl_site site;
+ if (dwarf_decl_line (die, &ln) == 0 && ln > 0)
+ site.line_number = static_cast<uint32_t> (ln);
+ const char *const file = dwarf_decl_file (die);
+ if (dwarf_source_file_is_usable (file))
+ {
+ const interned_string interned
+ = string_pool_.intern (canonical_source_path (file));
+ if (interned != nullptr)
+ site.file_path = interned->c_str ();
+ }
+ walk.type_stack.push_back (site);
+ }
+ if ((tag == DW_TAG_subprogram || tag == DW_TAG_inlined_subroutine)
+ && !walk.type_stack.empty ())
+ walk.out->enclosing_types[dwarf_dieoffset (die)] = walk.type_stack.back ();
+ /* ICF can leave an alias definition addressless, so keep its name and decl
+ site even though the DIE carries no code range. */
+ if (tag == DW_TAG_subprogram && !walk.identity_names->empty ()
+ && !die_is_declaration (die))
+ {
+ const char *const raw_name = resolve_die_symbol_name (die);
+ const intern_string_set &wanted = *walk.identity_names;
+ if (raw_name != nullptr
+ && (wanted.find (raw_name) != wanted.end ()
+ || wanted.find (gcc_symbol::symbol_base_name (raw_name))
+ != wanted.end ()))
+ {
+ const die_decl_site decl
+ = resolve_die_decl_site (die, &walk.out->enclosing_types);
+ interned_string file_path = nullptr;
+ if (dwarf_source_file_is_usable (decl.file_path))
+ file_path
+ = string_pool_.intern (canonical_source_path (decl.file_path));
+ walk.out->alias_identities.emplace_back
+ (string_pool_.intern (raw_name), file_path, 0, 0, decl.line_number);
+ }
+ }
+
+ Dwarf_Die child;
+ if (dwarf_child (die, &child) == 0)
+ {
+ record_global_die_metadata (&child, walk);
+ Dwarf_Die sibling;
+ while (dwarf_siblingof (&child, &sibling) == 0)
+ {
+ record_global_die_metadata (&sibling, walk);
+ child = sibling;
+ }
+ }
+
+ if (is_type)
+ walk.type_stack.pop_back ();
+}
+
+/* Walk every CU to index enclosing type decls and ICF alias identities, the
+ two things that cannot be resolved from a single CU's DIEs alone. */
+void
+dwarf_reader_impl::build_global_die_metadata ()
+{
+ dwarf_alias_identities_.clear ();
+ intern_string_set identity_names;
+ for (const function_alias_metadata &alias : symtab_aliases_)
+ {
+ const std::string &name = string_value (alias.function.name);
+ identity_names.insert (name);
+ identity_names.insert (gcc_symbol::symbol_base_name (name));
+ }
+
+ const std::vector<Dwarf_Off> cu_dies = all_cu_die_offsets ();
+ const size_t n_cus = cu_dies.size ();
+ if (n_cus == 0)
+ return;
+
+ const unsigned num_threads = dwarf_parse_thread_count (n_cus);
+ std::vector<global_die_metadata> partials (n_cus);
+ std::atomic<size_t> next_cu{0};
+
+ /* Drain the shared CU queue through DWARF, into each CU's own slot. A
+ worker that cannot open its session simply leaves its share to the
+ others, so the return value adds nothing here. */
+ for_each_libdw_worker (num_threads, [&] (Dwarf *dwarf)
+ {
+ global_die_walk walk;
+ walk.identity_names = &identity_names;
+ for (;;)
+ {
+ const size_t i = next_cu.fetch_add (1, std::memory_order_relaxed);
+ if (i >= n_cus)
+ break;
+ Dwarf_Die cu_die;
+ if (dwarf_offdie (dwarf, cu_dies[i], &cu_die) == nullptr)
+ continue;
+ walk.type_stack.clear ();
+ walk.out = &partials[i];
+ record_global_die_metadata (&cu_die, walk);
+ }
+ });
+
+ for (global_die_metadata &part : partials)
+ {
+ enclosing_type_by_die_.insert
+ (part.enclosing_types.begin (), part.enclosing_types.end ());
+ dwarf_alias_identities_.insert
+ (dwarf_alias_identities_.end (),
+ std::make_move_iterator (part.alias_identities.begin ()),
+ std::make_move_iterator (part.alias_identities.end ()));
+ }
+}
+
+/* Depth-first walk of one CU DIE tree. */
+void
+dwarf_reader_impl::parse_cu_die_tree_recursive (
+ Dwarf_Die *die, std::vector<inline_stack_span> &spans_out,
+ std::vector<function_metadata> &symbols_out, cu_die_parser &parser) const
+{
+ if (die == nullptr)
+ return;
+
+ const int tag = dwarf_tag (die);
+
+ if (tag == DW_TAG_subprogram)
+ record_subprogram_metadata (die, symbols_out);
+
+ const bool is_inline_die = tag == DW_TAG_inlined_subroutine;
+ const bool inline_frame_pushed
+ = is_inline_die
+ && visit_inlined_subroutine_die (*die, parser, spans_out);
+
+ Dwarf_Die child;
+ if (dwarf_child (die, &child) == 0)
+ {
+ parse_cu_die_tree_recursive (&child, spans_out, symbols_out, parser);
+ Dwarf_Die sibling;
+ while (dwarf_siblingof (&child, &sibling) == 0)
+ {
+ parse_cu_die_tree_recursive
+ (&sibling, spans_out, symbols_out, parser);
+ child = sibling;
+ }
+ }
+
+ if (is_inline_die)
+ {
+ if (inline_frame_pushed)
+ parser.inline_stack.pop_back ();
+ parser.inline_ranges_stack.pop_back ();
+ }
+}
+
+/* Parse one CU at cu_off into OUT (returns false if offdie fails): decode the
+ line table first (cheap state machine), then walk the DIE tree. Line ranges
+ are emitted after the DIE walk so the function-span overlap filter can use
+ out.symbols. OUT must be freshly constructed. */
+bool
+dwarf_reader_impl::parse_cu_symbols (Dwarf *dwarf, Dwarf_Off cu_off,
+ cu_parse_symbols &out,
+ cu_die_parser &parser) const
+{
+ Dwarf_Die cudie;
+ if (dwarf == nullptr || dwarf_offdie (dwarf, cu_off, &cudie) == nullptr)
+ return false;
+
+ cu_lines &lines_out = out.lines;
+
+ Dwarf_Lines *lines = nullptr;
+ size_t nlines = 0;
+ if (dwarf_getsrclines (&cudie, &lines, &nlines) == 0 && nlines != 0)
+ decode_cu_line_rows (lines, nlines, lines_out);
+
+ parse_cu_die_tree_recursive (&cudie, out.inline_spans, out.symbols, parser);
+ afdo_assert (parser.inline_stack.empty ());
+ afdo_assert (parser.inline_ranges_stack.empty ());
+
+ if (lines_out.line_rows.empty ())
+ return true;
+
+ /* Sort by PC so duplicate-address groups are contiguous. */
+ std::stable_sort (lines_out.line_rows.begin (),
+ lines_out.line_rows.end (),
+ [] (const cu_line_row &a, const cu_line_row &b)
+ {
+ return a.addr < b.addr;
+ });
+
+ /* Bound the final line row by its CU or function. Without aranges the
+ stored CU end is 0; use the tightest DIE end, else one past the last
+ line-table address, so a trailing sampled row is not dropped. */
+ const auto cu_end_it = cu_end_by_off_.find (cu_off);
+ uint64_t cu_end_excl
+ = cu_end_it != cu_end_by_off_.end () ? cu_end_it->second : 0;
+ if (cu_end_excl == 0)
+ {
+ for (const function_metadata &func : out.symbols)
+ if (func.end_addr > cu_end_excl)
+ cu_end_excl = func.end_addr;
+ if (cu_end_excl == 0 && !lines_out.line_rows.empty ())
+ {
+ const uint64_t last = lines_out.line_rows.back ().addr;
+ if (last < std::numeric_limits<uint64_t>::max ())
+ cu_end_excl = last + 1;
+ }
+ }
+
+ /* Group rows by PC, expand distinct (file,line,disc) variants, and emit
+ sample-overlapping ranges into lines_out.ranges. */
+ emit_cu_line_ranges (lines_out, cu_end_excl, out.symbols);
+ std::vector<cu_line_row> ().swap (lines_out.line_rows);
+ return true;
+}
+
+/* Merge line, DIE, and metadata thread buffers for one CU. Metadata is merged
+ first so global function metadata is available to later CU work. Line-table
+ sample filtering already happened in parse_cu_symbols (via
+ cu_local_functions), not at merge time. */
+void
+dwarf_reader_impl::merge_cu_parse_symbols (cu_parse_symbols &buf)
+{
+ if (!buf.symbols.empty ())
+ {
+ dwarf_functions_.insert (dwarf_functions_.end (),
+ std::make_move_iterator (buf.symbols.begin ()),
+ std::make_move_iterator (buf.symbols.end ()));
+ buf.symbols.clear ();
+ }
+ const size_t cu_begin = address_ranges_.size ();
+ for (address_range_entry &entry : buf.lines.ranges)
+ {
+ if (address_ranges_.size () > cu_begin)
+ {
+ address_range_entry &prev = address_ranges_.back ();
+ if (prev.end == entry.start
+ && source_location_equal (prev.loc, entry.loc)
+ && same_line_table_coalesce_group (prev, entry))
+ {
+ prev.end = entry.end;
+ prev.line_table_is_stmt
+ = prev.line_table_is_stmt || entry.line_table_is_stmt;
+ continue;
+ }
+ }
+ address_ranges_.push_back (std::move (entry));
+ }
+ std::vector<address_range_entry> ().swap (buf.lines.ranges);
+ inline_stack_spans_.insert
+ (inline_stack_spans_.end (),
+ std::make_move_iterator (buf.inline_spans.begin ()),
+ std::make_move_iterator (buf.inline_spans.end ()));
+ buf.inline_spans.clear ();
+}
+
+/* How many threads to walk N_CUS compilation units with: the configured
+ count, capped so none of them sits idle. */
+unsigned
+dwarf_reader_impl::dwarf_parse_thread_count (size_t n_cus) const
+{
+ return static_cast<unsigned> (std::min<size_t> (n_cus, parse_threads_));
+}
+
+/* Run BODY on NUM_THREADS workers, giving each its own libdw session since
+ handles are not thread-safe, or running it inline on the reader's session
+ when a single worker is asked for. False when a worker could not reopen
+ the binary; the others still finish, as they share one work queue. */
+bool
+dwarf_reader_impl::for_each_libdw_worker (unsigned num_threads,
+ const libdw_worker_body &body) const
+{
+ const unsigned sessions = std::min (num_threads, CU_PARSE_MAX_SESSIONS);
+ if (sessions <= 1)
+ {
+ body (dw_.dwarf);
+ return true;
+ }
+
+ std::atomic<bool> session_failed{false};
+ std::vector<std::thread> workers;
+ workers.reserve (sessions);
+ for (unsigned t = 0; t < sessions; ++t)
+ workers.emplace_back ([&] ()
+ {
+ libdw_file_session session;
+ if (!open_libdw_session (binary_path_, session))
+ {
+ session_failed.store (true, std::memory_order_relaxed);
+ return;
+ }
+ body (session.dwarf);
+ close_libdw_session (session);
+ });
+ for (std::thread &th : workers)
+ th.join ();
+ return !session_failed.load (std::memory_order_relaxed);
+}
+
+/* Coalesce adjacent line segments that share the same source location. */
+void
+dwarf_reader_impl::merge_consecutive_address_ranges ()
+{
+ if (address_ranges_.empty ())
+ return;
+
+ std::vector<address_range_entry> merged;
+ merged.reserve (address_ranges_.size ());
+ merged.push_back (address_ranges_.front ());
+ for (size_t i = 1; i < address_ranges_.size (); ++i)
+ {
+ address_range_entry &prev = merged.back ();
+ const address_range_entry &cur = address_ranges_[i];
+ if (prev.end != cur.start)
+ {
+ merged.push_back (cur);
+ continue;
+ }
+ if (!source_location_equal (prev.loc, cur.loc)
+ || !same_line_table_coalesce_group (prev, cur))
+ {
+ merged.push_back (cur);
+ continue;
+ }
+ prev.end = cur.end;
+ prev.line_table_is_stmt
+ = prev.line_table_is_stmt || cur.line_table_is_stmt;
+ }
+ address_ranges_ = std::move (merged);
+}
+
+/* Parse profile-touch CUs into per-CU line/die/symbol buffers, then merge. */
+bool
+dwarf_reader_impl::parse_profile_cus (unsigned num_threads)
+{
+ const size_t n_cus = profile_cu_offs_.size ();
+ std::vector<cu_parse_symbols> partials (n_cus);
+ std::atomic<size_t> next_cu{0};
+
+ /* CUs vary hugely in size, so hand them out one at a time rather than in
+ fixed blocks. Each lands in its own slot, leaving the merge below in CU
+ order however the workers interleave. */
+ const bool sessions_opened
+ = for_each_libdw_worker (num_threads, [&] (Dwarf *dwarf)
+ {
+ cu_die_parser parser;
+ for (;;)
+ {
+ const size_t i = next_cu.fetch_add (1, std::memory_order_relaxed);
+ if (i >= n_cus)
+ break;
+ parser.reset_for_cu ();
+ partials[i].parsed = parse_cu_symbols
+ (dwarf, profile_cu_offs_[i], partials[i], parser);
+ }
+ });
+ if (!sessions_opened)
+ {
+ fprintf (stderr,
+ _("Error: failed to open libdw for parallel CU parse\n"));
+ return false;
+ }
+
+ for (size_t i = 0; i < n_cus; ++i)
+ if (!partials[i].parsed)
+ fprintf (stderr,
+ _("Warning: skipping DWARF CU at offset 0x%llx in %s\n"),
+ static_cast<unsigned long long> (profile_cu_offs_[i]),
+ binary_path_.c_str ());
+
+ size_t total_ranges = 0;
+ for (const cu_parse_symbols &part : partials)
+ total_ranges += part.lines.ranges.size ();
+ address_ranges_.reserve (address_ranges_.size () + total_ranges);
+ for (cu_parse_symbols &part : partials)
+ merge_cu_parse_symbols (part);
+ return true;
+}
+
+/* Parse the profile-relevant CUs and flatten their line segments into the
+ address-sorted address_ranges_ that every PC lookup searches. */
+bool
+dwarf_reader_impl::build_address_range_index ()
+{
+ afdo_assert (dw_.dwarf != nullptr && !profile_cu_offs_.empty ());
+
+ const size_t n_cus = profile_cu_offs_.size ();
+ const unsigned num_threads = dwarf_parse_thread_count (n_cus);
+
+ address_ranges_.clear ();
+ if (!parse_profile_cus (num_threads))
+ return false;
+
+ if (address_ranges_.empty ())
+ {
+ fprintf (stderr,
+ _("Error: no line segments from profile-touch CUs in %s\n"),
+ binary_path_.c_str ());
+ return false;
+ }
+
+ std::stable_sort (address_ranges_.begin (), address_ranges_.end ());
+ merge_consecutive_address_ranges ();
+ build_range_max_tree (address_ranges_, address_range_index_);
+ std::stable_sort (inline_stack_spans_.begin (), inline_stack_spans_.end ());
+ build_range_max_tree (inline_stack_spans_, inline_stack_span_index_);
+ mappings_loaded = true;
+ return true;
+}
+
+/* Load correlatable STT_FUNC/.cold/.__part symtab entries. */
+void
+dwarf_reader_impl::build_symtab_func_index ()
+{
+ symtab_functions_.clear ();
+ if (elf_version (EV_CURRENT) == EV_NONE)
+ elf_version (EV_CURRENT);
+
+ afdo_assert (dw_.dwarf != nullptr);
+ Elf *const elf = dwarf_getelf (dw_.dwarf);
+ if (elf == nullptr)
+ {
+ fprintf (stderr,
+ _("Error: cannot access ELF binary %s: %s\n"),
+ binary_path_.c_str (),
+ elf_errmsg (-1));
+ return;
+ }
+ if (elf_kind (elf) != ELF_K_ELF)
+ {
+ fprintf (stderr, _("Error: not an ELF file: %s\n"),
+ binary_path_.c_str ());
+ return;
+ }
+
+ std::vector<elf_func_sym> funcs;
+ funcs.reserve (16384);
+ Elf_Scn *scn = nullptr;
+ while ((scn = elf_nextscn (elf, scn)) != nullptr)
+ {
+ GElf_Shdr shdr;
+ if (gelf_getshdr (scn, &shdr) == nullptr || shdr.sh_type != SHT_SYMTAB)
+ continue;
+ if (shdr.sh_entsize == 0)
+ continue;
+
+ Elf_Data *const data = elf_getdata (scn, nullptr);
+ if (data == nullptr)
+ continue;
+
+ const size_t count = static_cast<size_t> (shdr.sh_size /
shdr.sh_entsize);
+ for (size_t i = 0; i < count; ++i)
+ {
+ if (i > static_cast<size_t> (INT_MAX))
+ break;
+ GElf_Sym sym;
+ if (gelf_getsym (data, static_cast<int> (i), &sym) == nullptr)
+ continue;
+ if (sym.st_value == 0 || sym.st_shndx == SHN_UNDEF)
+ continue;
+
+ const char *const raw = elf_strptr (elf, shdr.sh_link, sym.st_name);
+ const uint64_t size = static_cast<uint64_t> (sym.st_size);
+ if (!symtab_accepts_function_symbol (
+ raw, size, GELF_ST_TYPE (sym.st_info)))
+ continue;
+
+ const uint64_t start = static_cast<uint64_t> (sym.st_value);
+ const uint64_t end = start + size;
+ if (end <= start)
+ continue;
+
+ elf_func_sym row;
+ row.meta = function_metadata (
+ string_pool_.intern (raw), nullptr, start, end);
+ row.bind = GELF_ST_BIND (sym.st_info);
+ funcs.push_back (std::move (row));
+ }
+ }
+ /* Binding an external reference would resolve to. */
+ const auto bind_pref = [] (unsigned char bind)
+ {
+ return bind == STB_GLOBAL ? 2 : bind == STB_WEAK ? 1 : 0;
+ };
+
+ /* Shared span owner: strongest binding, then lexicographically first name.
+ That prefers GCC's unsuffixed parent (foo over foo.localalias /
foo.isra.0)
+ and is independent of symtab order. */
+ std::sort (funcs.begin (), funcs.end (),
+ [&bind_pref] (const elf_func_sym &a, const elf_func_sym &b)
+ {
+ if (a.meta.start_addr != b.meta.start_addr)
+ return a.meta.start_addr < b.meta.start_addr;
+ if (a.meta.end_addr != b.meta.end_addr)
+ return a.meta.end_addr < b.meta.end_addr;
+ const int pa = bind_pref (a.bind);
+ const int pb = bind_pref (b.bind);
+ if (pa != pb)
+ return pa > pb;
+ return string_value (a.meta.name) < string_value (b.meta.name);
+ });
+
+ /* Keep the preferred owner per range; retain aliases for export. */
+ symtab_aliases_.clear ();
+ for (size_t i = 0; i < funcs.size ();)
+ {
+ const uint64_t start = funcs[i].meta.start_addr;
+ const uint64_t end = funcs[i].meta.end_addr;
+ const size_t owner = symtab_functions_.size ();
+ symtab_functions_.push_back (std::move (funcs[i].meta));
+ ++i;
+ while (i < funcs.size () && funcs[i].meta.start_addr == start
+ && funcs[i].meta.end_addr == end)
+ {
+ if (funcs[i].meta.name != symtab_functions_[owner].name)
+ symtab_aliases_.push_back ({std::move (funcs[i].meta), owner});
+ ++i;
+ }
+ }
+
+ build_symtab_name_equivalence_index ();
+}
+
+/* Fill in the declaring file and line of every exported symtab function and
+ alias. */
+void
+dwarf_reader_impl::resolve_touch_symtab_export_anchors ()
+{
+ correlate_session session;
+ for (function_metadata &sym : symtab_functions_)
+ resolve_symtab_export_anchor (sym, session);
+ for (function_alias_metadata &alias : symtab_aliases_)
+ resolve_symtab_export_anchor (alias.function, session);
+}
+
+/* Give SYM the declaring file and line it will be keyed by in the profile. */
+void
+dwarf_reader_impl::resolve_symtab_export_anchor (
+ function_metadata &sym, correlate_session &session)
+{
+ if (sym.end_addr <= sym.start_addr)
+ return;
+
+ /* Unsampled symbols still get an empty profile, so they need a file to
+ stay distinct from a same-named static in another TU. */
+ const bool sampled = function_overlaps_sampled (sym);
+ source_location root;
+ const bool need_root
+ = string_empty (sym.file_path) || (sampled && sym.start_line == 0);
+ const bool have_root
+ = need_root
+ && function_profile_root_at_addr (
+ sym.start_addr, string_value (sym.name), root);
+ if (string_empty (sym.file_path) && have_root
+ && !string_empty (root.file_path))
+ sym.file_path = root.file_path;
+
+ /* Entry-line symbolize is only worth its cost where samples land. */
+ if (!sampled || sym.start_line != 0)
+ return;
+
+ /* ROOT already includes the parse-time declaration lookup. */
+ sym.start_line = root.line_number;
+ if (sym.start_line != 0)
+ return;
+
+ /* Symbolizing the entry PC is much the dearest of the three, so it runs
+ only once the other two have come up empty. */
+ const symbolized_result sym_result = symbolize (sym.start_addr, session,
sym);
+ if (sym_result.function != nullptr && sym_result.locations != nullptr
+ && !sym_result.locations->empty ())
+ sym.start_line = (*sym_result.locations)[0].line_number;
+}
+
+/* Index the DWARF definitions and alias identities by name for lookup. */
+void
+dwarf_reader_impl::build_dwarf_name_indexes ()
+{
+ dwarf_func_name_index_.clear ();
+ for (size_t i = 0; i < dwarf_functions_.size (); ++i)
+ {
+ const function_metadata &func = dwarf_functions_[i];
+ dwarf_func_name_index_[func.name].push_back (i);
+ const std::string base
+ = gcc_symbol::symbol_base_name (string_value (func.name));
+ if (base != string_value (func.name))
+ dwarf_func_name_index_[string_pool_.intern (base)].push_back (i);
+ }
+
+ dwarf_alias_name_index_.clear ();
+ for (size_t i = 0; i < dwarf_alias_identities_.size (); ++i)
+ {
+ const function_metadata &func = dwarf_alias_identities_[i];
+ dwarf_alias_name_index_[func.name].push_back (i);
+ const std::string base
+ = gcc_symbol::symbol_base_name (string_value (func.name));
+ if (base != string_value (func.name))
+ dwarf_alias_name_index_[string_pool_.intern (base)].push_back (i);
+ }
+}
+
+/* Index symtab functions under both their mangled and base names, so an ICF
+ alias and its primary resolve to the same entry. */
+void
+dwarf_reader_impl::build_symtab_name_equivalence_index ()
+{
+ symtab_equiv_index_.clear ();
+ symtab_equiv_index_.reserve (symtab_functions_.size () * 2);
+
+ const auto add_key = [this] (interned_string key, size_t i)
+ {
+ auto &vec = symtab_equiv_index_[key];
+ if (vec.empty () || vec.back () != i)
+ vec.push_back (i);
+ };
+ const auto add_name = [this, &add_key] (interned_string name, size_t i)
+ {
+ add_key (name, i);
+ const std::string base
+ = gcc_symbol::symbol_base_name (string_value (name));
+ if (base != string_value (name))
+ add_key (string_pool_.intern (base), i);
+ };
+
+ for (size_t i = 0; i < symtab_functions_.size (); ++i)
+ add_name (symtab_functions_[i].name, i);
+ for (const function_alias_metadata &alias : symtab_aliases_)
+ add_name (alias.function.name, alias.primary_index);
+}
+
+/* Source location of ADDRESS from the line table alone: the tightest segment
+ covering it, with rows whose line-table PC is exactly ADDRESS winning
+ outright. False when no segment covers it. */
+bool
+dwarf_reader_impl::lookup_primary_source_location_at_address (
+ uint64_t address, source_location &out) const
+{
+ afdo_assert (mappings_loaded);
+ afdo_assert (!address_ranges_.empty ());
+
+ bool have_exact_line_pc = false;
+ bool have_best = false;
+ source_location best_loc;
+ uint64_t best_start = 0;
+ uint32_t best_line_row_seq = 0;
+ bool best_line_table_primary = false;
+ visit_overlapping_ranges
+ (address_ranges_,
+ address_range_index_,
+ address,
+ address + 1,
+ [&] (size_t i)
+ {
+ const address_range_entry &entry = address_ranges_[i];
+ const bool exact_line_pc = entry.line_table_pc == address;
+ if (have_exact_line_pc && !exact_line_pc)
+ return;
+ if (exact_line_pc && !have_exact_line_pc)
+ {
+ have_exact_line_pc = true;
+ have_best = false;
+ }
+ if (address < entry.start || address >= entry.end)
+ return;
+
+ /* Keep the highest range start, then a primary line-table row, then a
+ later row. */
+ if (have_best)
+ {
+ if (entry.start < best_start)
+ return;
+ if (entry.start == best_start)
+ {
+ if (entry.line_table_primary != best_line_table_primary)
+ {
+ if (!entry.line_table_primary)
+ return;
+ }
+ else if (entry.line_row_seq <= best_line_row_seq)
+ return;
+ }
+ }
+
+ best_loc = entry.loc;
+ best_start = entry.start;
+ best_line_row_seq = entry.line_row_seq;
+ best_line_table_primary = entry.line_table_primary;
+ have_best = true;
+ });
+ if (!have_best)
+ return false;
+ out = best_loc;
+ return true;
+}
+
+dwarf_reader::dwarf_reader (std::string binary_path,
+ pc_range_list sampled_ranges)
+ : impl_ (std::make_unique<dwarf_reader_impl> (
+ std::move (binary_path), std::move (sampled_ranges)))
+{}
+
+dwarf_reader::~dwarf_reader () = default;
+
+void
+dwarf_reader::set_parse_threads (unsigned n)
+{
+ impl_->set_parse_threads (n);
+}
+
+bool
+dwarf_reader::load ()
+{
+ return impl_->load ();
+}
+
+const dwarf_load_summary &
+dwarf_reader::load_summary () const
+{
+ return impl_->load_summary ();
+}
+
+const std::vector<function_metadata> &
+dwarf_reader::symtab_functions () const
+{
+ return impl_->symtab_functions ();
+}
+
+const std::vector<function_alias_metadata> &
+dwarf_reader::symtab_aliases () const
+{
+ return impl_->symtab_aliases ();
+}
+
+const function_metadata *
+dwarf_reader::containing_function_at_pc (uint64_t address) const
+{
+ return impl_->containing_function_at_pc (address);
+}
+
+symbolized_result
+dwarf_reader::symbolize (uint64_t file_offset, correlate_session &session,
+ const function_metadata &containing_function) const
+{
+ return impl_->symbolize (file_offset, session, containing_function);
+}
+
+void
+dwarf_reader::collect_line_span_seed_pcs (uint64_t start, uint64_t end,
+ std::vector<uint64_t> &out) const
+{
+ impl_->collect_line_span_seed_pcs (start, end, out);
+}
+
+source_location
+dwarf_reader::resolve_inline_frame_decl_location (
+ const inline_frame &frame, const std::string &caller_file) const
+{
+ return impl_->resolve_inline_frame_decl_location (frame, caller_file);
+}
+
+} /* namespace dwarf_debug. */
diff --git a/afdo_tools/src/dwarf_reader_impl.h
b/afdo_tools/src/dwarf_reader_impl.h
new file mode 100644
index 00000000000..85c99d0e49e
--- /dev/null
+++ b/afdo_tools/src/dwarf_reader_impl.h
@@ -0,0 +1,359 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#ifndef GCC_DWARF_READER_IMPL_H
+#define GCC_DWARF_READER_IMPL_H
+
+#include "dwarf_reader.h"
+
+#include <algorithm>
+#include <array>
+#include <cstdint>
+#include <elfutils/libdw.h>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace dwarf_debug {
+
+/* CU parse workers to use when the caller does not pick a count. */
+constexpr unsigned CU_PARSE_DEFAULT_THREADS = 8;
+
+/* Concurrent libdw sessions to allow, however many workers were asked for.
+ Each session maps the whole binary, and resident pages are charged per
+ mapping, so sessions cost memory in a way plain threads do not. */
+constexpr unsigned CU_PARSE_MAX_SESSIONS = 32;
+
+/* Per-thread libdw open (handles are not thread-safe). */
+struct libdw_file_session
+{
+ int fd = -1;
+ Dwarf *dwarf = nullptr;
+};
+
+/* One worker of a parallel DWARF walk, called with that worker's own handle.
+ Workers share a queue, so each must be able to process any compilation
+ unit. */
+using libdw_worker_body = std::function<void (Dwarf *)>;
+
+/* Decl site of a DIE. FILE_PATH points into the interned string pool when
+ retained beyond the libdw operation that read it. */
+struct die_decl_site
+{
+ const char *file_path = nullptr;
+ uint32_t line_number = 0;
+};
+
+/* Interned name -> indices into a function table. */
+using name_index_map = std::unordered_map<interned_string,
std::vector<size_t>>;
+
+using intern_string_set = std::set<std::string, std::less<>>;
+
+/* Thread-safe interned-string pool (stable pointer identity). */
+class StringPool
+{
+public:
+ static constexpr size_t SHARD_COUNT = 32;
+
+ interned_string intern (const std::string &str);
+
+ /* As above, tolerating the null that DWARF and ELF readers hand back. */
+ interned_string intern (const char *str)
+ {
+ if (str == nullptr || str[0] == '\0')
+ return nullptr;
+ return intern (std::string (str));
+ }
+ interned_string lookup (const std::string &str) const;
+
+private:
+ static size_t shard (const std::string &str)
+ {
+ return std::hash<std::string>{}(str) % SHARD_COUNT;
+ }
+
+ std::array<intern_string_set, SHARD_COUNT> shards_{};
+ mutable std::array<std::mutex, SHARD_COUNT> shard_mus_{};
+};
+
+struct cu_line_row
+{
+ uint64_t addr = 0;
+ uint32_t run_line = 0;
+ uint32_t disc = 0;
+ interned_string file_path = nullptr;
+ bool is_stmt = false;
+};
+
+/* One PC range of an inlined_subroutine DIE; stack is shared across the
+ DIE's discontiguous ranges. */
+struct inline_stack_span
+{
+ uint64_t start = 0;
+ uint64_t end = 0;
+ std::shared_ptr<const std::vector<inline_frame>> stack;
+
+ bool operator< (const inline_stack_span &other) const
+ {
+ return start < other.start;
+ }
+};
+
+struct range_max_tree_index
+{
+ size_t leaf_base = 0;
+ std::vector<uint64_t> max_ends;
+};
+
+/* Sample-scoped DWARF + ELF symtab index (pimpl for dwarf_reader). */
+class dwarf_reader_impl
+{
+public:
+ /* Sampled [start,end) file-offset ranges (merged internally). */
+ explicit dwarf_reader_impl (std::string binary_path,
+ pc_range_list sampled_ranges);
+ ~dwarf_reader_impl ();
+
+ dwarf_reader_impl (const dwarf_reader_impl &) = delete;
+ dwarf_reader_impl &operator= (const dwarf_reader_impl &) = delete;
+
+ /* Lifecycle. */
+ void set_parse_threads (unsigned n);
+ bool load ();
+ const dwarf_load_summary &load_summary () const { return load_summary_; }
+
+ /* Symtab / containing function. */
+ /* Correlatable ELF symtab functions, sorted by start address. */
+ const std::vector<function_metadata> &symtab_functions () const
+ {
+ return symtab_functions_;
+ }
+ const std::vector<function_alias_metadata> &symtab_aliases () const
+ {
+ return symtab_aliases_;
+ }
+ const function_metadata *containing_function_at_pc (uint64_t address) const;
+
+ /* Symbolize + inline stack. */
+ symbolized_result
+ symbolize (uint64_t file_offset, correlate_session &session,
+ const function_metadata &containing_function) const;
+ void collect_line_span_seed_pcs (uint64_t start, uint64_t end,
+ std::vector<uint64_t> &out) const;
+ source_location
+ resolve_inline_frame_decl_location (const inline_frame &frame,
+ const std::string &caller_file) const;
+
+private:
+ /* Nested load / parse types. */
+ struct address_range_entry
+ {
+ uint64_t start = 0;
+ uint64_t end = 0;
+ source_location loc;
+ /* Index of the last .debug_line row (within the CU) that contributed this
+ (file,line) variant. */
+ uint32_t line_row_seq = 0;
+ /* Primary variant at a duplicate PC (final line-table row). */
+ bool line_table_primary = false;
+ bool line_table_is_stmt = false;
+ /* .debug_line Address column for this variant's duplicate-PC group. */
+ uint64_t line_table_pc = 0;
+
+ bool operator< (const address_range_entry &other) const
+ {
+ return start < other.start;
+ }
+ };
+
+ struct cu_lines
+ {
+ std::vector<cu_line_row> line_rows;
+ std::vector<address_range_entry> ranges;
+ };
+
+ struct cu_parse_symbols
+ {
+ cu_lines lines;
+ std::vector<inline_stack_span> inline_spans;
+ std::vector<function_metadata> symbols;
+ bool parsed = false;
+ };
+
+ /* Results of the all-CU metadata walk, accumulated one CU at a time. */
+ struct global_die_metadata
+ {
+ std::unordered_map<Dwarf_Off, die_decl_site> enclosing_types;
+ std::vector<function_metadata> alias_identities;
+ };
+
+ /* Inputs and scratch for the all-CU metadata walk; one instance per worker
+ thread. OUT points at the CU currently being walked: it is read back
+ during the walk, so it must not span CUs or results would depend on
+ thread scheduling. */
+ struct global_die_walk
+ {
+ /* Symtab alias names whose DWARF source identity is wanted. */
+ const intern_string_set *identity_names = nullptr;
+ /* Enclosing type decls of the DIEs currently open. */
+ std::vector<die_decl_site> type_stack;
+ global_die_metadata *out = nullptr;
+ };
+
+ /* Reusable working memory for the CU DIE-tree walk (per worker thread). */
+ struct cu_die_parser
+ {
+ pc_range_list pc_ranges;
+ std::vector<inline_frame> inline_stack;
+ /* Ancestor PC ranges; reject child ranges not a subset of the parent. */
+ std::vector<pc_range_list> inline_ranges_stack;
+
+ void reset_for_cu () noexcept
+ {
+ pc_ranges.clear ();
+ inline_stack.clear ();
+ inline_ranges_stack.clear ();
+ }
+ };
+
+ /* Load orchestration. */
+ unsigned dwarf_parse_thread_count (size_t n_cus) const;
+ bool for_each_libdw_worker (unsigned num_threads,
+ const libdw_worker_body &body) const;
+ bool open_binary_dwarf ();
+ bool build_profile_cu_index (Dwarf_Aranges *aranges, size_t naranges);
+ std::vector<Dwarf_Off> all_cu_die_offsets () const;
+ bool build_address_range_index ();
+ void resolve_touch_symtab_export_anchors ();
+ void resolve_symtab_export_anchor (function_metadata &sym,
+ correlate_session &session);
+
+ /* Sample-scope predicates. */
+ bool range_overlaps_sampled (uint64_t lo, uint64_t hi) const;
+ bool function_overlaps_sampled (const function_metadata &func) const;
+ bool address_range_overlaps_sampled (
+ uint64_t range_start, uint64_t range_end_excl,
+ const std::vector<function_metadata> &cu_local_functions) const;
+
+ /* CU parse + merge. */
+ bool parse_profile_cus (unsigned num_threads);
+ bool parse_cu_symbols (Dwarf *dwarf, Dwarf_Off cu_off, cu_parse_symbols &out,
+ cu_die_parser &parser) const;
+ void parse_cu_die_tree_recursive (Dwarf_Die *die,
+ std::vector<inline_stack_span> &spans_out,
+ std::vector<function_metadata> &symbols_out,
+ cu_die_parser &parser) const;
+ void record_global_die_metadata (Dwarf_Die *die, global_die_walk &walk)
const;
+ void build_global_die_metadata ();
+ void decode_cu_line_rows (Dwarf_Lines *lines, size_t nlines,
+ cu_lines &out) const;
+ void emit_cu_line_ranges (
+ cu_lines &out, uint64_t cu_end_excl,
+ const std::vector<function_metadata> &cu_local_functions) const;
+ void record_subprogram_metadata (
+ Dwarf_Die *die, std::vector<function_metadata> &symbols_out) const;
+ bool visit_inlined_subroutine_die
+ (Dwarf_Die &die, cu_die_parser &parser,
+ std::vector<inline_stack_span> &spans_out) const;
+ void merge_cu_parse_symbols (cu_parse_symbols &buf);
+ void merge_consecutive_address_ranges ();
+
+ /* Post-parse indexes. */
+ void build_symtab_func_index ();
+ void build_dwarf_name_indexes ();
+ void build_symtab_name_equivalence_index ();
+
+ /* Line-table / address-range lookup. */
+ const std::vector<inline_frame> *inline_stack_at_pc (uint64_t pc) const;
+ bool lookup_primary_source_location_at_address (uint64_t address,
+ source_location &out) const;
+ static bool same_line_table_coalesce_group (const address_range_entry &a,
+ const address_range_entry &b);
+
+ /* Symbolize (on-demand; session scratch). FUNC is the already-resolved
+ containing function. */
+ bool build_correlate_locations_at_pc (uint64_t file_offset,
+ const function_metadata &func,
+ std::vector<source_location> &out,
+ size_t &dwarf_line_index) const;
+
+ /* Decl helpers used by the public identity API. */
+ /* Indices for NAME, falling back to its GCC-suffix base name; nullptr when
+ neither is indexed. */
+ const std::vector<size_t> *find_name_index (const name_index_map &index,
+ const std::string &name) const;
+
+ using function_range
+ = std::pair<std::vector<function_metadata>::const_iterator,
+ std::vector<function_metadata>::const_iterator>;
+ /* Code-bearing DWARF subprograms entered at ADDR; ICF, aliasing, and COMDAT
+ merging can put several at one address. */
+ function_range dwarf_functions_at_entry (uint64_t addr) const;
+
+ const function_metadata *find_function_at_entry (const std::string &name,
+ uint64_t addr) const;
+ const function_metadata *
+ find_unique_alias_identity (const std::string &name) const;
+ uint32_t parse_time_decl_line (const std::string &name, uint64_t start_addr,
+ const function_metadata *at_entry,
+ const function_metadata *identity) const;
+ bool function_profile_root_at_addr (uint64_t addr, const std::string &name,
+ source_location &out) const;
+
+ /* Configuration / load state. */
+ std::string binary_path_;
+ pc_range_list profile_sample_ranges_;
+ bool mappings_loaded = false;
+ unsigned parse_threads_ = CU_PARSE_DEFAULT_THREADS;
+ dwarf_load_summary load_summary_;
+
+ /* Function tables and name indexes. */
+ std::vector<function_metadata> dwarf_functions_;
+ /* Name-only DWARF definitions for aliases whose code-bearing DIE was
+ eliminated by ICF; their address range remains zero. */
+ std::vector<function_metadata> dwarf_alias_identities_;
+ std::vector<function_metadata> symtab_functions_;
+ name_index_map dwarf_func_name_index_;
+ name_index_map dwarf_alias_name_index_;
+ name_index_map symtab_equiv_index_;
+ /* ELF names dropped by symtab range dedupe. */
+ std::vector<function_alias_metadata> symtab_aliases_;
+ /* Subprogram/inlined DIE offset -> enclosing class/struct/union decl. */
+ std::unordered_map<Dwarf_Off, die_decl_site> enclosing_type_by_die_;
+ /* Line segments and inline DIE spans. */
+ std::vector<address_range_entry> address_ranges_;
+ range_max_tree_index address_range_index_;
+ std::vector<inline_stack_span> inline_stack_spans_;
+ range_max_tree_index inline_stack_span_index_;
+
+ /* libdw session + CU aranges index. */
+ mutable StringPool string_pool_;
+ libdw_file_session dw_;
+ /* Both filled at the aranges scan and cleared after load. */
+ std::unordered_map<Dwarf_Off, uint64_t> cu_end_by_off_;
+ std::vector<Dwarf_Off> profile_cu_offs_;
+};
+
+} /* namespace dwarf_debug. */
+
+#endif /* GCC_DWARF_READER_IMPL_H. */
--
2.34.1
