Add mmap-backed parsing of perf.data headers, event records, attributes, and
address mappings. The parser exposes normalized event streams while keeping
sampling-mode interpretation in later readers.
afdo_tools/ChangeLog:
* src/perf_file_reader.cc: New file.
* src/perf_file_reader.h: New file.
Signed-off-by: Kugan Vivekanandarajah <[email protected]>
Co-authored-by: Dhruv Chawla <[email protected]>
Co-authored-by: Rahul Shinde <[email protected]>
---
afdo_tools/src/perf_file_reader.cc | 1101 ++++++++++++++++++++++++++++
afdo_tools/src/perf_file_reader.h | 176 +++++
2 files changed, 1277 insertions(+)
create mode 100644 afdo_tools/src/perf_file_reader.cc
create mode 100644 afdo_tools/src/perf_file_reader.h
diff --git a/afdo_tools/src/perf_file_reader.cc
b/afdo_tools/src/perf_file_reader.cc
new file mode 100644
index 00000000000..1c49ec24a15
--- /dev/null
+++ b/afdo_tools/src/perf_file_reader.cc
@@ -0,0 +1,1101 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#include "perf_file_reader.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+#include <cerrno>
+#include <unordered_map>
+#include <vector>
+
+#include <fcntl.h>
+#include <sys/mman.h>
+#include <unistd.h>
+
+#include <elf.h>
+#include <gelf.h>
+#include <libelf.h>
+
+namespace perf_data {
+namespace {
+#ifndef NT_GNU_BUILD_ID
+#define NT_GNU_BUILD_ID 3
+#endif
+
+/* Perf file format constants. */
+constexpr uint64_t PERF_MAGIC = 0x32454c4946524550ULL; /* "PERFILE2". */
+constexpr uint64_t PERF_MAGIC_SWAPPED
+ = 0x50455246494c4532ULL; /* Byte-swapped PERFILE2. */
+
+/* Lowercase hexadecimal digits for build IDs. */
+constexpr char HEX_DIGITS[] = "0123456789abcdef";
+
+/* User-space address validation constants. */
+constexpr uint64_t MIN_USER_ADDRESS = 0x1000;
+constexpr uint64_t KERNEL_ADDRESS_MASK = 0xFFFF000000000000ULL;
+
+/* Linux perf_event_attr field offsets. */
+constexpr size_t PERF_EVENT_ATTR_SAMPLE_TYPE_OFF = 24;
+constexpr size_t PERF_EVENT_ATTR_SIZE_FIELD_OFF = 4;
+constexpr size_t PERF_EVENT_ATTR_MIN_BYTES_FOR_SAMPLE_TYPE = 32;
+constexpr size_t PERF_EVENT_ATTR_SAMPLE_PERIOD_OFF = 16;
+constexpr size_t PERF_EVENT_ATTR_MIN_BYTES_THROUGH_SAMPLE_PERIOD = 24;
+constexpr size_t PERF_EVENT_ATTR_READ_FORMAT_OFF = 32;
+constexpr size_t PERF_EVENT_ATTR_MIN_BYTES_THROUGH_READ_FORMAT = 40;
+constexpr size_t PERF_EVENT_ATTR_FLAGS_OFF = 40;
+constexpr size_t PERF_EVENT_ATTR_MIN_BYTES_THROUGH_FLAGS = 48;
+constexpr uint64_t PERF_EVENT_ATTR_FREQ = 1ULL << 10;
+constexpr size_t PERF_EVENT_ATTR_BRANCH_SAMPLE_TYPE_OFF = 72;
+constexpr size_t PERF_EVENT_ATTR_MIN_BYTES_THROUGH_BRANCH_SAMPLE_TYPE = 80;
+constexpr size_t PERF_EVENT_ATTR_SAMPLE_REGS_USER_OFF = 80;
+constexpr size_t PERF_EVENT_ATTR_MIN_BYTES_THROUGH_SAMPLE_REGS_USER = 88;
+constexpr size_t PERF_EVENT_ATTR_SAMPLE_REGS_INTR_OFF = 96;
+constexpr size_t PERF_EVENT_ATTR_MIN_BYTES_THROUGH_SAMPLE_REGS_INTR = 104;
+constexpr uint16_t PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT = 1U << 12;
+/* The kernel reuses misc bit 13 with a meaning per record type. */
+constexpr uint16_t PERF_RECORD_MISC_MMAP_DATA = 1U << 13;
+constexpr uint16_t PERF_RECORD_MISC_COMM_EXEC = 1U << 13;
+constexpr uint16_t PERF_RECORD_MISC_MMAP_BUILD_ID = 1U << 14;
+constexpr uint16_t PERF_RECORD_MISC_BUILD_ID_SIZE = 1U << 15;
+constexpr size_t PERF_EVENT_ATTR_TYPE_OFF = 0;
+constexpr size_t PERF_EVENT_ATTR_CONFIG_OFF = 8;
+constexpr size_t PERF_EVENT_ATTR_MIN_BYTES_THROUGH_CONFIG = 16;
+constexpr uint32_t PERF_TYPE_SOFTWARE = 1;
+constexpr uint64_t PERF_COUNT_SW_DUMMY = 9;
+constexpr unsigned HEADER_COMPRESSED_FEATURE_BIT = 27;
+
+/* Bytes of ATTR that both the file and this reader agree are present. STRIDE
+ is the file's per-attribute stride, which also covers the trailing id
+ section, so the attribute's own size field bounds it. */
+uint64_t
+effective_attr_size (const uint8_t *attr, uint64_t stride)
+{
+ if (stride < PERF_EVENT_ATTR_SIZE_FIELD_OFF + sizeof (uint32_t))
+ return stride;
+ const uint32_t published = read_u32 (attr + PERF_EVENT_ATTR_SIZE_FIELD_OFF);
+ return published > 0 ? std::min<uint64_t> (stride, published) : stride;
+}
+
+/* True for the software dummy event perf adds to track mmaps for AUX and
+ system-wide records. It emits side-band records but never samples. */
+bool
+attr_is_dummy (const uint8_t *attr, uint64_t eff_size)
+{
+ if (eff_size < PERF_EVENT_ATTR_MIN_BYTES_THROUGH_CONFIG)
+ return false;
+ return read_u32 (attr + PERF_EVENT_ATTR_TYPE_OFF) == PERF_TYPE_SOFTWARE
+ && read_u64 (attr + PERF_EVENT_ATTR_CONFIG_OFF) == PERF_COUNT_SW_DUMMY;
+}
+
+attr_sample_layout
+read_attr_sample_layout (const uint8_t *attr, uint64_t eff_size)
+{
+ attr_sample_layout layout;
+ layout.eff_size = eff_size;
+ if (eff_size >= PERF_EVENT_ATTR_MIN_BYTES_THROUGH_SAMPLE_PERIOD)
+ layout.sample_period = read_u64 (attr + PERF_EVENT_ATTR_SAMPLE_PERIOD_OFF);
+ if (eff_size >= PERF_EVENT_ATTR_MIN_BYTES_FOR_SAMPLE_TYPE)
+ layout.sample_type = read_u64 (attr + PERF_EVENT_ATTR_SAMPLE_TYPE_OFF);
+ if (eff_size >= PERF_EVENT_ATTR_MIN_BYTES_THROUGH_READ_FORMAT)
+ layout.read_format = read_u64 (attr + PERF_EVENT_ATTR_READ_FORMAT_OFF);
+ if (eff_size >= PERF_EVENT_ATTR_MIN_BYTES_THROUGH_BRANCH_SAMPLE_TYPE)
+ layout.branch_sample_type
+ = read_u64 (attr + PERF_EVENT_ATTR_BRANCH_SAMPLE_TYPE_OFF);
+ if (eff_size >= PERF_EVENT_ATTR_MIN_BYTES_THROUGH_SAMPLE_REGS_USER)
+ layout.sample_regs_user
+ = read_u64 (attr + PERF_EVENT_ATTR_SAMPLE_REGS_USER_OFF);
+ if (eff_size >= PERF_EVENT_ATTR_MIN_BYTES_THROUGH_SAMPLE_REGS_INTR)
+ layout.sample_regs_intr
+ = read_u64 (attr + PERF_EVENT_ATTR_SAMPLE_REGS_INTR_OFF);
+ if (eff_size >= PERF_EVENT_ATTR_MIN_BYTES_THROUGH_FLAGS)
+ layout.fixed_sample_period
+ = layout.sample_period != 0
+ && (layout.sample_type & perf_sample_type::PERIOD) != 0
+ && (read_u64 (attr + PERF_EVENT_ATTR_FLAGS_OFF)
+ & PERF_EVENT_ATTR_FREQ) == 0;
+ return layout;
+}
+
+/* True when A and B's samples decode the same way. The period is not part
+ of this: a hybrid CPU gives each PMU its own default period. */
+bool
+sample_layouts_agree (const attr_sample_layout &a, const attr_sample_layout &b)
+{
+ return a.sample_type == b.sample_type && a.read_format == b.read_format
+ && a.branch_sample_type == b.branch_sample_type
+ && a.sample_regs_user == b.sample_regs_user
+ && a.sample_regs_intr == b.sample_regs_intr;
+}
+
+/* Perf appends this marker to mappings of unlinked files. */
+std::string
+normalize_mmap_filename (std::string filename)
+{
+ const size_t deleted = filename.find (" (deleted)");
+ if (deleted != std::string::npos)
+ filename.erase (deleted);
+ return filename;
+}
+
+struct segment_info
+{
+ uint64_t p_offset;
+ uint64_t p_filesz;
+ uint64_t p_vaddr;
+};
+
+struct elf_segment_cache
+{
+ std::vector<segment_info> segments;
+ bool is_pie = false;
+};
+
+std::unordered_map<std::string, elf_segment_cache> segment_cache;
+
+/* PT_LOAD table from ELF; empty on failure. */
+std::vector<segment_info>
+build_segment_table (Elf *elf)
+{
+ std::vector<segment_info> segments;
+ size_t phnum = 0;
+ if (elf_getphdrnum (elf, &phnum) != 0)
+ return segments;
+ for (size_t i = 0; i < phnum; ++i)
+ {
+ GElf_Phdr phdr;
+ if (gelf_getphdr (elf, static_cast<int> (i), &phdr) != &phdr)
+ continue;
+ if (phdr.p_type != PT_LOAD)
+ continue;
+ segments.push_back ({static_cast<uint64_t> (phdr.p_offset),
+ static_cast<uint64_t> (phdr.p_filesz),
+ static_cast<uint64_t> (phdr.p_vaddr)});
+ }
+ return segments;
+}
+
+/* p_vaddr for pgoff from the segment table. */
+bool
+lookup_vaddr (const std::vector<segment_info> &segments, uint64_t pgoff,
+ uint64_t &out)
+{
+ if (segments.empty ())
+ return false;
+
+ uint64_t fallback = 0;
+ bool have_fallback = false;
+ uint64_t first_nonzero_p_vaddr = 0;
+ bool have_first_nonzero = false;
+
+ for (const auto &s : segments)
+ {
+ if (!have_first_nonzero && s.p_vaddr != 0)
+ {
+ first_nonzero_p_vaddr = s.p_vaddr;
+ have_first_nonzero = true;
+ }
+
+ if (s.p_offset == pgoff)
+ {
+ out = s.p_vaddr;
+ return true;
+ }
+
+ if (s.p_offset <= pgoff && pgoff - s.p_offset < s.p_filesz)
+ {
+ const uint64_t offset = pgoff - s.p_offset;
+ if (!have_fallback && offset <= UINT64_MAX - s.p_vaddr)
+ {
+ fallback = s.p_vaddr + offset;
+ have_fallback = true;
+ }
+ }
+ }
+ if (have_fallback)
+ {
+ out = fallback;
+ return true;
+ }
+ /* pgoff 0 fallback: first nonzero PT_LOAD p_vaddr. */
+ if (pgoff == 0 && have_first_nonzero)
+ {
+ out = first_nonzero_p_vaddr;
+ return true;
+ }
+ return false;
+}
+
+/* Lowercase hex text for the BUILD_ID_LEN bytes at BUILD_ID. */
+std::string
+hex_encode_build_id (const uint8_t *build_id, size_t build_id_len)
+{
+ if (build_id == nullptr || build_id_len > SIZE_MAX / 2)
+ return {};
+ std::string encoded (build_id_len * 2, '\0');
+ for (size_t i = 0; i < build_id_len; i++)
+ {
+ encoded[2 * i] = HEX_DIGITS[build_id[i] >> 4];
+ encoded[2 * i + 1] = HEX_DIGITS[build_id[i] & 0x0f];
+ }
+ return encoded;
+}
+
+/* Parse .note.gnu.build-id from SHT_NOTE section payload. */
+bool
+gnu_build_id_from_note_data (Elf_Data *data, std::string &out)
+{
+ size_t offset = 0;
+ GElf_Nhdr nhdr;
+ size_t name_off, desc_off;
+ while ((offset = gelf_getnote (data, offset, &nhdr, &name_off, &desc_off))
+ != 0)
+ {
+ if (nhdr.n_type != NT_GNU_BUILD_ID)
+ continue;
+ const char *note_name
+ = static_cast<const char *> (data->d_buf) + name_off;
+ if (nhdr.n_namesz < 4 || memcmp (note_name, "GNU", 4) != 0)
+ continue;
+
+ const uint8_t *build_id
+ = static_cast<const uint8_t *> (data->d_buf) + desc_off;
+ size_t build_id_len = nhdr.n_descsz;
+ out = hex_encode_build_id (build_id, build_id_len);
+ return true;
+ }
+ return false;
+}
+
+/* Open BINARY_PATH read-only, retrying a relative path through realpath.
+ Returns -1 on failure. */
+int
+open_binary_read_only (const std::string &binary_path)
+{
+ int fd = open (binary_path.c_str (), O_RDONLY);
+ if (fd < 0 && !binary_path.empty () && binary_path[0] != '/')
+ {
+ char *resolved = realpath (binary_path.c_str (), nullptr);
+ if (resolved)
+ {
+ fd = open (resolved, O_RDONLY);
+ std::free (resolved);
+ }
+ }
+ return fd;
+}
+
+/* ELF .note.gnu.build-id as hex. */
+std::string
+get_buildid_from_binary (const std::string &binary_path)
+{
+ if (elf_version (EV_CURRENT) == EV_NONE)
+ return {};
+
+ int fd = open_binary_read_only (binary_path);
+ if (fd < 0)
+ return {};
+
+ Elf *elf = elf_begin (fd, ELF_C_READ, nullptr);
+ if (!elf)
+ {
+ close (fd);
+ return {};
+ }
+
+ if (elf_kind (elf) != ELF_K_ELF)
+ {
+ elf_end (elf);
+ close (fd);
+ return {};
+ }
+
+ Elf_Scn *scn = nullptr;
+ while ((scn = elf_nextscn (elf, scn)) != nullptr)
+ {
+ GElf_Shdr shdr;
+ if (gelf_getshdr (scn, &shdr) == nullptr)
+ continue;
+ if (shdr.sh_type != SHT_NOTE)
+ continue;
+
+ Elf_Data *data = elf_getdata (scn, nullptr);
+ if (data == nullptr || data->d_size == 0)
+ continue;
+
+ std::string decrypted;
+ if (gnu_build_id_from_note_data (data, decrypted))
+ {
+ elf_end (elf);
+ close (fd);
+ return decrypted;
+ }
+ }
+
+ elf_end (elf);
+ close (fd);
+ return {};
+}
+
+} /* anonymous namespace. */
+
+/* PT_LOAD p_vaddr for pgoff; false if not ELF or no match. */
+bool
+get_elf_segment_vaddr_for_pgoff (const std::string &binary_path, uint64_t
pgoff,
+ uint64_t &out)
+{
+ if (elf_version (EV_CURRENT) == EV_NONE)
+ return false;
+
+ auto it = segment_cache.find (binary_path);
+ if (it == segment_cache.end ())
+ {
+ int fd = open_binary_read_only (binary_path);
+ if (fd < 0)
+ return false;
+
+ /* libelf reads lazily, so FD must outlive every call up to elf_end. */
+ Elf *elf = elf_begin (fd, ELF_C_READ, nullptr);
+ if (!elf)
+ {
+ close (fd);
+ return false;
+ }
+
+ if (elf_kind (elf) != ELF_K_ELF)
+ {
+ elf_end (elf);
+ close (fd);
+ return false;
+ }
+
+ elf_segment_cache cache;
+ cache.segments = build_segment_table (elf);
+ GElf_Ehdr ehdr;
+ if (gelf_getehdr (elf, &ehdr) == &ehdr)
+ cache.is_pie = (ehdr.e_type == ET_DYN);
+ elf_end (elf);
+ close (fd);
+ it = segment_cache.emplace (binary_path, std::move (cache)).first;
+ }
+
+ return lookup_vaddr (it->second.segments, pgoff, out);
+}
+
+/* True if binary is PIE (ET_DYN); requires prior segment cache fill. */
+bool
+get_elf_is_pie (const std::string &binary_path)
+{
+ auto it = segment_cache.find (binary_path);
+ return it != segment_cache.end () && it->second.is_pie;
+}
+
+perf_file_reader::~perf_file_reader ()
+{
+ cleanup ();
+}
+
+bool
+perf_file_reader::load ()
+{
+ cleanup ();
+ if (!open_and_map_file ())
+ return false;
+
+ if (!parse_header ())
+ return false;
+
+ if (!parse_perf_data_section ())
+ return false;
+
+ if (lost_events_ != 0)
+ fprintf (stderr,
+ _("Warning: the kernel dropped %" PRIu64 " event(s) while "
+ "recording %s; the profile is incomplete\n"),
+ lost_events_,
+ perf_data_path.c_str ());
+
+ if (target_mmaps_by_pid_.empty ())
+ {
+ std::string base
+ = binary_path.substr (binary_path.find_last_of ("/\\") + 1);
+ fprintf (stderr,
+ _("Error: No memory mappings found for '%s'. perf.data may be "
+ "from a different binary. Ensure it was collected while "
+ "running: %s\n"),
+ base.c_str (),
+ binary_path.c_str ());
+ return false;
+ }
+
+ /* Samples were translated through their own PID's mappings as they were
+ ingested, so the mappings are no longer needed. */
+ clear_runtime_translation_state ();
+ return true;
+}
+
+bool
+perf_file_reader::open_and_map_file ()
+{
+ if (!mapped_file_.map (perf_data_path))
+ {
+ fprintf (stderr,
+ _("Failed to mmap perf.data file: %s (errno: %d)\n"),
+ perf_data_path.c_str (),
+ errno);
+ return false;
+ }
+ return true;
+}
+
+void
+perf_file_reader::cleanup ()
+{
+ mapped_file_.unmap ();
+ reset_profile_state ();
+ address_frequencies.clear ();
+ target_mmaps_by_pid_.clear ();
+ target_pid_by_tid_.clear ();
+ executable_mmaps_by_pid_.clear ();
+ target_mapping_pids_.clear ();
+ unstable_target_pids_.clear ();
+ buildid_table_.clear ();
+ buildid_table_valid_ = false;
+ cached_target_build_id_.clear ();
+ cached_target_build_id_valid_ = false;
+ reported_build_id_mismatch_ = false;
+ total_samples = 0;
+ lost_events_ = 0;
+ header = {};
+ sample_layout = attr_sample_layout ();
+}
+
+/* Parse perf.data file header. */
+bool
+perf_file_reader::parse_header ()
+{
+ if (mapped_file_.size () < sizeof (header))
+ {
+ fprintf (stderr, _("File too small to contain perf header\n"));
+ return false;
+ }
+
+ std::memcpy (&header, mapped_file_.data (), sizeof (header));
+
+ if (header.magic != PERF_MAGIC)
+ {
+ if (header.magic == PERF_MAGIC_SWAPPED)
+ fprintf (stderr,
+ _("Error: big-endian perf.data is not supported\n"));
+ else
+ fprintf (stderr,
+ _("Invalid perf.data magic number: 0x%" PRIx64 "\n"),
+ header.magic);
+ return false;
+ }
+
+ const uint8_t *const base = mapped_file_.data ();
+ if ((header.features >> HEADER_COMPRESSED_FEATURE_BIT) & 1ULL)
+ {
+ fprintf (stderr, _("Error: compressed perf.data is not supported\n"));
+ return false;
+ }
+
+ uint64_t sampling_attrs = 0;
+ if (header.attrs_size > 0 && header.attrs_offset > 0)
+ {
+ if (header.attrs_offset >= mapped_file_.size ()
+ || header.attrs_size > mapped_file_.size () - header.attrs_offset)
+ {
+ fprintf (stderr,
+ _("Error: perf.data attribute table is truncated\n"));
+ return false;
+ }
+
+ if (header.attr_size == 0)
+ {
+ fprintf (stderr,
+ _("Error: perf.data declares a zero attribute size\n"));
+ return false;
+ }
+
+ /* SAMPLE records carry no attribute id, so every sampling event has to
+ decode the same way. A hybrid CPU records one event per PMU. */
+ const uint8_t *const attrs = base + header.attrs_offset;
+ const uint64_t num_attrs = header.attrs_size / header.attr_size;
+ attr_sample_layout chosen;
+ bool periods_differ = false;
+
+ for (uint64_t i = 0; i < num_attrs; ++i)
+ {
+ const uint8_t *const attr = attrs + i * header.attr_size;
+ const uint64_t eff_size
+ = effective_attr_size (attr, header.attr_size);
+ if (attr_is_dummy (attr, eff_size))
+ continue;
+
+ const attr_sample_layout layout
+ = read_attr_sample_layout (attr, eff_size);
+ if (sampling_attrs != 0 && !sample_layouts_agree (layout, chosen))
+ {
+ fprintf (stderr,
+ _("Error: perf.data records sampling events that "
+ "disagree on sample layout; record one event at a "
+ "time\n"));
+ return false;
+ }
+ if (sampling_attrs != 0
+ && (layout.sample_period != chosen.sample_period
+ || layout.fixed_sample_period != chosen.fixed_sample_period))
+ periods_differ = true;
+ ++sampling_attrs;
+ chosen = layout;
+ }
+
+ sample_layout = chosen;
+ /* TODO: weight each sample by its own event's period. The events are
+ told apart by the id that PERF_SAMPLE_IDENTIFIER puts in every
+ sample. */
+ if (periods_differ)
+ fprintf (stderr,
+ _("Warning: perf.data records sampling events with "
+ "different periods; samples are still counted as equal "
+ "mass\n"));
+ }
+
+ if (sampling_attrs == 0)
+ {
+ fprintf (stderr, _("Error: perf.data has no sampling event\n"));
+ return false;
+ }
+
+ if ((sample_layout.sample_type & perf_sample_type::TID) == 0)
+ {
+ fprintf (stderr,
+ _("Error: perf.data samples must include PERF_SAMPLE_TID\n"));
+ return false;
+ }
+ /* Correlation gives every sample equal mass, which only reflects the
+ workload when the event counts on a fixed period. */
+ if (use_branch_stack_for_correlation_
+ && (sample_layout.sample_type & perf_sample_type::BRANCH_STACK) != 0
+ && !sample_layout.fixed_sample_period)
+ fprintf (stderr,
+ _("Warning: perf.data was not recorded with a fixed sampling "
+ "period; samples are still counted as equal mass\n"));
+
+ return true;
+}
+
+void
+perf_file_reader::clear_runtime_translation_state ()
+{
+ target_mmaps_by_pid_.clear ();
+ target_mmaps_by_pid_.rehash (0);
+ target_pid_by_tid_.clear ();
+ target_pid_by_tid_.rehash (0);
+ executable_mmaps_by_pid_.clear ();
+ executable_mmaps_by_pid_.rehash (0);
+ target_mapping_pids_.clear ();
+ unstable_target_pids_.clear ();
+ buildid_table_.clear ();
+ buildid_table_valid_ = false;
+ cached_target_build_id_.clear ();
+ cached_target_build_id_valid_ = false;
+}
+
+/* Invoke FN for every record of the perf.data data section. LOG selects the
+ diagnostic prefix. Returns false when the file is truncated or a malformed
+ record ends the scan early, so callers do not accept a prefix of the
+ profile as complete. Kernel-reported loss is tallied in LOST_EVENTS_. */
+bool
+perf_file_reader::for_each_mapped_perf_data_record (
+ const std::function<void (const perf_event_header &hdr,
+ const uint8_t *payload, size_t payload_size,
+ size_t skip_whole)> &fn,
+ perf_scan_log log)
+{
+ const char *const tag = (log == perf_scan_log::SPE ? "SPE" : "PERF");
+ if (header.data_offset > mapped_file_.size ())
+ {
+ fprintf (stderr,
+ _("%s: perf.data data section starts past end of file.\n"),
+ tag);
+ return false;
+ }
+ const size_t data_offset = static_cast<size_t> (header.data_offset);
+ const size_t available = mapped_file_.size () - data_offset;
+
+ bool complete = true;
+ if (header.data_size > available)
+ {
+ fprintf (stderr,
+ _("%s: perf.data is truncated: header declares %" PRIu64
+ " data bytes, %zu present.\n"),
+ tag,
+ header.data_size,
+ available);
+ complete = false;
+ }
+
+ const uint8_t *local_ptr = mapped_file_.data () + data_offset;
+ size_t local_rem = header.data_size > available
+ ? available
+ : static_cast<size_t> (header.data_size);
+ size_t total_records = 0;
+ uint64_t lost = 0;
+
+ while (local_rem >= sizeof (perf_event_header))
+ {
+ total_records++;
+ perf_event_header record_header;
+ std::memcpy (&record_header, local_ptr, sizeof (perf_event_header));
+ if (record_header.size == 0
+ || record_header.size < sizeof (perf_event_header)
+ || record_header.size > local_rem)
+ {
+ fprintf (stderr,
+ _("%s: Invalid record size (%u) at record %zu. "
+ "Stopping.\n"),
+ tag,
+ (unsigned) record_header.size,
+ total_records);
+ complete = false;
+ break;
+ }
+
+ const uint8_t *payload_ptr = local_ptr + sizeof (perf_event_header);
+ const size_t payload_size
+ = record_header.size - sizeof (perf_event_header);
+
+ if (record_header.type == perf_record_type::COMPRESSED
+ || record_header.type == perf_record_type::COMPRESSED2)
+ {
+ fprintf (stderr,
+ _("%s: compressed perf records are not supported\n"),
+ tag);
+ return false;
+ }
+ if ((record_header.type == perf_record_type::MMAP
+ || record_header.type == perf_record_type::MMAP2)
+ && (record_header.misc & PERF_RECORD_MISC_PROC_MAP_PARSE_TIMEOUT))
+ {
+ fprintf (stderr,
+ _("%s: process mapping scan timed out; the mapping set is "
+ "incomplete\n"),
+ tag);
+ return false;
+ }
+
+ const size_t skip
+ = perf_record_skip_bytes (record_header, payload_ptr, local_rem);
+ if (skip == 0)
+ {
+ fprintf (stderr,
+ _("%s: Invalid record advance at record %zu (type %u). "
+ "Stopping.\n"),
+ tag,
+ total_records,
+ (unsigned) record_header.type);
+ complete = false;
+ break;
+ }
+
+ /* PERF_RECORD_LOST is { u64 id; u64 lost; }; PERF_RECORD_LOST_SAMPLES
+ is { u64 lost; }. */
+ if (record_header.type == perf_record_type::LOST
+ && payload_size >= 2 * sizeof (uint64_t))
+ lost += read_u64 (payload_ptr + sizeof (uint64_t));
+ else if (record_header.type == perf_record_type::LOST_SAMPLES
+ && payload_size >= sizeof (uint64_t))
+ lost += read_u64 (payload_ptr);
+
+ local_ptr += skip;
+ local_rem -= skip;
+ fn (record_header, payload_ptr, payload_size, skip);
+ }
+
+ /* A well-formed data section ends on a record boundary. Only report the
+ tail when the scan itself found nothing else to complain about. */
+ if (complete && local_rem != 0)
+ {
+ fprintf (stderr,
+ _("%s: %zu trailing byte(s) after the last record.\n"),
+ tag,
+ local_rem);
+ complete = false;
+ }
+
+ /* Each pass sees the same records, so keep the largest tally. */
+ if (lost > lost_events_)
+ lost_events_ = lost;
+ return complete;
+}
+
+/* Bytes to advance past this record, counting the AUXTRACE inline tail.
+ Returns 0 when the record does not fit in DATA_REMAINING. */
+size_t
+perf_file_reader::perf_record_skip_bytes (
+ const perf_event_header &record_header, const uint8_t *payload,
+ size_t data_remaining) const
+{
+ if (record_header.size < sizeof (perf_event_header))
+ return 0;
+ if (record_header.size > data_remaining)
+ return 0;
+
+ if (record_header.type != static_cast<uint32_t> (perf_record_type::AUXTRACE))
+ return record_header.size;
+
+ const size_t payload_size = record_header.size - sizeof (perf_event_header);
+ if (payload_size < sizeof (uint64_t))
+ return record_header.size;
+
+ uint64_t inline_tail = read_u64 (payload);
+ const size_t base = record_header.size;
+ if (inline_tail > SIZE_MAX - base)
+ return 0;
+ const size_t total = base + static_cast<size_t> (inline_tail);
+ if (total > data_remaining)
+ return 0;
+ return total;
+}
+
+void
+perf_file_reader::observe_executable_mmap (uint32_t pid, uint64_t start,
+ uint64_t end, uint64_t pgoff,
+ bool is_target)
+{
+ std::vector<mmap_observation> &history = executable_mmaps_by_pid_[pid];
+ for (const mmap_observation &old : history)
+ {
+ if (start >= old.end || end <= old.start)
+ continue;
+ if (old.is_target != is_target
+ || (is_target
+ && (start != old.start || end != old.end
+ || pgoff != old.pgoff)))
+ unstable_target_pids_.insert (pid);
+ }
+ history.push_back ({start, end, pgoff, is_target});
+}
+
+void
+perf_file_reader::note_process_lifetime_record (
+ const perf_event_header &record_header, const uint8_t *payload,
+ size_t payload_size)
+{
+ if (payload == nullptr || payload_size < 2 * sizeof (uint32_t))
+ return;
+ if (record_header.type == perf_record_type::COMM
+ && (record_header.misc & PERF_RECORD_MISC_COMM_EXEC) != 0)
+ {
+ const uint32_t pid = read_u32 (payload);
+ if (target_mapping_pids_.count (pid) != 0)
+ unstable_target_pids_.insert (pid);
+ }
+}
+
+bool
+perf_file_reader::target_mapping_lifetime_valid () const
+{
+ if (unstable_target_pids_.empty ())
+ return true;
+ fprintf (stderr,
+ _("Error: perf.data changes the executable mapping lifetime for "
+ "a target process; this is not supported\n"));
+ /* TODO: Use sample_id_all timestamps and ordered per-PID mapping history
+ to support exec, unmap, PID reuse and cross-CPU record reordering. */
+ return false;
+}
+
+/* Record the target binary's half-open START to END range and resolve its
+ link-time address. Keep each mapping separate: pgoff alone does not
+ establish that two runtime intervals have the same affine translation. */
+void
+perf_file_reader::add_target_mmap (uint32_t pid, uint32_t tid,
+ const std::string &mmap_filename,
+ uint64_t start, uint64_t end, uint64_t pgoff)
+{
+ target_mapping_pids_.insert (pid);
+ std::string elf_path = normalize_mmap_filename (mmap_filename);
+ if (elf_path.empty ())
+ elf_path = binary_path;
+
+ uint64_t vaddr = 0;
+ bool have_vaddr
+ = get_elf_segment_vaddr_for_pgoff (elf_path, pgoff, vaddr);
+ if (!have_vaddr && elf_path != binary_path)
+ get_elf_segment_vaddr_for_pgoff (binary_path, pgoff, vaddr);
+ const bool is_pie
+ = get_elf_is_pie (elf_path) || get_elf_is_pie (binary_path);
+ target_mmaps_by_pid_[pid].emplace_back (
+ start, end, pgoff, vaddr, is_pie);
+ target_pid_by_tid_[tid] = pid;
+}
+
+bool
+perf_file_reader::parse_mmap_record (const uint8_t *record_data,
+ size_t record_size, uint16_t misc)
+{
+ if (record_size < sizeof (perf_mmap_record))
+ return false; /* Record too small. */
+
+ perf_mmap_record mmap;
+ std::memcpy (&mmap, record_data, sizeof (mmap));
+
+ /* Filename follows the fixed-size header; copy its bounded record bytes. */
+ const char *filename
+ = reinterpret_cast<const char *> (record_data + sizeof (perf_mmap_record));
+ const size_t filename_max_len = record_size - sizeof (perf_mmap_record);
+ const std::string mmap_filename
+ = perf_mmap_filename_view (filename, filename_max_len);
+
+ if (misc & PERF_RECORD_MISC_MMAP_DATA)
+ return true;
+ if (!is_valid_user_address (mmap.addr))
+ return true;
+
+ uint64_t end = 0;
+ if (!try_compute_mmap_end (mmap.addr, mmap.len, &end))
+ return true;
+ const bool is_target = is_target_binary (mmap_filename, "");
+ observe_executable_mmap (
+ mmap.pid, mmap.addr, end, mmap.pgoff, is_target);
+ if (!is_target)
+ return true;
+
+ add_target_mmap (
+ mmap.pid, mmap.tid, mmap_filename, mmap.addr, end, mmap.pgoff);
+ return true;
+}
+
+bool
+perf_file_reader::parse_mmap2_record (const uint8_t *record_data,
+ size_t record_size, uint16_t misc)
+{
+ if (record_size < sizeof (perf_mmap2_record))
+ return false; /* Record too small. */
+
+ perf_mmap2_record mmap2;
+ std::memcpy (&mmap2, record_data, sizeof (mmap2));
+
+ const char *filename
+ = reinterpret_cast<const char *> (record_data + sizeof
(perf_mmap2_record));
+ const size_t filename_max_len = record_size - sizeof (perf_mmap2_record);
+ const std::string mmap_filename
+ = perf_mmap_filename_view (filename, filename_max_len);
+
+ if ((mmap2.prot & PROT_EXEC) == 0)
+ return true;
+ if (!is_valid_user_address (mmap2.addr))
+ return true;
+
+ uint64_t end = 0;
+ if (!try_compute_mmap_end (mmap2.addr, mmap2.len, &end))
+ return true;
+
+ std::string build_id;
+ if (misc & PERF_RECORD_MISC_MMAP_BUILD_ID)
+ {
+ const size_t build_id_size = std::min<size_t> (
+ mmap2.id.build.build_id_size, sizeof (mmap2.id.build.build_id));
+ build_id = hex_encode_build_id (mmap2.id.build.build_id, build_id_size);
+ }
+ const bool is_target = is_target_binary (mmap_filename, build_id);
+ observe_executable_mmap (
+ mmap2.pid, mmap2.addr, end, mmap2.pgoff, is_target);
+ if (!is_target)
+ return true;
+
+ add_target_mmap (
+ mmap2.pid, mmap2.tid, mmap_filename, mmap2.addr, end, mmap2.pgoff);
+ return true;
+}
+
+/* Match a mapping against the profiled binary, by build ID where one is
+ known and by basename otherwise. */
+bool
+perf_file_reader::is_target_binary (const std::string &mmap_filename,
+ const std::string &build_id) const
+{
+ if (mmap_filename.empty ())
+ return false;
+ const std::string mmap_path = normalize_mmap_filename (mmap_filename);
+
+ if (!cached_target_build_id_valid_)
+ {
+ cached_target_build_id_ = get_buildid_from_binary (binary_path);
+ cached_target_build_id_valid_ = true;
+ }
+ if (!buildid_table_valid_)
+ parse_build_id_table ();
+
+ size_t tp = binary_path.find_last_of ("/\\");
+ std::string target_basename
+ = (tp != std::string::npos) ? binary_path.substr (tp + 1) : binary_path;
+ size_t mp = mmap_path.find_last_of ("/\\");
+ std::string mmap_basename
+ = (mp != std::string::npos) ? mmap_path.substr (mp + 1) : mmap_path;
+ const bool basename_matches = mmap_basename == target_basename;
+
+ /* Prefer the build ID from MMAP2, else the one HEADER_BUILD_ID recorded for
+ this path, which also covers a renamed or relocated binary. */
+ std::string recorded_build_id = build_id;
+ if (recorded_build_id.empty ())
+ {
+ const auto it = buildid_table_.find (mmap_path);
+ if (it != buildid_table_.end ())
+ recorded_build_id = it->second;
+ }
+
+ /* A recorded build ID decides the match whenever the requested binary has
+ one too; falling back to the basename would accept a different build. */
+ if (!recorded_build_id.empty () && !cached_target_build_id_.empty ())
+ {
+ if (recorded_build_id == cached_target_build_id_)
+ return true;
+ if (basename_matches)
+ report_build_id_mismatch (mmap_path, recorded_build_id);
+ return false;
+ }
+
+ return basename_matches;
+}
+
+/* Report once that perf.data profiled a different build of the binary named
+ by MMAP_PATH, which is otherwise the requested one. */
+void
+perf_file_reader::report_build_id_mismatch (
+ const std::string &mmap_path, const std::string &recorded_build_id) const
+{
+ if (reported_build_id_mismatch_)
+ return;
+ reported_build_id_mismatch_ = true;
+ fprintf (stderr,
+ _("Warning: perf.data profiled a different build of %s\n"
+ " build ID in perf.data: %s\n"
+ " build ID of %s: %s\n"),
+ mmap_path.c_str (),
+ recorded_build_id.c_str (),
+ binary_path.c_str (),
+ cached_target_build_id_.c_str ());
+}
+
+/* Reject kernel and very low addresses. */
+bool
+perf_file_reader::is_valid_user_address (uint64_t addr) const
+{
+ const bool is_kernel = (addr & KERNEL_ADDRESS_MASK) == KERNEL_ADDRESS_MASK;
+ return addr >= MIN_USER_ADDRESS && !is_kernel;
+}
+
+const std::vector<simple_mmap> *
+perf_file_reader::target_mmaps_for_pid (uint32_t pid) const
+{
+ const auto it = target_mmaps_by_pid_.find (pid);
+ return it == target_mmaps_by_pid_.end () ? nullptr : &it->second;
+}
+
+size_t
+perf_file_reader::target_mmap_count () const
+{
+ size_t count = 0;
+ for (const auto &entry : target_mmaps_by_pid_)
+ count += entry.second.size ();
+ return count;
+}
+
+/* Parse HEADER_BUILD_ID into a filename-to-hex-build-id map. */
+void
+perf_file_reader::parse_build_id_table () const
+{
+ buildid_table_valid_ = true;
+ if (!mapped_file_)
+ return;
+
+ constexpr int BUILD_ID_FEATURE_BIT = 2;
+ const uint64_t feats = header.features;
+ if (!((feats >> BUILD_ID_FEATURE_BIT) & 1ULL))
+ return;
+
+ const uint8_t *const base = mapped_file_.data ();
+ const size_t file_size = mapped_file_.size ();
+ if (header.data_offset > file_size
+ || header.data_size > file_size - header.data_offset)
+ return;
+ const uint64_t sec_table = header.data_offset + header.data_size;
+ /* Section index = number of feature bits set below bit 2. */
+ int index = 0;
+ for (int b = 0; b < BUILD_ID_FEATURE_BIT; ++b)
+ if ((feats >> b) & 1ULL)
+ ++index;
+ const uint64_t entry_off
+ = sec_table + static_cast<uint64_t> (index) * 16; /* {u64 off; u64 size;}
*/
+ if (entry_off > file_size || 16 > file_size - entry_off)
+ return;
+ uint64_t bid_off = read_u64 (base + entry_off);
+ uint64_t bid_size = read_u64 (base + entry_off + 8);
+ if (bid_size == 0 || bid_off > file_size || bid_size > file_size - bid_off)
+ return;
+
+ const uint8_t *p = base + bid_off;
+ const uint8_t *const end = p + bid_size;
+ /* build_id_event: hdr(8) + pid(4) + build_id[24] + filename. */
+ constexpr size_t BUILD_ID_NAME_OFF = sizeof (perf_event_header) + 4 + 24;
+ constexpr size_t BUILD_ID_OFF = sizeof (perf_event_header) + 4;
+ constexpr size_t BUILD_ID_DATA_BYTES = 20;
+ constexpr size_t BUILD_ID_SIZE_OFF = BUILD_ID_OFF + BUILD_ID_DATA_BYTES;
+ while (static_cast<size_t> (end - p) >= sizeof (perf_event_header))
+ {
+ perf_event_header h;
+ std::memcpy (&h, p, sizeof h);
+ if (h.size < sizeof (perf_event_header)
+ || h.size > static_cast<size_t> (end - p))
+ break;
+ if (h.size > BUILD_ID_NAME_OFF)
+ {
+ size_t build_id_len = BUILD_ID_DATA_BYTES;
+ if (h.misc & PERF_RECORD_MISC_BUILD_ID_SIZE)
+ {
+ build_id_len = p[BUILD_ID_SIZE_OFF];
+ if (build_id_len == 0 || build_id_len > BUILD_ID_DATA_BYTES)
+ {
+ p += h.size;
+ continue;
+ }
+ }
+ std::string hex
+ = hex_encode_build_id (p + BUILD_ID_OFF, build_id_len);
+ const char *fname
+ = reinterpret_cast<const char *> (p + BUILD_ID_NAME_OFF);
+ const size_t maxlen = h.size - BUILD_ID_NAME_OFF;
+ const size_t flen = ::strnlen (fname, maxlen);
+ if (flen != 0)
+ buildid_table_.emplace (
+ normalize_mmap_filename (std::string (fname, flen)),
+ std::move (hex));
+ }
+ p += h.size;
+ }
+}
+
+} /* namespace perf_data. */
diff --git a/afdo_tools/src/perf_file_reader.h
b/afdo_tools/src/perf_file_reader.h
new file mode 100644
index 00000000000..5fed76efe8f
--- /dev/null
+++ b/afdo_tools/src/perf_file_reader.h
@@ -0,0 +1,176 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#ifndef GCC_PERF_FILE_READER_H
+#define GCC_PERF_FILE_READER_H
+
+#include "file_mmap.h"
+#include "perf.h"
+#include "profile_reader.h"
+
+#include <cstdint>
+#include <functional>
+#include <map>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <utility>
+#include <vector>
+
+namespace perf_data {
+
+enum class sampling_mode : uint8_t
+{
+ BRANCH_STACK,
+ SPE
+};
+
+/* Scan-log prefix for record diagnostics. */
+enum class perf_scan_log : uint8_t
+{
+ GENERIC,
+ SPE
+};
+
+/* perf_event_attr fields that determine the SAMPLE record layout. */
+struct attr_sample_layout
+{
+ uint64_t eff_size = 0;
+ uint64_t sample_type = DEFAULT_BRANCH_STACK_SAMPLE_TYPE;
+ uint64_t read_format = 0;
+ uint64_t branch_sample_type = 0;
+ uint64_t sample_regs_user = 0;
+ uint64_t sample_regs_intr = 0;
+ uint64_t sample_period = 0;
+ /* True when every sample carries the same known mass: the event counts on a
+ fixed period and each sample reports that period. */
+ bool fixed_sample_period = false;
+};
+
+/* Offline perf.data loader; subclasses ingest samples. */
+class perf_file_reader : public profile_reader
+{
+public:
+ perf_file_reader (const std::string &_perf_data_path,
+ const std::string &_binary_path)
+ : perf_data_path (_perf_data_path), binary_path (_binary_path),
+ total_samples (0), header{}
+ {}
+
+ ~perf_file_reader () override;
+
+ bool load () override;
+ const std::unordered_map<uint64_t, uint64_t> &
+ get_address_frequencies () const override
+ {
+ return address_frequencies;
+ }
+ uint64_t get_total_samples () const override { return total_samples; }
+
+protected:
+ /* Lifecycle / I/O. */
+ bool open_and_map_file ();
+ void cleanup ();
+ bool parse_header ();
+ virtual bool parse_perf_data_section () = 0;
+ void clear_runtime_translation_state ();
+
+ /* Record scan. Returns false on truncation or a malformed record. */
+ bool for_each_mapped_perf_data_record (
+ const std::function<void (const perf_event_header &hdr,
+ const uint8_t *payload, size_t payload_size,
+ size_t skip_whole)> &fn,
+ perf_scan_log log = perf_scan_log::GENERIC);
+ size_t perf_record_skip_bytes (const perf_event_header &record_header,
+ const uint8_t *payload,
+ size_t data_remaining) const;
+ bool parse_mmap_record (const uint8_t *record_data, size_t record_size,
+ uint16_t misc);
+ bool parse_mmap2_record (const uint8_t *record_data, size_t record_size,
+ uint16_t misc);
+ void add_target_mmap (uint32_t pid, uint32_t tid,
+ const std::string &mmap_filename, uint64_t start,
+ uint64_t end, uint64_t pgoff);
+ void note_process_lifetime_record (const perf_event_header &header,
+ const uint8_t *payload,
+ size_t payload_size);
+ void observe_executable_mmap (uint32_t pid, uint64_t start, uint64_t end,
+ uint64_t pgoff, bool is_target);
+ bool target_mapping_lifetime_valid () const;
+
+ /* Target binary / address filters. */
+ bool is_target_binary (const std::string &mmap_filename,
+ const std::string &build_id) const;
+ void report_build_id_mismatch (const std::string &mmap_path,
+ const std::string &recorded_build_id) const;
+ bool is_valid_user_address (uint64_t addr) const;
+ const std::vector<simple_mmap> *target_mmaps_for_pid (uint32_t pid) const;
+ size_t target_mmap_count () const;
+ void parse_build_id_table () const;
+
+ const uint8_t *mapped_bytes () const { return mapped_file_.data (); }
+ size_t mapped_size () const { return mapped_file_.size (); }
+
+ /* Paths and sample mass. */
+ std::string perf_data_path;
+ std::string binary_path;
+ uint64_t total_samples;
+ std::unordered_map<uint64_t, uint64_t> address_frequencies;
+ pid_mmap_map target_mmaps_by_pid_;
+ std::unordered_map<uint32_t, uint32_t> target_pid_by_tid_;
+
+ struct mmap_observation
+ {
+ uint64_t start;
+ uint64_t end;
+ uint64_t pgoff;
+ bool is_target;
+ };
+ std::unordered_map<uint32_t, std::vector<mmap_observation>>
+ executable_mmaps_by_pid_;
+ std::unordered_set<uint32_t> target_mapping_pids_;
+ std::unordered_set<uint32_t> unstable_target_pids_;
+
+ /* Build-id cache and HEADER_BUILD_ID table. */
+ mutable std::string cached_target_build_id_;
+ mutable bool cached_target_build_id_valid_ = false;
+ mutable std::map<std::string, std::string, std::less<>> buildid_table_;
+ mutable bool buildid_table_valid_ = false;
+ mutable bool reported_build_id_mismatch_ = false;
+
+ /* Mapped perf.data and sample attributes. */
+ afdo_tools::file_mmap mapped_file_;
+ perf_file_header header;
+ attr_sample_layout sample_layout;
+ /* Events the kernel dropped while recording (PERF_RECORD_LOST*). */
+ uint64_t lost_events_ = 0;
+};
+
+std::unique_ptr<profile_reader>
+create_perf_reader (const std::string &perf_data_path,
+ const std::string &binary_path, sampling_mode mode);
+
+bool get_elf_segment_vaddr_for_pgoff (const std::string &binary_path,
+ uint64_t pgoff, uint64_t &out);
+
+bool get_elf_is_pie (const std::string &binary_path);
+
+} /* namespace perf_data. */
+
+#endif /* GCC_PERF_FILE_READER_H. */
--
2.34.1
