Add perf.data ARM SPE ingestion on top of the auxiliary packet decoder.
SPE records are translated into normalized address and branch information
before entering the common correlation pipeline, so SPE and branch-stack
profiles share the correlation and export paths. The afdo_create option
that selects SPE mode is added together with that command later in the
series.
afdo_tools/ChangeLog:
* src/perf_spe_reader.cc: New file.
* src/perf_spe_reader.h: New file.
Signed-off-by: Kugan Vivekanandarajah <[email protected]>
Co-authored-by: Dhruv Chawla <[email protected]>
Co-authored-by: Rahul Shinde <[email protected]>
---
afdo_tools/src/perf_spe_reader.cc | 488 ++++++++++++++++++++++++++++++
afdo_tools/src/perf_spe_reader.h | 68 +++++
2 files changed, 556 insertions(+)
create mode 100644 afdo_tools/src/perf_spe_reader.cc
create mode 100644 afdo_tools/src/perf_spe_reader.h
diff --git a/afdo_tools/src/perf_spe_reader.cc
b/afdo_tools/src/perf_spe_reader.cc
new file mode 100644
index 00000000000..d545c09cdc0
--- /dev/null
+++ b/afdo_tools/src/perf_spe_reader.cc
@@ -0,0 +1,488 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#include "perf_spe_reader.h"
+
+#include "arm_spe_aux_decode.h"
+
+#include <algorithm>
+#include <cinttypes>
+#include <cstdio>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+namespace perf_data {
+namespace {
+
+constexpr size_t PERF_RECORD_AUX_FIXED_BYTES = 3 * sizeof (uint64_t);
+constexpr uint64_t PERF_AUX_FLAG_TRUNCATED = 1ULL << 0;
+constexpr uint64_t PERF_AUX_FLAG_OVERWRITE = 1ULL << 1;
+constexpr uint64_t PERF_AUX_FLAG_PARTIAL = 1ULL << 2;
+constexpr uint64_t PERF_AUX_FLAG_COLLISION = 1ULL << 3;
+constexpr uint64_t PERF_AUX_DATA_LOSS_FLAGS
+ = PERF_AUX_FLAG_TRUNCATED | PERF_AUX_FLAG_OVERWRITE
+ | PERF_AUX_FLAG_PARTIAL | PERF_AUX_FLAG_COLLISION;
+
+/* Prefer synthesized range mass; else not-taken / SAMPLE PC mass. */
+uint64_t
+spe_profile_total_mass (
+ const branch_stack_edge_count_map &range_count_map,
+ const std::unordered_map<uint64_t, uint64_t> &address_frequencies)
+{
+ const uint64_t from_ranges
+ = total_mass_from_runtime_range_map (range_count_map);
+ if (from_ranges != 0)
+ return from_ranges;
+ return total_mass_from_address_frequencies (address_frequencies);
+}
+
+/* Add RUNTIME_COUNTS, translated through MMAPS, into DWARF_COUNTS, which may
+ already hold the counts of another process. */
+void
+translate_pc_counts_to_dwarf (
+ const std::unordered_map<uint64_t, uint64_t> &runtime_counts,
+ const std::vector<simple_mmap> &mmaps,
+ std::unordered_map<uint64_t, uint64_t> *dwarf_counts)
+{
+ for (const auto &entry : runtime_counts)
+ {
+ uint64_t off = 0;
+ if (!try_translate_runtime_with_mmaps (mmaps, entry.first, &off))
+ continue;
+ (*dwarf_counts)[off] += entry.second;
+ }
+}
+
+/* As TRANSLATE_PC_COUNTS_TO_DWARF, for branch edges. */
+void
+translate_edges_to_dwarf (const branch_stack_edge_count_map &runtime_edges,
+ const std::vector<simple_mmap> &mmaps,
+ branch_stack_edge_count_map *dwarf_edges)
+{
+ for (const auto &entry : runtime_edges)
+ {
+ uint64_t from = 0;
+ uint64_t to = 0;
+ if (!try_translate_runtime_with_mmaps (mmaps, entry.first.first, &from)
+ || !try_translate_runtime_with_mmaps (mmaps, entry.first.second, &to))
+ continue;
+ (*dwarf_edges)[{from, to}] += entry.second;
+ }
+}
+
+/* PERF_RECORD_AUXTRACE descriptor, excluding perf_event_header. */
+constexpr size_t PERF_AUXTRACE_DESCRIPTOR_BYTES
+ = 3 * sizeof (uint64_t) + 4 * sizeof (uint32_t);
+constexpr size_t PERF_AUXTRACE_OFFSET_OFF = sizeof (uint64_t);
+constexpr size_t PERF_AUXTRACE_IDX_OFF = 3 * sizeof (uint64_t);
+constexpr size_t PERF_AUXTRACE_TID_OFF
+ = PERF_AUXTRACE_IDX_OFF + sizeof (uint32_t);
+
+/* AUXTRACE payload after the descriptor; empty tail is ok. */
+bool
+spe_auxtrace_tail (const perf_event_header &hdr, const uint8_t *payload,
+ size_t skip, const uint8_t *file_base, size_t file_size,
+ const uint8_t **tail_out, size_t *tail_size_out,
+ uint64_t *aux_offset_out, uint32_t *idx_out,
+ uint32_t *tid_out)
+{
+ if (payload == nullptr || file_base == nullptr || tail_out == nullptr
+ || tail_size_out == nullptr || aux_offset_out == nullptr
+ || idx_out == nullptr || tid_out == nullptr || skip < hdr.size
+ || hdr.size < sizeof (perf_event_header)
+ + PERF_AUXTRACE_DESCRIPTOR_BYTES)
+ return false;
+
+ const size_t tail = skip - hdr.size;
+ const size_t payload_size = hdr.size - sizeof (perf_event_header);
+ if (read_u64 (payload) != tail
+ || payload_size < PERF_AUXTRACE_DESCRIPTOR_BYTES)
+ return false;
+ if (payload < file_base)
+ return false;
+ const size_t payload_offset = static_cast<size_t> (payload - file_base);
+ if (payload_offset > file_size || payload_offset < sizeof
(perf_event_header))
+ return false;
+ const size_t record_offset = payload_offset - sizeof (perf_event_header);
+ if (hdr.size > file_size - record_offset)
+ return false;
+ const size_t tail_offset = record_offset + hdr.size;
+ if (tail > file_size - tail_offset)
+ return false;
+
+ *tail_out = file_base + tail_offset;
+ *tail_size_out = tail;
+ *aux_offset_out = read_u64 (payload + PERF_AUXTRACE_OFFSET_OFF);
+ *idx_out = read_u32 (payload + PERF_AUXTRACE_IDX_OFF);
+ *tid_out = read_u32 (payload + PERF_AUXTRACE_TID_OFF);
+ return true;
+}
+
+struct spe_stream_state
+{
+ arm_spe_stream_decoder decoder;
+ uint64_t next_aux_offset = 0;
+ uint32_t pid = 0;
+ bool have_offset = false;
+ bool have_pid = false;
+};
+
+} /* anonymous namespace. */
+
+perf_spe_reader::perf_spe_reader (const std::string &_perf_data_path,
+ const std::string &_binary_path)
+ : perf_file_reader (_perf_data_path, _binary_path)
+{}
+
+bool
+perf_spe_reader::accept_target_addr_cb (uint64_t runtime_addr, void *ctx)
+{
+ const perf_spe_reader *const reader = static_cast<perf_spe_reader *> (ctx);
+ return reader->active_mmaps_ != nullptr
+ && target_mmaps_contain (*reader->active_mmaps_, runtime_addr);
+}
+
+void
+perf_spe_reader::note_timestamp_cb (uint64_t runtime_addr, uint64_t timestamp,
+ void *ctx)
+{
+ perf_spe_reader *const reader = static_cast<perf_spe_reader *> (ctx);
+ uint64_t dwarf_addr = 0;
+ if (reader->active_mmaps_ != nullptr
+ && try_translate_runtime_with_mmaps (
+ *reader->active_mmaps_, runtime_addr, &dwarf_addr))
+ reader->note_sample_timestamp (dwarf_addr, timestamp);
+}
+
+bool
+perf_spe_reader::parse_perf_data_section ()
+{
+ if (header.data_offset > mapped_size ())
+ {
+ fprintf (stderr,
+ _("SPE: perf.data data section starts past end of file "
+ "(data_offset=%" PRIu64 ", file_size=%zu).\n"),
+ header.data_offset,
+ mapped_size ());
+ return false;
+ }
+
+ size_t auxtrace_records = 0;
+ size_t aux_chunks = 0;
+ size_t mmap_records_seen = 0;
+ bool spe_aux_decode_ok = true;
+ bool aux_metadata_ok = true;
+ uint64_t aux_loss_records = 0;
+ uint64_t aux_truncated_records = 0;
+ uint64_t aux_overwrite_records = 0;
+ uint64_t aux_partial_records = 0;
+ uint64_t aux_collision_records = 0;
+ uint64_t aux_forward_gaps = 0;
+ uint64_t aux_overlaps = 0;
+
+ reset_branch_stack_profile ();
+ range_count_map_.clear ();
+ branch_count_map_.clear ();
+ address_frequencies.clear ();
+
+ /* Pass 1: bind MMAPs before address filtering (same reason as LBR). */
+ if (!for_each_mapped_perf_data_record (
+ [&] (const perf_event_header &hdr,
+ const uint8_t *payload,
+ size_t payload_size,
+ size_t)
+ {
+ if (hdr.type == perf_record_type::MMAP
+ || hdr.type == perf_record_type::MMAP2)
+ {
+ if (hdr.type == perf_record_type::MMAP)
+ parse_mmap_record (payload, payload_size, hdr.misc);
+ else
+ parse_mmap2_record (payload, payload_size, hdr.misc);
+ ++mmap_records_seen;
+ }
+ else if (hdr.type == perf_record_type::AUX)
+ {
+ if (payload_size < PERF_RECORD_AUX_FIXED_BYTES)
+ aux_metadata_ok = false;
+ else
+ {
+ const uint64_t flags
+ = read_u64 (payload + 2 * sizeof (uint64_t));
+ if ((flags & PERF_AUX_DATA_LOSS_FLAGS) != 0)
+ ++aux_loss_records;
+ if ((flags & PERF_AUX_FLAG_TRUNCATED) != 0)
+ ++aux_truncated_records;
+ if ((flags & PERF_AUX_FLAG_OVERWRITE) != 0)
+ ++aux_overwrite_records;
+ if ((flags & PERF_AUX_FLAG_PARTIAL) != 0)
+ ++aux_partial_records;
+ if ((flags & PERF_AUX_FLAG_COLLISION) != 0)
+ ++aux_collision_records;
+ }
+ }
+ else if (hdr.type == perf_record_type::COMM)
+ note_process_lifetime_record (hdr, payload, payload_size);
+ },
+ perf_scan_log::SPE))
+ return false;
+ if (!target_mapping_lifetime_valid ())
+ return false;
+ if (target_mmaps_by_pid_.empty ())
+ {
+ fprintf (stderr,
+ _("SPE: no target mappings (saw %zu MMAP/MMAP2, none matched "
+ "-b %s)\n"),
+ mmap_records_seen,
+ binary_path.c_str ());
+ return false;
+ }
+
+ if (!aux_metadata_ok)
+ {
+ fprintf (stderr, _("SPE: truncated PERF_RECORD_AUX metadata\n"));
+ return false;
+ }
+ if (aux_loss_records != 0)
+ fprintf (stderr,
+ _("Warning: %" PRIu64 " SPE AUX records report data loss "
+ "(truncated=%" PRIu64 ", overwrite=%" PRIu64 ", partial=%"
+ PRIu64 ", collision=%" PRIu64 ")\n"),
+ aux_loss_records, aux_truncated_records, aux_overwrite_records,
+ aux_partial_records, aux_collision_records);
+
+ /* Pass 2: decode AUXTRACE SPE tails into address_frequencies. */
+ arm_spe_profile_accumulator acc{};
+ acc.accept_addr = accept_target_addr_cb;
+ acc.note_timestamp = note_timestamp_cb;
+ acc.ctx = this;
+ std::unordered_map<uint32_t, branch_stack_edge_count_map>
+ runtime_branches_by_pid;
+ std::unordered_map<uint32_t, std::unordered_map<uint64_t, uint64_t>>
+ runtime_not_taken_by_pid;
+ std::unordered_map<uint32_t, spe_stream_state> streams;
+
+ const uint8_t *const file_base = mapped_bytes ();
+
+ const bool scan_complete = for_each_mapped_perf_data_record (
+ [&] (const perf_event_header &hdr,
+ const uint8_t *payload,
+ size_t,
+ size_t skip)
+ {
+ if (!spe_aux_decode_ok || hdr.type != perf_record_type::AUXTRACE)
+ return;
+
+ ++auxtrace_records;
+ const uint8_t *tail_ptr = nullptr;
+ size_t tail = 0;
+ uint64_t aux_offset = 0;
+ uint32_t idx = 0;
+ uint32_t tid = 0;
+ if (!spe_auxtrace_tail (
+ hdr, payload, skip, file_base, mapped_size (), &tail_ptr, &tail,
+ &aux_offset, &idx, &tid))
+ {
+ spe_aux_decode_ok = false;
+ fprintf (stderr,
+ _("SPE: invalid AUXTRACE record (hdr.size=%u skip=%zu) "
+ "at file offset %zu.\n"),
+ (unsigned) hdr.size,
+ skip,
+ payload != nullptr
+ ? static_cast<size_t> (payload - file_base)
+ : (size_t) 0);
+ return;
+ }
+
+ /* Zero-size tails are metadata / empty AUXTRACE; keep scanning. */
+ if (tail == 0)
+ return;
+
+ /* Per-CPU AUXTRACE records carry no usable thread id, so they can only
+ be attributed when one process mapped the target. */
+ uint32_t pid = 0;
+ const auto tid_it = target_pid_by_tid_.find (tid);
+ if (tid_it != target_pid_by_tid_.end ())
+ pid = tid_it->second;
+ else if (target_mmaps_by_pid_.size () == 1)
+ pid = target_mmaps_by_pid_.begin ()->first;
+ else
+ {
+ spe_aux_decode_ok = false;
+ fprintf (stderr,
+ _("SPE: cannot associate AUXTRACE thread %u with one target "
+ "process\n"),
+ tid);
+ return;
+ }
+ const std::vector<simple_mmap> *const pid_mmaps
+ = target_mmaps_for_pid (pid);
+ if (pid_mmaps == nullptr)
+ return;
+
+ ++aux_chunks;
+ spe_stream_state &stream = streams[idx];
+ if (stream.have_pid && stream.pid != pid)
+ {
+ spe_aux_decode_ok = false;
+ fprintf (stderr,
+ _("SPE: AUXTRACE stream %u changes target process\n"),
+ idx);
+ return;
+ }
+ if (stream.have_offset && aux_offset != stream.next_aux_offset)
+ {
+ if (aux_offset > stream.next_aux_offset)
+ ++aux_forward_gaps;
+ else
+ ++aux_overlaps;
+ stream = spe_stream_state{};
+ }
+ stream.pid = pid;
+ stream.have_pid = true;
+ if (tail > UINT64_MAX - aux_offset)
+ {
+ spe_aux_decode_ok = false;
+ fprintf (stderr,
+ _("SPE: AUXTRACE offset and size overflow for stream %u\n"),
+ idx);
+ return;
+ }
+ stream.next_aux_offset = aux_offset + tail;
+ stream.have_offset = true;
+
+ active_mmaps_ = pid_mmaps;
+ acc.branch_counts = &runtime_branches_by_pid[pid];
+ acc.not_taken_counts = &runtime_not_taken_by_pid[pid];
+ if (!stream.decoder.consume (tail_ptr, tail, &acc))
+ {
+ spe_aux_decode_ok = false;
+ fprintf (stderr,
+ _("SPE: AUXTRACE decode failed (corrupt SPE) at file "
+ "offset %zu, tail %zu B.\n"),
+ static_cast<size_t> (tail_ptr - file_base),
+ tail);
+ }
+ },
+ perf_scan_log::SPE);
+ active_mmaps_ = nullptr;
+
+ if (spe_aux_decode_ok)
+ for (const auto &entry : streams)
+ if (!entry.second.decoder.finish ())
+ {
+ spe_aux_decode_ok = false;
+ fprintf (stderr,
+ _("SPE: AUXTRACE stream %u ends in a truncated packet\n"),
+ entry.first);
+ }
+
+ if (aux_forward_gaps != 0 || aux_overlaps != 0)
+ fprintf (stderr,
+ _("Warning: SPE AUX offsets have %" PRIu64 " forward gaps "
+ "and %" PRIu64 " overlaps; dropped partial decoder state "
+ "at each discontinuity\n"),
+ aux_forward_gaps, aux_overlaps);
+
+ if (!spe_aux_decode_ok || !scan_complete)
+ return false;
+
+ arm_text_image text;
+ if (!text.load (binary_path))
+ {
+ fprintf (stderr,
+ _("SPE: failed to load executable text from %s\n"),
+ binary_path.c_str ());
+ return false;
+ }
+ /* Insn decode needs static addresses; translate before synthesizing. */
+ branch_stack_edge_count_map dwarf_branches;
+ std::unordered_map<uint64_t, uint64_t> dwarf_not_taken;
+ for (const auto &entry : runtime_branches_by_pid)
+ {
+ const std::vector<simple_mmap> *const pid_mmaps
+ = target_mmaps_for_pid (entry.first);
+ if (pid_mmaps != nullptr)
+ translate_edges_to_dwarf (entry.second, *pid_mmaps, &dwarf_branches);
+ }
+ for (const auto &entry : runtime_not_taken_by_pid)
+ {
+ const std::vector<simple_mmap> *const pid_mmaps
+ = target_mmaps_for_pid (entry.first);
+ if (pid_mmaps != nullptr)
+ translate_pc_counts_to_dwarf (entry.second, *pid_mmaps,
+ &dwarf_not_taken);
+ }
+ arm_spe_synthesize_ranges (
+ dwarf_branches, dwarf_not_taken, text, &range_count_map_);
+ branch_count_map_ = std::move (dwarf_branches);
+ address_frequencies = std::move (dwarf_not_taken);
+ total_samples
+ = spe_profile_total_mass (range_count_map_, address_frequencies);
+
+ if (total_samples == 0)
+ {
+ if (auxtrace_records == 0)
+ fprintf (stderr,
+ _("SPE: No PERF_RECORD_AUXTRACE records in %s "
+ "(not an SPE perf.data, or empty capture).\n"),
+ perf_data_path.c_str ());
+ else if (aux_chunks == 0)
+ fprintf (stderr,
+ _("SPE: %zu AUXTRACE record(s) but no SPE payload "
+ "(all empty tails) in %s.\n"),
+ auxtrace_records,
+ perf_data_path.c_str ());
+ else
+ fprintf (stderr,
+ _("SPE: No profile hits matched -b %s "
+ "(check path / build-id / MMAP; decoded %zu AUXTRACE "
+ "tail(s)).\n"),
+ binary_path.c_str (),
+ aux_chunks);
+ return false;
+ }
+ publish_dwarf_branch_stack_maps (
+ range_count_map_, branch_count_map_, &sorted_touch_pcs_);
+ if (!use_branch_stack_for_correlation_)
+ seed_address_frequencies_from_ranges ();
+ return true;
+}
+
+/* SPE reports ranges, not sampled IPs, so per-PC scoring needs the range
+ mass spread over the instructions the range covers. */
+void
+perf_spe_reader::seed_address_frequencies_from_ranges ()
+{
+ const uint64_t step = range_mass_addr_step ();
+ for (const profile_range &row : sorted_ranges_)
+ {
+ /* An unaligned start is a decode error, not an instruction address. */
+ if (row.end < row.beg || (row.beg % step) != 0)
+ continue;
+ const uint64_t steps = (row.end - row.beg) / step + 1;
+ uint64_t pc = row.beg;
+ for (uint64_t i = 0; i < steps; ++i, pc += step)
+ /* Keep a count already recorded for this PC. */
+ address_frequencies.emplace (pc, row.count);
+ }
+}
+
+} /* namespace perf_data. */
diff --git a/afdo_tools/src/perf_spe_reader.h b/afdo_tools/src/perf_spe_reader.h
new file mode 100644
index 00000000000..e632c1143db
--- /dev/null
+++ b/afdo_tools/src/perf_spe_reader.h
@@ -0,0 +1,68 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#ifndef GCC_PERF_SPE_READER_H
+#define GCC_PERF_SPE_READER_H
+
+#include "branch_stack_reader_base.h"
+#include "perf_file_reader.h"
+
+#include <cstdint>
+#include <string>
+
+namespace perf_data {
+
+/* Offline ARM SPE: AUXTRACE branches + synthesized ranges. */
+class perf_spe_reader : public perf_file_reader, public
branch_stack_reader_base
+{
+public:
+ perf_spe_reader (const std::string &_perf_data_path,
+ const std::string &_binary_path);
+ ~perf_spe_reader () override = default;
+
+ /* AArch64 instructions are four bytes wide. */
+ unsigned range_mass_addr_step () const override { return 4; }
+
+ branch_stack_reader_base *as_branch_stack_reader () override { return this; }
+ const branch_stack_reader_base *as_branch_stack_reader () const override
+ {
+ return this;
+ }
+
+private:
+ /* Decoder callbacks; CTX is the perf_spe_reader. */
+ static bool accept_target_addr_cb (uint64_t runtime_addr, void *ctx);
+ static void note_timestamp_cb (uint64_t runtime_addr, uint64_t timestamp,
+ void *ctx);
+
+ /* Decode the AUXTRACE SPE payloads and synthesize range mass from them. */
+ bool parse_perf_data_section () override;
+
+ /* Spread range mass over the instructions covered, for per-PC scoring. */
+ void seed_address_frequencies_from_ranges ();
+
+ /* Mappings of the PID owning the AUXTRACE buffer being decoded; the decoder
+ callbacks translate through them and it is null outside that decode. */
+ const std::vector<simple_mmap> *active_mmaps_ = nullptr;
+ branch_stack_edge_count_map range_count_map_;
+ branch_stack_edge_count_map branch_count_map_;
+};
+
+} /* namespace perf_data. */
+
+#endif /* GCC_PERF_SPE_READER_H. */
--
2.34.1
