Implement offline BRBE and LBR ingestion from perf.data. The reader
converts sampled branch stacks into range, branch, address-frequency, and
mapping data used by correlation.
afdo_tools/ChangeLog:
* src/perf_branch_stack_reader.cc: New file.
* src/perf_branch_stack_reader.h: New file.
Signed-off-by: Kugan Vivekanandarajah <[email protected]>
Co-authored-by: Dhruv Chawla <[email protected]>
Co-authored-by: Rahul Shinde <[email protected]>
---
afdo_tools/src/perf_branch_stack_reader.cc | 289 +++++++++++++++++++++
afdo_tools/src/perf_branch_stack_reader.h | 69 +++++
2 files changed, 358 insertions(+)
create mode 100644 afdo_tools/src/perf_branch_stack_reader.cc
create mode 100644 afdo_tools/src/perf_branch_stack_reader.h
diff --git a/afdo_tools/src/perf_branch_stack_reader.cc
b/afdo_tools/src/perf_branch_stack_reader.cc
new file mode 100644
index 00000000000..fe63f1b4d53
--- /dev/null
+++ b/afdo_tools/src/perf_branch_stack_reader.cc
@@ -0,0 +1,289 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#include "perf_branch_stack_reader.h"
+
+#include "perf_event_sample_util.h"
+
+#include <cstdint>
+#include <cstdio>
+#include <string>
+
+namespace perf_data {
+namespace {
+
+/* Branch-stack profiles: IP|TID|TIME|PERIOD|BRANCH_STACK (0x907). */
+constexpr uint64_t BRANCH_STACK_SAMPLE_TYPE_IP_TID_TIME_PERIOD
+ = perf_sample_type::IP | perf_sample_type::TID | perf_sample_type::TIME
+ | perf_sample_type::PERIOD | perf_sample_type::BRANCH_STACK;
+
+constexpr size_t BRANCH_STACK_FIXED_PREFIX_BYTES = 4 * sizeof (uint64_t);
+
+bool
+branch_stack_sample_has_fixed_prefix_layout (uint64_t sample_type,
+ uint64_t read_format)
+{
+ return sample_type == BRANCH_STACK_SAMPLE_TYPE_IP_TID_TIME_PERIOD
+ && read_format == 0;
+}
+
+} /* anonymous namespace. */
+
+perf_branch_stack_reader::perf_branch_stack_reader (
+ const std::string &_perf_data_path, const std::string &_binary_path)
+ : perf_file_reader (_perf_data_path, _binary_path)
+{}
+
+bool
+perf_branch_stack_reader::parse_perf_data_section ()
+{
+ reset_branch_stack_profile ();
+ range_count_map.clear ();
+ branch_count_map.clear ();
+ address_frequencies.clear ();
+
+ use_fixed_layout_branch_stack_parse_
+ = branch_stack_sample_has_fixed_prefix_layout (sample_layout.sample_type,
+ sample_layout.read_format)
+ && (sample_layout.branch_sample_type
+ & (perf_branch_stack_format::HW_INDEX
+ | perf_branch_stack_format::COUNTERS))
+ == 0;
+
+ size_t processed_samples = 0;
+
+ /* MMAPs can appear after samples in the ring buffer; bind mappings first.
*/
+ if (!for_each_mapped_perf_data_record (
+ [&] (const perf_event_header &record_header,
+ const uint8_t *payload_ptr,
+ size_t payload_size,
+ size_t)
+ {
+ switch (record_header.type)
+ {
+ case perf_record_type::MMAP:
+ parse_mmap_record (payload_ptr, payload_size,
+ record_header.misc);
+ break;
+ case perf_record_type::MMAP2:
+ parse_mmap2_record (payload_ptr, payload_size,
+ record_header.misc);
+ break;
+ case perf_record_type::COMM:
+ note_process_lifetime_record (
+ record_header, payload_ptr, payload_size);
+ break;
+ default:
+ break;
+ }
+ }))
+ return false;
+ if (!target_mapping_lifetime_valid ())
+ return false;
+
+ if (!for_each_mapped_perf_data_record (
+ [&] (const perf_event_header &record_header,
+ const uint8_t *payload_ptr,
+ size_t payload_size,
+ size_t)
+ {
+ switch (record_header.type)
+ {
+ case perf_record_type::SAMPLE:
+ if (payload_size > 0
+ && parse_sample_record (payload_ptr, payload_size))
+ processed_samples++;
+ break;
+ default:
+ break;
+ }
+ }))
+ return false;
+
+ compute_profile_total_count ();
+
+ if (processed_samples == 0)
+ {
+ if (target_mmaps_by_pid_.empty ())
+ fprintf (stderr,
+ _("PERF-LBR: No samples matched -b %s "
+ "(no target MMAP/MMAP2 found).\n"),
+ binary_path.c_str ());
+ else
+ fprintf (stderr,
+ _("PERF-LBR: No branch-stack samples accepted for -b %s "
+ "(%zu target mapping(s); check sample_type / LBR "
+ "filters).\n"),
+ binary_path.c_str (),
+ target_mmap_count ());
+ return false;
+ }
+ publish_dwarf_branch_stack_maps (
+ range_count_map, branch_count_map, &sorted_touch_pcs_);
+ return true;
+}
+
+bool
+perf_branch_stack_reader::parse_sample_record (const uint8_t *record_data,
+ size_t record_size)
+{
+ if (use_fixed_layout_branch_stack_parse_)
+ return parse_sample_record_fixed_branch_stack_layout (
+ record_data, record_size);
+
+ const uint8_t *ptr = record_data;
+ const uint8_t *const end = record_data + record_size;
+
+ uint64_t ip = 0;
+ bool have_ip = false;
+ uint32_t pid = 0;
+ uint64_t sample_period = 0;
+ uint64_t sample_time = 0;
+
+ if (!perf_event_sample_util::walk_sample_before_branch (
+ ptr, end, sample_layout.sample_type, sample_layout.read_format, &ip,
+ &have_ip, &pid, nullptr, &sample_period, &sample_time))
+ {
+ fprintf (stderr,
+ _("PERF: WARNING: Truncated or invalid sample (prefix)\n"));
+ return false;
+ }
+ if (sample_layout.fixed_sample_period
+ && sample_period != sample_layout.sample_period)
+ {
+ fprintf (stderr,
+ _("PERF: WARNING: sample period does not match the fixed "
+ "event period\n"));
+ return false;
+ }
+
+ uint64_t nr = 0;
+ const perf_branch_stack_entry *entries = nullptr;
+ if (!perf_event_sample_util::walk_sample_branch_stack (
+ ptr, end, sample_layout.branch_sample_type, sample_layout.sample_type,
+ &nr, &entries))
+ {
+ fprintf (stderr,
+ _("PERF: WARNING: Truncated sample record (BRANCH_STACK)\n"));
+ return false;
+ }
+
+ if (!perf_event_sample_util::walk_sample_after_branch (
+ ptr, end, sample_layout.sample_type, sample_layout.sample_regs_user,
+ sample_layout.sample_regs_intr)
+ || ptr != end)
+ {
+ fprintf (stderr,
+ _("PERF: WARNING: Truncated or invalid sample suffix\n"));
+ return false;
+ }
+
+ const std::vector<simple_mmap> *const pid_mmaps = target_mmaps_for_pid (pid);
+ if (pid_mmaps == nullptr)
+ return false;
+ uint64_t dwarf_ip = 0;
+ const bool ip_is_target
+ = have_ip && ip != 0
+ && try_translate_runtime_with_mmaps (*pid_mmaps, ip, &dwarf_ip);
+ bool recorded_sample = record_sample_ip (
+ dwarf_ip, ip_is_target, sample_time, accumulate_sample_ips_,
+ address_frequencies);
+ if (use_branch_stack_for_correlation_
+ && (sample_layout.sample_type & perf_sample_type::BRANCH_STACK) != 0
+ && nr > 0
+ && accumulate_branch_stack_sample (
+ entries,
+ nr,
+ *this,
+ sample_time,
+ *pid_mmaps,
+ range_count_map,
+ branch_count_map))
+ recorded_sample = true;
+
+ return recorded_sample;
+}
+
+bool
+perf_branch_stack_reader::parse_sample_record_fixed_branch_stack_layout (
+ const uint8_t *record_data, size_t record_size)
+{
+ if (record_size < BRANCH_STACK_FIXED_PREFIX_BYTES + sizeof (uint64_t))
+ return false;
+
+ uint64_t ip = read_u64 (record_data);
+ const uint32_t pid = read_u32 (record_data + sizeof (uint64_t));
+ uint64_t sample_time = read_u64 (record_data + 2 * sizeof (uint64_t));
+ const uint64_t sample_period
+ = read_u64 (record_data + 3 * sizeof (uint64_t));
+ if (sample_layout.fixed_sample_period
+ && sample_period != sample_layout.sample_period)
+ return false;
+
+ const uint8_t *ptr = record_data + BRANCH_STACK_FIXED_PREFIX_BYTES;
+ const uint8_t *const end = record_data + record_size;
+
+ uint64_t nr = read_u64 (ptr);
+ ptr += sizeof (uint64_t);
+
+ if (nr > SIZE_MAX / perf_event_sample_util::BRANCH_ENTRY_BYTES)
+ return false;
+ const size_t esz
+ = static_cast<size_t> (nr) * perf_event_sample_util::BRANCH_ENTRY_BYTES;
+ if (esz > static_cast<size_t> (end - ptr))
+ return false;
+
+ const perf_branch_stack_entry *entries
+ = nr > 0 ? reinterpret_cast<const perf_branch_stack_entry *> (ptr)
+ : nullptr;
+
+ const std::vector<simple_mmap> *const pid_mmaps = target_mmaps_for_pid (pid);
+ if (pid_mmaps == nullptr)
+ return false;
+ uint64_t dwarf_ip = 0;
+ const bool ip_is_target
+ = ip != 0 && try_translate_runtime_with_mmaps (*pid_mmaps, ip, &dwarf_ip);
+ bool recorded_sample = record_sample_ip (
+ dwarf_ip, ip_is_target, sample_time, accumulate_sample_ips_,
+ address_frequencies);
+ if (use_branch_stack_for_correlation_ && nr > 0
+ && accumulate_branch_stack_sample (
+ entries,
+ nr,
+ *this,
+ sample_time,
+ *pid_mmaps,
+ range_count_map,
+ branch_count_map))
+ recorded_sample = true;
+
+ return recorded_sample;
+}
+
+void
+perf_branch_stack_reader::compute_profile_total_count ()
+{
+ if (!range_count_map.empty ())
+ {
+ total_samples = total_mass_from_runtime_range_map (range_count_map);
+ return;
+ }
+ total_samples = total_mass_from_address_frequencies (address_frequencies);
+}
+
+} /* namespace perf_data. */
diff --git a/afdo_tools/src/perf_branch_stack_reader.h
b/afdo_tools/src/perf_branch_stack_reader.h
new file mode 100644
index 00000000000..52a148ef556
--- /dev/null
+++ b/afdo_tools/src/perf_branch_stack_reader.h
@@ -0,0 +1,69 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#ifndef GCC_PERF_BRANCH_STACK_READER_H
+#define GCC_PERF_BRANCH_STACK_READER_H
+
+#include "branch_stack_reader_base.h"
+#include "perf_file_reader.h"
+
+#include <cstdint>
+#include <string>
+
+namespace perf_data {
+
+/* Offline LBR/BRBE: SAMPLE + BRANCH_STACK into range/branch maps. */
+class perf_branch_stack_reader : public perf_file_reader,
+ public branch_stack_reader_base
+{
+public:
+ perf_branch_stack_reader (const std::string &_perf_data_path,
+ const std::string &_binary_path);
+ ~perf_branch_stack_reader () override = default;
+
+ branch_stack_reader_base *as_branch_stack_reader () override { return this; }
+ const branch_stack_reader_base *as_branch_stack_reader () const override
+ {
+ return this;
+ }
+
+private:
+ /* Bind the target mappings, then ingest the branch-stack samples. */
+ bool parse_perf_data_section () override;
+
+ /* Ingest one sample; returns true when it contributed mass. */
+ bool parse_sample_record (const uint8_t *record_data, size_t record_size);
+
+ /* As PARSE_SAMPLE_RECORD, for the common sample layout whose field offsets
+ are known up front, so the record needs no field-by-field walk. */
+ bool
+ parse_sample_record_fixed_branch_stack_layout (const uint8_t *record_data,
+ size_t record_size);
+
+ /* Report range mass when there is any, else per-PC mass. */
+ void compute_profile_total_count ();
+
+ branch_stack_edge_count_map range_count_map;
+ branch_stack_edge_count_map branch_count_map;
+
+ bool use_fixed_layout_branch_stack_parse_ = false;
+};
+
+} /* namespace perf_data. */
+
+#endif /* GCC_PERF_BRANCH_STACK_READER_H. */
--
2.34.1
