Add perf_event_open-based live LBR capture. The backend manages per-CPU
ring buffers and feeds captured branch stacks through the same normalization
path as offline perf.data, so live and offline samples reach correlation in
the same form. The afdo_create command line that selects live capture is
added together with that command later in the series.
afdo_tools/ChangeLog:
* src/live_branch_stack_reader.cc: New file.
* src/live_branch_stack_reader.h: New file.
Signed-off-by: Kugan Vivekanandarajah <[email protected]>
Co-authored-by: Dhruv Chawla <[email protected]>
Co-authored-by: Rahul Shinde <[email protected]>
---
afdo_tools/src/live_branch_stack_reader.cc | 844 +++++++++++++++++++++
afdo_tools/src/live_branch_stack_reader.h | 94 +++
2 files changed, 938 insertions(+)
create mode 100644 afdo_tools/src/live_branch_stack_reader.cc
create mode 100644 afdo_tools/src/live_branch_stack_reader.h
diff --git a/afdo_tools/src/live_branch_stack_reader.cc
b/afdo_tools/src/live_branch_stack_reader.cc
new file mode 100644
index 00000000000..ca7789dce7e
--- /dev/null
+++ b/afdo_tools/src/live_branch_stack_reader.cc
@@ -0,0 +1,844 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#include "live_branch_stack_reader.h"
+
+#include "perf_event_sample_util.h"
+#include "perf.h"
+#include "perf_file_reader.h"
+
+#include <algorithm>
+#include <cerrno>
+#include <cinttypes>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
+
+#include <linux/perf_event.h>
+#include <poll.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/syscall.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#include <vector>
+
+#ifndef __NR_perf_event_open
+#if defined(__x86_64__)
+#define PERF_EVENT_OPEN_NR 298
+#elif defined(__aarch64__)
+#define PERF_EVENT_OPEN_NR 241
+#else
+#error "Unsupported architecture for perf_event_open"
+#endif
+#else
+#define PERF_EVENT_OPEN_NR __NR_perf_event_open
+#endif
+
+namespace perf_data {
+namespace {
+
+/* Sum of each row's count weighted by its inclusive byte span. */
+uint64_t
+total_mass_from_range_counts (const std::vector<profile_range> &ranges)
+{
+ uint64_t total = 0;
+ for (const profile_range &row : ranges)
+ {
+ if (row.end >= row.beg)
+ total += row.count * (row.end - row.beg + 1);
+ }
+ return total;
+}
+
+/* Cap on data pages per CPU, to bound total buffer memory. */
+constexpr size_t MAX_RING_DATA_PAGES = 256;
+
+/* One CPU's perf event and its ring buffer. */
+struct cpu_ring_buffer
+{
+ int fd = -1;
+ void *base = nullptr;
+ size_t len = 0;
+ uint64_t data_offset = 0;
+ uint64_t data_size = 0;
+};
+
+/* Online CPU ids, parsed from the kernel's list ("0-3,8"). Falls back to
+ 0 to N-1 when the list cannot be read. */
+std::vector<int>
+online_cpu_ids ()
+{
+ std::vector<int> cpus;
+ if (FILE *f = fopen ("/sys/devices/system/cpu/online", "r"))
+ {
+ char mask[512] = "";
+ const bool read_ok = fgets (mask, sizeof (mask), f) != nullptr;
+ fclose (f);
+ for (const char *p = read_ok ? mask : ""; *p != '\0';)
+ {
+ char *after = nullptr;
+ const long first = strtol (p, &after, 10);
+ if (after == p)
+ break;
+ long last = first;
+ if (*after == '-')
+ {
+ const char *range_start = after + 1;
+ last = strtol (range_start, &after, 10);
+ if (after == range_start)
+ break;
+ }
+ for (long cpu = first; cpu <= last; ++cpu)
+ cpus.push_back (static_cast<int> (cpu));
+ if (*after != ',')
+ break;
+ p = after + 1;
+ }
+ }
+ if (cpus.empty ())
+ {
+ const long ncpu = sysconf (_SC_NPROCESSORS_ONLN);
+ for (long cpu = 0; cpu < ncpu; ++cpu)
+ cpus.push_back (static_cast<int> (cpu));
+ }
+ return cpus;
+}
+
+/* Data pages to request per CPU. perf_event_mlock_kb is a per-CPU limit, so
+ asking for more fails once every CPU is mapped. */
+size_t
+ring_data_pages (size_t page_size)
+{
+ /* The kernel's default, if the sysctl cannot be read. */
+ unsigned long mlock_kb = 512;
+ if (FILE *f = fopen ("/proc/sys/kernel/perf_event_mlock_kb", "r"))
+ {
+ unsigned long value = 0;
+ if (fscanf (f, "%lu", &value) == 1 && value != 0)
+ mlock_kb = value;
+ fclose (f);
+ }
+
+ /* One page goes to the metadata page, and the count must be a power of
+ two. */
+ const size_t budget = (mlock_kb * 1024) / page_size;
+ if (budget < 2)
+ return 1;
+ size_t pages = 1;
+ while (pages * 2 <= budget - 1 && pages < MAX_RING_DATA_PAGES)
+ pages *= 2;
+ return pages;
+}
+
+/* Map BUFFER's ring, halving DATA_PAGES while the kernel refuses it: other
+ perf users share the same limit. */
+bool
+map_cpu_ring_buffer (cpu_ring_buffer &buffer, size_t page_size,
+ size_t data_pages)
+{
+ while (data_pages != 0)
+ {
+ const size_t len = page_size * (1 + data_pages);
+ void *base
+ = mmap (nullptr, len, PROT_READ | PROT_WRITE, MAP_SHARED, buffer.fd, 0);
+ if (base != MAP_FAILED)
+ {
+ buffer.base = base;
+ buffer.len = len;
+ return true;
+ }
+ data_pages /= 2;
+ }
+ return false;
+}
+
+} /* anonymous namespace. */
+
+static long
+sys_perf_event_open (perf_event_attr *attr, pid_t pid, int cpu, int group_fd,
+ unsigned long flags)
+{
+ return syscall (PERF_EVENT_OPEN_NR, attr, pid, cpu, group_fd, flags);
+}
+
+/* The component of PATH after its last separator, or PATH when it has none.
*/
+static std::string
+path_basename (const std::string &path)
+{
+ size_t slash = path.find_last_of ("/\\");
+ return slash != std::string::npos ? path.substr (slash + 1) : path;
+}
+
+/* Wait for PID, retrying when a signal interrupts the wait, and store its wait
+ status in *STATUS when STATUS is non-null. Returns false when PID could not
+ be reaped, in which case *STATUS is left alone: a wait status of zero is
+ indistinguishable from a clean exit and must not be reported as one. */
+static bool
+reap_child (pid_t pid, int *status = nullptr)
+{
+ int wait_status = 0;
+ while (waitpid (pid, &wait_status, 0) < 0)
+ if (errno != EINTR)
+ return false;
+ if (status != nullptr)
+ *status = wait_status;
+ return true;
+}
+
+live_branch_stack_reader::live_branch_stack_reader (
+ const std::string &_binary_path,
+ const std::vector<std::string> &_command_argv, uint64_t _sample_period)
+ : binary_path (_binary_path), command_argv (_command_argv),
+ sample_period (_sample_period)
+{}
+
+live_branch_stack_reader::~live_branch_stack_reader () = default;
+
+/* True when MMAP_FILENAME names the binary we are profiling. */
+bool
+live_branch_stack_reader::is_target_binary (
+ const std::string &mmap_filename) const
+{
+ if (mmap_filename.empty ())
+ return false;
+
+ /* The kernel reports unlinked files as "<path> (deleted)". */
+ std::string mmap_path = mmap_filename;
+ size_t del = mmap_path.find (" (deleted)");
+ if (del != std::string::npos)
+ mmap_path.erase (del);
+
+ if (mmap_path == binary_path
+ || (!binary_path_resolved.empty () && mmap_path == binary_path_resolved))
+ return true;
+
+ /* Fall back to the basename: the child may be exec'd through a different
+ path (symlink, $PATH lookup) than the one named on the command line. */
+ const std::string mmap_base = path_basename (mmap_path);
+ return mmap_base == path_basename (binary_path)
+ || (!binary_path_resolved.empty ()
+ && mmap_base == path_basename (binary_path_resolved));
+}
+
+/* Record the half-open START to END range at PGOFF as PID's mapping of the
+ profiled binary. Keep mappings separate: pgoff alone does not establish
+ that two runtime intervals have the same affine translation. */
+void
+live_branch_stack_reader::add_target_mapping (uint32_t pid, uint64_t start,
+ uint64_t end, uint64_t pgoff)
+{
+ target_mapping_pids.insert (pid);
+ const std::string &path
+ = binary_path_resolved.empty () ? binary_path : binary_path_resolved;
+ uint64_t vaddr = 0;
+ get_elf_segment_vaddr_for_pgoff (path, pgoff, vaddr);
+ bool is_pie = get_elf_is_pie (path);
+ target_mmaps_by_pid[pid].emplace_back (start, end, pgoff, vaddr, is_pie);
+}
+
+/* Note that PID mapped something else over one of its target mappings, which
+ leaves its later samples unattributable. */
+void
+live_branch_stack_reader::note_replaced_target_mapping (uint32_t pid,
+ uint64_t start,
+ uint64_t end)
+{
+ const auto it = target_mmaps_by_pid.find (pid);
+ if (it == target_mmaps_by_pid.end ())
+ return;
+ for (const simple_mmap &old : it->second)
+ if (start < old.end_addr && end > old.start_addr)
+ target_executable_mappings_changed = true;
+}
+
+void
+live_branch_stack_reader::process_mmap_record (const uint8_t *payload,
+ size_t payload_size,
+ uint16_t misc)
+{
+ if (payload_size < sizeof (perf_mmap_record))
+ return;
+ perf_mmap_record mmap;
+ std::memcpy (&mmap, payload, sizeof (mmap));
+ std::string name = perf_mmap_filename_view (
+ reinterpret_cast<const char *> (payload + sizeof (perf_mmap_record)),
+ payload_size - sizeof (perf_mmap_record));
+ uint64_t end = 0;
+ if ((misc & PERF_RECORD_MISC_MMAP_DATA) != 0
+ || !try_compute_mmap_end (mmap.addr, mmap.len, &end))
+ return;
+ if (!is_target_binary (name))
+ {
+ note_replaced_target_mapping (mmap.pid, mmap.addr, end);
+ return;
+ }
+ add_target_mapping (mmap.pid, mmap.addr, end, mmap.pgoff);
+}
+
+void
+live_branch_stack_reader::process_mmap2_record (const uint8_t *payload,
+ size_t payload_size)
+{
+ if (payload_size < sizeof (perf_mmap2_record))
+ return;
+ perf_mmap2_record mmap2;
+ std::memcpy (&mmap2, payload, sizeof (mmap2));
+ std::string name = perf_mmap_filename_view (
+ reinterpret_cast<const char *> (payload + sizeof (perf_mmap2_record)),
+ payload_size - sizeof (perf_mmap2_record));
+ uint64_t end = 0;
+ if ((mmap2.prot & PROT_EXEC) == 0
+ || !try_compute_mmap_end (mmap2.addr, mmap2.len, &end))
+ return;
+ if (!is_target_binary (name))
+ {
+ note_replaced_target_mapping (mmap2.pid, mmap2.addr, end);
+ return;
+ }
+ add_target_mapping (mmap2.pid, mmap2.addr, end, mmap2.pgoff);
+}
+
+/* Ingest one PERF_RECORD_SAMPLE; returns true when it contributed mass. */
+bool
+live_branch_stack_reader::process_sample_record (const uint8_t *payload,
+ size_t payload_size)
+{
+ if (payload_size == 0)
+ return false;
+
+ const uint8_t *ptr = payload;
+ const uint8_t *const end = payload + payload_size;
+ uint64_t ip = 0;
+ bool have_ip = false;
+ uint32_t pid = 0;
+ uint64_t observed_period = 0;
+ uint64_t sample_time = 0;
+
+ if (!perf_event_sample_util::walk_sample_before_branch (
+ ptr, end, sample_type, 0, &ip, &have_ip, &pid, nullptr,
+ &observed_period, &sample_time))
+ return false;
+ if (observed_period != sample_period)
+ return false;
+
+ uint64_t nr = 0;
+ const perf_branch_stack_entry *entries = nullptr;
+ if (!perf_event_sample_util::walk_sample_branch_stack (
+ ptr, end, branch_sample_type, sample_type, &nr, &entries))
+ return false;
+
+ if (!perf_event_sample_util::walk_sample_after_branch (
+ ptr, end, sample_type, 0, 0)
+ || ptr != end)
+ return false;
+
+ const auto mmap_it = target_mmaps_by_pid.find (pid);
+ if (mmap_it == target_mmaps_by_pid.end ())
+ return false;
+ const std::vector<simple_mmap> &pid_mmaps = mmap_it->second;
+
+ /* Primary path: LBR branch stack. Sample IP fallbacks improve coverage
+ when the stack is empty or has no target branches. */
+ uint64_t dwarf_ip = 0;
+ const bool ip_is_target
+ = have_ip && ip != 0
+ && try_translate_runtime_with_mmaps (pid_mmaps, ip, &dwarf_ip);
+ bool recorded = record_sample_ip (
+ dwarf_ip, ip_is_target, sample_time, accumulate_sample_ips_,
+ address_frequencies);
+ if ((sample_type & perf_sample_type::BRANCH_STACK) != 0 && nr > 0
+ && accumulate_branch_stack_sample (entries,
+ nr,
+ *this,
+ sample_time,
+ pid_mmaps,
+ range_count_map,
+ branch_count_map))
+ recorded = true;
+
+ return recorded;
+}
+
+/* Drain the records the kernel has published into the perf ring buffer,
+ handling records that wrap around its end. */
+void
+live_branch_stack_reader::process_ring_buffer (void *mmap_base,
+ uint64_t data_offset,
+ uint64_t data_size)
+{
+ if (mmap_base == nullptr || data_size == 0)
+ return;
+ char *base = static_cast<char *> (mmap_base);
+ perf_event_mmap_page *meta = reinterpret_cast<perf_event_mmap_page *> (base);
+ char *data = base + data_offset;
+ uint64_t ring_size = data_size;
+
+ for (;;)
+ {
+ uint64_t head = __atomic_load_n (&meta->data_head, __ATOMIC_ACQUIRE);
+ uint64_t tail = __atomic_load_n (&meta->data_tail, __ATOMIC_RELAXED);
+ if (tail == head)
+ break;
+ if (tail > head || head - tail > ring_size)
+ {
+ /* Once the producer has lapped the consumer, head - ring_size is
+ not necessarily a record boundary. Discard this snapshot rather
+ than interpreting overwritten bytes as event headers. */
+ ring_overwrite = true;
+ __atomic_store_n (&meta->data_tail, head, __ATOMIC_RELEASE);
+ break;
+ }
+ const uint64_t initial_tail = tail;
+ while (tail != head)
+ {
+ if (tail > head || head - tail < sizeof (perf_event_header)
+ || ring_size < sizeof (perf_event_header))
+ break;
+ uint64_t offset = tail % ring_size;
+ perf_event_header header;
+ const size_t header_first
+ = std::min<uint64_t> (sizeof (header), ring_size - offset);
+ std::memcpy (&header, data + offset, header_first);
+ if (header_first < sizeof (header))
+ std::memcpy (reinterpret_cast<uint8_t *> (&header) + header_first,
+ data,
+ sizeof (header) - header_first);
+ uint32_t size = header.size;
+ if (size < sizeof (perf_event_header) || size > ring_size)
+ {
+ /* Skip the bad size, so the scan cannot loop forever. */
+ tail += sizeof (perf_event_header);
+ continue;
+ }
+ if (size > head - tail)
+ break;
+
+ std::vector<uint8_t> wrap_buf;
+ const uint8_t *payload = nullptr;
+ if (size > ring_size - offset)
+ {
+ wrap_buf.resize (size);
+ const size_t first = ring_size - offset;
+ std::memcpy (wrap_buf.data (), data + offset, first);
+ std::memcpy (wrap_buf.data () + first, data, size - first);
+ payload = wrap_buf.data () + sizeof (perf_event_header);
+ }
+ else
+ payload = reinterpret_cast<const uint8_t *> (data + offset)
+ + sizeof (perf_event_header);
+ size_t payload_size = size - sizeof (perf_event_header);
+
+ switch (header.type)
+ {
+ case perf_record_type::SAMPLE:
+ if (process_sample_record (payload, payload_size))
+ total_samples++;
+ break;
+ case perf_record_type::MMAP:
+ process_mmap_record (payload, payload_size, header.misc);
+ break;
+ case perf_record_type::MMAP2:
+ process_mmap2_record (payload, payload_size);
+ break;
+ case perf_record_type::LOST:
+ /* Payload is { u64 id; u64 lost; }. */
+ if (payload_size >= 2 * sizeof (uint64_t))
+ {
+ uint64_t lost = read_u64 (payload + sizeof (uint64_t));
+ lost_records += lost;
+ }
+ break;
+ case perf_record_type::COMM:
+ if ((header.misc & PERF_RECORD_MISC_COMM_EXEC) != 0
+ && payload_size >= 2 * sizeof (uint32_t)
+ && target_mapping_pids.count (read_u32 (payload)) != 0)
+ target_executable_mappings_changed = true;
+ break;
+ default:
+ break;
+ }
+ tail += size;
+ }
+ __atomic_store_n (&meta->data_tail, tail, __ATOMIC_RELEASE);
+ if (tail == initial_tail)
+ break;
+ }
+}
+
+/* Run the command under a branch-stack perf event and collect its profile. */
+bool
+live_branch_stack_reader::load ()
+{
+ range_count_map.clear ();
+ branch_count_map.clear ();
+ address_frequencies.clear ();
+ reset_branch_stack_profile ();
+ reset_profile_state ();
+ target_mmaps_by_pid.clear ();
+ target_mapping_pids.clear ();
+ target_executable_mappings_changed = false;
+ total_samples = 0;
+ lost_records = 0;
+ ring_overwrite = false;
+ if (command_argv.empty ())
+ {
+ fprintf (stderr, _("Error: No command to run for direct profiling\n"));
+ return false;
+ }
+ if (sample_period == 0)
+ {
+ fprintf (stderr, _("Error: Live sample period must be nonzero\n"));
+ return false;
+ }
+
+ /* Resolve binary path so we can match kernel MMAP paths (often absolute).
*/
+ char *resolved = realpath (binary_path.c_str (), nullptr);
+ if (resolved)
+ {
+ binary_path_resolved = resolved;
+ free (resolved);
+ }
+ else
+ binary_path_resolved.clear ();
+
+ /* Prepare ATTR before fork so the parent can call perf_event_open
+ immediately (the child may exit quickly). */
+ perf_event_attr attr = {};
+ attr.type = PERF_TYPE_HARDWARE;
+ attr.size = sizeof (attr);
+ attr.config = PERF_COUNT_HW_CPU_CYCLES;
+ attr.sample_period = sample_period;
+ attr.sample_type
+ = use_branch_stack_for_correlation_
+ ? DEFAULT_BRANCH_STACK_SAMPLE_TYPE
+ : DEFAULT_BRANCH_STACK_SAMPLE_TYPE & ~perf_sample_type::BRANCH_STACK;
+ attr.branch_sample_type
+ = use_branch_stack_for_correlation_ ? PERF_SAMPLE_BRANCH_ANY : 0;
+ attr.exclude_kernel = 1;
+ attr.exclude_hv = 1;
+ attr.inherit = 1;
+ attr.mmap = 1;
+ attr.mmap2 = 1;
+
+ int sync_pipe[2];
+ if (pipe (sync_pipe) != 0)
+ {
+ fprintf (stderr, _("Error: pipe failed: %s\n"), strerror (errno));
+ return false;
+ }
+
+ pid_t pid = fork ();
+ if (pid < 0)
+ {
+ fprintf (stderr, _("Error: fork failed: %s\n"), strerror (errno));
+ close (sync_pipe[0]);
+ close (sync_pipe[1]);
+ return false;
+ }
+
+ if (pid == 0)
+ {
+ /* We start the command here; we block in read () until the parent
+ attaches perf, then exec. */
+ close (sync_pipe[1]);
+ char ignore;
+ if (read (sync_pipe[0], &ignore, 1) != 1)
+ _exit (126);
+ close (sync_pipe[0]);
+ std::vector<char *> argv;
+ for (auto &s : command_argv)
+ argv.push_back (const_cast<char *> (s.c_str ()));
+ argv.push_back (nullptr);
+ execvp (argv[0], argv.data ());
+ fprintf (stderr, _("Error: execvp %s failed: %s\n"), argv[0],
+ strerror (errno));
+ _exit (127);
+ }
+
+ /* Parent: attach perf to the child, which is blocked in read () and so
+ cannot exit. */
+ close (sync_pipe[0]);
+
+ const long system_page_size = sysconf (_SC_PAGESIZE);
+ if (system_page_size <= 0)
+ {
+ fprintf (stderr, _("Error: unable to determine system page size\n"));
+ close (sync_pipe[1]);
+ reap_child (pid);
+ return false;
+ }
+ const size_t page_size = static_cast<size_t> (system_page_size);
+
+ /* The kernel refuses to mmap an inheriting event opened with cpu == -1, so
+ open one event per CPU; attr.inherit still follows the workload's
+ threads. */
+ const std::vector<int> cpus = online_cpu_ids ();
+ const size_t data_pages = ring_data_pages (page_size);
+ std::vector<cpu_ring_buffer> buffers;
+ int open_errno = 0;
+ int mmap_errno = 0;
+ for (const int cpu : cpus)
+ {
+ const long opened = sys_perf_event_open (&attr, pid, cpu, -1, 0);
+ if (opened < 0)
+ {
+ if (open_errno == 0)
+ open_errno = errno;
+ continue;
+ }
+ cpu_ring_buffer buffer;
+ buffer.fd = static_cast<int> (opened);
+ if (!map_cpu_ring_buffer (buffer, page_size, data_pages))
+ {
+ if (mmap_errno == 0)
+ mmap_errno = errno;
+ close (buffer.fd);
+ continue;
+ }
+ buffers.push_back (buffer);
+ }
+
+ if (buffers.empty ())
+ {
+ close (sync_pipe[1]);
+ reap_child (pid);
+ if (open_errno != 0 && use_branch_stack_for_correlation_)
+ fprintf (stderr,
+ _("Error: live capture requires LBR/BRBE branch-stack support"
+ " (perf_event_open failed: %s)\n"),
+ strerror (open_errno));
+ else if (open_errno != 0)
+ fprintf (stderr, _("Error: live perf_event_open failed: %s\n"),
+ strerror (open_errno));
+ else
+ fprintf (stderr, _("Error: mmap perf buffer failed: %s\n"),
+ strerror (mmap_errno));
+ fprintf (stderr,
+ _(" Check kernel.perf_event_paranoid or grant CAP_PERFMON to "
+ "afdo_create; do not run the profiled workload as root.\n"));
+ fprintf (stderr,
+ _(" Offline mode (-p <perf.data>) still works without live"
+ " branch-stack.\n"));
+ return false;
+ }
+ /* A CPU we could not open costs only the samples that run on it. */
+ if (buffers.size () != cpus.size ())
+ fprintf (stderr,
+ _("Warning: profiling %zu of %zu CPUs (%s); samples that run on"
+ " the rest are not collected\n"),
+ buffers.size (), cpus.size (),
+ strerror (open_errno != 0 ? open_errno : mmap_errno));
+
+ sample_type = attr.sample_type;
+ branch_sample_type = attr.branch_sample_type;
+
+ /* Find each data area, ignoring a layout that would read past the
+ mapping. */
+ for (cpu_ring_buffer &buffer : buffers)
+ {
+ const perf_event_mmap_page *meta
+ = static_cast<const perf_event_mmap_page *> (buffer.base);
+ buffer.data_offset = meta->data_offset;
+ buffer.data_size = meta->data_size;
+ if (buffer.data_size == 0 || buffer.data_offset >= buffer.len
+ || buffer.data_size > buffer.len - buffer.data_offset)
+ {
+ buffer.data_offset = page_size;
+ buffer.data_size = buffer.len - page_size;
+ }
+ }
+
+ bool enabled = false;
+ for (const cpu_ring_buffer &buffer : buffers)
+ if (ioctl (buffer.fd, PERF_EVENT_IOC_ENABLE, 0) == 0)
+ enabled = true;
+ if (!enabled)
+ {
+ fprintf (stderr, _("Error: PERF_EVENT_IOC_ENABLE failed: %s\n"),
+ strerror (errno));
+ for (const cpu_ring_buffer &buffer : buffers)
+ {
+ munmap (buffer.base, buffer.len);
+ close (buffer.fd);
+ }
+ close (sync_pipe[1]);
+ reap_child (pid);
+ return false;
+ }
+ /* Unblock child only after perf is enabled. */
+ {
+ char one = 1;
+ ssize_t written;
+ do
+ written = write (sync_pipe[1], &one, 1);
+ while (written < 0 && errno == EINTR);
+ /* Closing the write end unblocks the child either way, but on a short
+ write it reads EOF and exits instead of running the workload. */
+ if (written != 1)
+ fprintf (stderr,
+ _("Error: could not release the profiled workload: %s\n"),
+ strerror (errno));
+ close (sync_pipe[1]);
+ }
+
+ fprintf (stderr,
+ use_branch_stack_for_correlation_
+ ? _("Profiling %s (PID %d, branch-stack)...\n")
+ : _("Profiling %s (PID %d, sampled IPs)...\n"),
+ command_argv[0].c_str (),
+ pid);
+ int status = 0;
+ bool have_status = false;
+ std::vector<pollfd> pfds;
+ pfds.reserve (buffers.size ());
+ for (const cpu_ring_buffer &buffer : buffers)
+ pfds.push_back (pollfd{buffer.fd, POLLIN, 0});
+ for (;;)
+ {
+ const pid_t reaped = waitpid (pid, &status, WNOHANG);
+ if (reaped == pid)
+ {
+ have_status = true;
+ break;
+ }
+ if (reaped < 0 && errno != EINTR)
+ break;
+ const int r = poll (pfds.data (), pfds.size (), 100);
+ if (r < 0 && errno != EINTR)
+ {
+ /* Without poll () there is nothing left to pace this loop, so stop
+ draining incrementally and wait for the workload instead. */
+ fprintf (stderr,
+ _("Warning: poll on the perf event fds failed: %s\n"),
+ strerror (errno));
+ break;
+ }
+ if (r <= 0)
+ continue;
+ for (size_t i = 0; i < buffers.size (); ++i)
+ if ((pfds[i].revents & POLLIN) != 0)
+ process_ring_buffer (buffers[i].base, buffers[i].data_offset,
+ buffers[i].data_size);
+ }
+ int reap_errno = 0;
+ if (!have_status)
+ {
+ have_status = reap_child (pid, &status);
+ if (!have_status)
+ reap_errno = errno;
+ }
+
+ /* Closing the fds below stops sampling, so a failed disable costs nothing
+ beyond the records already drained. */
+ for (const cpu_ring_buffer &buffer : buffers)
+ {
+ process_ring_buffer (buffer.base, buffer.data_offset, buffer.data_size);
+ if (ioctl (buffer.fd, PERF_EVENT_IOC_DISABLE, 0) < 0)
+ fprintf (stderr, _("Warning: PERF_EVENT_IOC_DISABLE failed: %s\n"),
+ strerror (errno));
+ munmap (buffer.base, buffer.len);
+ close (buffer.fd);
+ }
+
+ if (lost_records != 0)
+ fprintf (stderr,
+ _("Warning: kernel dropped %" PRIu64 " perf records; raise"
+ " --sample-period to sample less often\n"),
+ lost_records);
+ if (ring_overwrite)
+ fprintf (stderr,
+ _("Warning: the perf ring buffer was overwritten; discarded "
+ "an incomplete snapshot. Raise --sample-period to sample "
+ "less often\n"));
+
+ if (!have_status)
+ fprintf (stderr,
+ _("Warning: could not reap the profiled workload (%s); its exit"
+ " status is unknown and the profile may be incomplete\n"),
+ strerror (reap_errno));
+ else if (WIFEXITED (status) && WEXITSTATUS (status) == 0)
+ fprintf (stderr, _("Process exited with code 0\n"));
+ else if (WIFEXITED (status))
+ fprintf (stderr,
+ _("Warning: process exited with code %d; the workload did not"
+ " run to completion and the profile may be incomplete\n"),
+ WEXITSTATUS (status));
+ else if (WIFSIGNALED (status))
+ fprintf (stderr,
+ _("Warning: process killed by signal %d (e.g. SIGSEGV); profile"
+ " may be incomplete.\n"),
+ WTERMSIG (status));
+
+ if (target_mmaps_by_pid.empty ())
+ {
+ fprintf (stderr,
+ _("Error: No memory mappings found for target binary '%s'\n"),
+ path_basename (binary_path).c_str ());
+ return false;
+ }
+ if (target_executable_mappings_changed)
+ {
+ fprintf (stderr,
+ _("Error: target executable mappings changed during live "
+ "profiling; this is not supported\n"));
+ return false;
+ }
+
+ if (range_count_map.empty () && branch_count_map.empty ()
+ && address_frequencies.empty ())
+ {
+ fprintf (stderr,
+ use_branch_stack_for_correlation_
+ ? _("Error: No branch-stack samples collected (binary "
+ "may not match or workload too short)\n")
+ : _("Error: No sampled IPs collected (binary may not match "
+ "or workload too short)\n"));
+ return false;
+ }
+
+ fprintf (stderr,
+ _("Collected %" PRIu64 " samples, %zu ranges, %zu branches"),
+ total_samples,
+ range_count_map.size (),
+ branch_count_map.size ());
+ if (!address_frequencies.empty ())
+ fprintf (stderr, _(", %zu sample IPs"), address_frequencies.size ());
+ fprintf (stderr, "\n");
+ finalize_profile_after_capture ();
+ /* Report mass like offline, not raw record count. */
+ if (!sorted_ranges_.empty ())
+ total_samples = total_mass_from_range_counts (sorted_ranges_);
+ else
+ total_samples = total_mass_from_address_frequencies (address_frequencies);
+ return true;
+}
+
+/* Publish the profile, whose samples were translated through PID-scoped
+ mappings while being ingested. */
+void
+live_branch_stack_reader::finalize_profile_after_capture ()
+{
+ publish_dwarf_branch_stack_maps (
+ range_count_map, branch_count_map, &sorted_touch_pcs_);
+ target_mmaps_by_pid.clear ();
+ target_mmaps_by_pid.rehash (0);
+ target_mapping_pids.clear ();
+}
+
+} /* namespace perf_data. */
diff --git a/afdo_tools/src/live_branch_stack_reader.h
b/afdo_tools/src/live_branch_stack_reader.h
new file mode 100644
index 00000000000..a1099c5486a
--- /dev/null
+++ b/afdo_tools/src/live_branch_stack_reader.h
@@ -0,0 +1,94 @@
+/* Copyright The GNU Toolchain Authors
+
+This file is part of GCC.
+
+GCC is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 3, or (at your option) any later
+version.
+
+GCC is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
+for more details.
+
+You should have received a copy of the GNU General Public License
+along with GCC; see the file COPYING3. If not see
+<http://www.gnu.org/licenses/>. */
+
+#ifndef GCC_LIVE_BRANCH_STACK_READER_H
+#define GCC_LIVE_BRANCH_STACK_READER_H
+
+#include "branch_stack_reader_base.h"
+#include "profile_reader.h"
+
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <unordered_set>
+#include <vector>
+
+namespace perf_data {
+
+/* Live LBR/BRBE: run a command under perf and ingest its branch stacks. */
+class live_branch_stack_reader : public profile_reader,
+ public branch_stack_reader_base
+{
+ std::string binary_path;
+ std::vector<std::string> command_argv;
+ uint64_t total_samples = 0;
+ std::unordered_map<uint64_t, uint64_t> address_frequencies;
+
+ branch_stack_edge_count_map range_count_map;
+ branch_stack_edge_count_map branch_count_map;
+
+ std::string binary_path_resolved;
+ pid_mmap_map target_mmaps_by_pid;
+ std::unordered_set<uint32_t> target_mapping_pids;
+ bool target_executable_mappings_changed = false;
+ uint64_t sample_period;
+ uint64_t sample_type = 0;
+ uint64_t branch_sample_type = 0;
+ /* Records the kernel dropped because the ring buffer was full. */
+ uint64_t lost_records = 0;
+ /* True when producer head lapped the consumer tail. */
+ bool ring_overwrite = false;
+
+ bool is_target_binary (const std::string &mmap_filename) const;
+ void add_target_mapping (uint32_t pid, uint64_t start, uint64_t end,
+ uint64_t pgoff);
+ void note_replaced_target_mapping (uint32_t pid, uint64_t start,
+ uint64_t end);
+ void process_mmap_record (const uint8_t *payload, size_t payload_size,
+ uint16_t misc);
+ void process_mmap2_record (const uint8_t *payload, size_t payload_size);
+ bool process_sample_record (const uint8_t *payload, size_t payload_size);
+ void process_ring_buffer (void *mmap_base, uint64_t data_offset,
+ uint64_t data_size);
+ void finalize_profile_after_capture ();
+
+public:
+ live_branch_stack_reader (const std::string &_binary_path,
+ const std::vector<std::string> &_command_argv,
+ uint64_t _sample_period);
+
+ ~live_branch_stack_reader () override;
+
+ bool load () override;
+ const std::unordered_map<uint64_t, uint64_t> &
+ get_address_frequencies () const override
+ {
+ return address_frequencies;
+ }
+ uint64_t get_total_samples () const override { return total_samples; }
+
+ branch_stack_reader_base *as_branch_stack_reader () override { return this; }
+ const branch_stack_reader_base *as_branch_stack_reader () const override
+ {
+ return this;
+ }
+};
+
+} /* namespace perf_data. */
+
+#endif /* GCC_LIVE_BRANCH_STACK_READER_H. */
--
2.34.1
