From: Pengfei Li <[email protected]> Add stack_map_bin, a native-endian version 1 binary export containing a header followed by stack id, depth, reference count and IP records. Userspace detects byte order from the magic value.
Stream records through seq_write() instead of allocating a private payload snapshot. open() records populated slot numbers in a bitmap and sets nr_stacks to its exact count. The bitmap uses at most 64 KiB, and exactly those records are emitted. It fixes record membership, not payload values, so ref_count can change before a selected record is written. seq_file releases reader_sem between read() calls. Track resets with a u64 generation sampled at open and checked at the start of each later seq pass. All runtime generation accesses are under reader_sem, so a plain u64 is sufficient. A changed generation returns -ESTALE rather than -EAGAIN, which seq_file reserves for internal traversal retries. Data already buffered by the current pass can be returned first. Once iteration has reached the end, a later read returns normal EOF even if reset intervened because no new-generation record can be appended. A reader that receives -ESTALE must reopen the file. Export the trampoline sentinel unchanged and apply trace_adjust_address() to other addresses, matching the text export. The records contain raw kernel instruction pointers after trace_adjust_address(). The auxiliary file has mode 0440 and open() uses tracing_check_open_get_tr(), matching the access boundary of existing raw tracing interfaces. This rejects LOCKDOWN_TRACEFS and pins the owning trace array. All open failure paths and release drop that reference after seq_file state is released. Failure to create the auxiliary file does not disable stackmap. Signed-off-by: Pengfei Li <[email protected]> --- kernel/trace/trace.c | 18 +- kernel/trace/trace_stackmap.c | 327 ++++++++++++++++++++++++++++++++-- kernel/trace/trace_stackmap.h | 20 +++ 3 files changed, 343 insertions(+), 22 deletions(-) diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 17df5f85da7a..68f5eb9a1e96 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -4109,8 +4109,8 @@ int set_tracer_flag(struct trace_array *tr, u64 mask, int enabled) #ifdef CONFIG_FTRACE_STACKMAP /* * STACKMAP is intentionally global-instance-only: the dedup map, - * its tracefs files and the lifetime/reset semantics are tied - * to the global trace + * its tracefs files (stack_map / stack_map_stat / stack_map_bin) + * and the lifetime/reset semantics are tied to the global trace * array. options/stackmap is hidden on secondary instances via * TOP_LEVEL_TRACE_FLAGS, but writes still reach set_tracer_flag() * through the aggregate trace_options file. Reject the enable on @@ -9412,17 +9412,21 @@ static __init void tracer_init_tracefs_work_func(struct work_struct *work) smp_store_release(&global_trace.stackmap, smap); WRITE_ONCE(stackmap_init_state, STACKMAP_INIT_DONE); /* - * stat is an auxiliary observability - * surface. If it fails to be created we keep - * dedup enabled -- the kernel side still - * works and stack_map alone is enough to - * resolve and reset; trace_create_file() + * stat and bin are auxiliary observability + * surfaces. If they fail to be created we + * keep dedup enabled (the kernel side still + * works, and stack_map alone is enough to + * resolve and reset); trace_create_file() * already pr_warn()s on failure. */ trace_create_file("stack_map_stat", TRACE_MODE_READ, NULL, smap, &ftrace_stackmap_stat_fops); + trace_create_file("stack_map_bin", + TRACE_MODE_READ, NULL, + smap, + &ftrace_stackmap_bin_fops); } } else { pr_warn("ftrace stackmap init failed, dedup disabled\n"); diff --git a/kernel/trace/trace_stackmap.c b/kernel/trace/trace_stackmap.c index 2382c7459712..439d3be9e351 100644 --- a/kernel/trace/trace_stackmap.c +++ b/kernel/trace/trace_stackmap.c @@ -35,17 +35,19 @@ * collision, linear probing finds the next slot and full memcmp * confirms the match. * - * Concurrent userspace readers (cat stack_map) get a best-effort - * snapshot. They are coherent with the hot path (smp_load_acquire on - * entry->val); they are also serialized against reset via - * smap->reader_sem (readers take it in shared mode, reset in + * Concurrent userspace readers (cat stack_map / stack_map_bin) get a + * best-effort snapshot. They are coherent with the hot path + * (smp_load_acquire on entry->val); they are also serialized against + * reset via smap->reader_sem (readers take it in shared mode, reset in * exclusive mode), so a reset cannot tear a single seq_file pass -- * it waits for the active read() pass to drop the rwsem before clearing - * the map. The hot path is coordinated with reset separately, via - * acquire/release on smap->resetting. + * the map. Binary readers spanning multiple read() calls additionally + * use a generation check. The hot path is coordinated with reset + * separately, via acquire/release on smap->resetting. */ #include <linux/kernel.h> +#include <linux/bitmap.h> #include <linux/slab.h> #include <linux/jhash.h> #include <linux/seq_file.h> @@ -74,7 +76,7 @@ /* * Memory ordering of entry->val: published with smp_store_release() * by the inserter; consumed with smp_load_acquire() by every reader - * that dereferences the elt (get_id, seq_show). This pairs + * that dereferences the elt (get_id, seq_show, bin_show). This pairs * the writes to elt->{nr,ips,ref_count} (initialized BEFORE the * publish) with the reads of those fields (which happen AFTER the * load). seq_start / seq_next only test val for NULL and use the @@ -119,11 +121,19 @@ struct ftrace_stackmap { struct stackmap_entry *entries; /* hash table */ struct stackmap_elt *elts; /* flat element pool */ atomic_t resetting; + /* + * Bumped by every completed reset, inside the reader_sem write + * section. A reader that spans several read() calls samples this + * at open() and revalidates it on each pass, so a reset landing + * between two reads is detected instead of silently splicing two + * generations of the map into one output stream. + */ + u64 generation; /* * Reader/reset serialization. Held in shared mode (read lock) - * across seq_file iteration; held in exclusive mode (write - * lock) by reset's clearing phase. The hot path (get_id) does - * not take this lock — it + * across each seq_file pass of the text and binary exports; + * held in exclusive mode (write lock) by reset's clearing + * phase. The hot path (get_id) does not take this lock — it * uses smp_load_acquire/smp_store_release on entry->val and * the resetting flag for the lock-free protocol. */ @@ -206,6 +216,7 @@ struct ftrace_stackmap *ftrace_stackmap_create(struct trace_array *tr) smap->hash_seed = get_random_u32(); atomic_set(&smap->next_elt, 0); atomic_set(&smap->resetting, 0); + smap->generation = 0; init_rwsem(&smap->reader_sem); return smap; @@ -307,10 +318,10 @@ static int ftrace_stackmap_reset(struct ftrace_stackmap *smap) /* * Take the reader_sem in exclusive mode. This serializes the - * memset against any tracefs reader (seq_file iteration) that - * may currently hold the rwsem for read. The Tasks RCU grace - * periods already drained the hot path; this rwsem covers - * process-context export readers. + * memset against any tracefs reader (a seq_file pass of the text + * or binary export) that may currently hold the rwsem for read. + * The Tasks RCU grace periods already drained the hot path; this + * rwsem covers process-context export readers. */ down_write(&smap->reader_sem); @@ -323,6 +334,13 @@ static int ftrace_stackmap_reset(struct ftrace_stackmap *smap) atomic_long_set(per_cpu_ptr(smap->drops, cpu), 0); } + /* + * Bump inside the write section so a reader holding the read + * side either sees the whole pre-reset generation or the whole + * post-reset one, never a counter from each. + */ + smap->generation++; + up_write(&smap->reader_sem); /* Release resetting=0 so new get_id() observes a cleared map. */ @@ -402,7 +420,7 @@ int ftrace_stackmap_get_id(struct ftrace_stackmap *smap, val = stackmap_load_elt(entry); /* * READ_ONCE(val->nr) keeps style consistent with - * the seq_show reader. nr is write-once + * the seq_show / bin_show readers. nr is write-once * (set before publish, never modified afterwards), * so the load is data-race-free, but READ_ONCE * silences any analysis tool that flags a plain @@ -853,3 +871,282 @@ const struct file_operations ftrace_stackmap_stat_fops = { .llseek = seq_lseek, .release = stackmap_stat_release, }; + +/* --- Binary export --- */ + +/* + * Binary counterpart of the text export, for userspace that would + * rather not parse symbols out of stack_map. + * + * Between resets, the element pool only grows and populated slots are + * not recycled. open() records the populated slot numbers in a bitmap, + * then the table is streamed in index order through seq_write(). This + * fixes the record set and makes header.nr_stacks exact without copying + * stack payloads. Even at the largest supported table, the bitmap is + * only 64 KiB per open, rather than a roughly 135 MiB element snapshot. + * A reset clears and reuses both the table and pool. + * + * seq_file drops reader_sem between read() calls, so a reset landing + * between two reads of the same fd would otherwise splice two + * generations of the map into a single stream. Text output would merely + * look confusing, but a binary consumer would parse it as one coherent + * dump, so open() samples the generation counter and every pass + * revalidates it, failing the read with -ESTALE once it has moved. ESTALE + * is intentional: seq_file treats -EAGAIN as an internal traverse retry, + * which would make lseek() on a stale fd spin forever. A reader that loses + * the race must reopen the file. + * + * The records contain raw kernel instruction pointers after + * trace_adjust_address(). VFS mode checks provide the tracefs permission + * boundary, and open additionally uses tracing_check_open_get_tr() for the + * tracing lockdown check and to pin the owning trace array for the lifetime + * of the file. + */ +struct stackmap_bin_iter { + struct ftrace_stackmap *smap; + u64 generation; /* sampled at open() */ + u32 nr_stacks; /* selected at open() */ + bool locked; /* reader_sem held by start() */ + /* + * Staging buffer for one full-depth entry, kept here rather than + * on the stack: the IP array alone is 512 bytes, which has no + * business in a seq_file show() frame. Emitting an entry in a + * single seq_write() also keeps it from being torn across a + * buffer-growth retry. + */ + char ebuf[sizeof(struct ftrace_stackmap_bin_entry) + + FTRACE_STACKMAP_MAX_DEPTH * sizeof(u64)] + __aligned(8); + /* Populated table slots selected during the open-time scan. */ + unsigned long *slots; +}; + +/* + * Advance to the first populated slot at or after *idx. Sets *idx to + * map_size when the table is exhausted, so a later call terminates + * immediately. + */ +static struct stackmap_entry *stackmap_bin_seek(struct stackmap_bin_iter *iter, + loff_t *idx) +{ + struct ftrace_stackmap *smap = iter->smap; + unsigned long i; + + i = find_next_bit(iter->slots, smap->map_size, *idx); + if (i >= smap->map_size) { + *idx = smap->map_size; + return NULL; + } + + *idx = i; + return &smap->entries[i]; +} + +/* + * Stream position 0 is the file header; position idx+1 is table slot + * idx. Deriving it from the slot index keeps the iteration idempotent + * across the retries seq_file performs when its buffer has to grow. + */ +static void *stackmap_bin_seq_start(struct seq_file *m, loff_t *pos) +{ + struct stackmap_bin_iter *iter = m->private; + struct ftrace_stackmap *smap = iter->smap; + struct stackmap_entry *entry; + loff_t idx; + + /* + * Serializes against the clearing phase of ftrace_stackmap_reset(). + * Dropped in stackmap_bin_seq_stop(), which seq_file calls after + * every pass, including the ones where start() reports an error. + */ + down_read(&smap->reader_sem); + iter->locked = true; + + /* + * The table has already been walked to the end. There is nothing + * left to hand over, so a reset that happened in the meantime + * cannot splice generations into this stream: report EOF rather + * than an error. Otherwise a reader that had already consumed the + * whole export would take a failure on its final read() and treat + * the data it holds as invalid. + */ + if (*pos > smap->map_size) + return NULL; + + if (smap->generation != iter->generation) + return ERR_PTR(-ESTALE); + + if (*pos == 0) + return SEQ_START_TOKEN; + + idx = *pos - 1; + entry = stackmap_bin_seek(iter, &idx); + *pos = idx + 1; + return entry; +} + +static void *stackmap_bin_seq_next(struct seq_file *m, void *v, loff_t *pos) +{ + struct stackmap_bin_iter *iter = m->private; + struct ftrace_stackmap *smap = iter->smap; + struct stackmap_entry *entry; + loff_t idx; + + if (v == SEQ_START_TOKEN) + idx = 0; + else + idx = ((struct stackmap_entry *)v - smap->entries) + 1; + + entry = stackmap_bin_seek(iter, &idx); + *pos = idx + 1; + return entry; +} + +static void stackmap_bin_seq_stop(struct seq_file *m, void *v) +{ + struct stackmap_bin_iter *iter = m->private; + + if (iter->locked) { + up_read(&iter->smap->reader_sem); + iter->locked = false; + } +} + +static int stackmap_bin_seq_show(struct seq_file *m, void *v) +{ + struct stackmap_bin_iter *iter = m->private; + struct ftrace_stackmap_bin_entry *e; + struct stackmap_entry *entry; + struct stackmap_elt *elt; + u32 i, nr; + + if (v == SEQ_START_TOKEN) { + struct ftrace_stackmap_bin_header hdr = { + .magic = FTRACE_STACKMAP_BIN_MAGIC, + .version = FTRACE_STACKMAP_BIN_VERSION, + .nr_stacks = iter->nr_stacks, + .reserved = 0, + }; + + seq_write(m, &hdr, sizeof(hdr)); + return 0; + } + + entry = v; + elt = stackmap_load_elt(entry); + if (!elt) + return 0; + + nr = READ_ONCE(elt->nr); + if (nr > FTRACE_STACKMAP_MAX_DEPTH) + nr = FTRACE_STACKMAP_MAX_DEPTH; + + e = (struct ftrace_stackmap_bin_entry *)iter->ebuf; + e->stack_id = entry - iter->smap->entries; + e->nr = nr; + e->ref_count = atomic_read(&elt->ref_count); + e->reserved = 0; + + for (i = 0; i < nr; i++) { + unsigned long ip = elt->ips[i]; + + /* + * Emit the trampoline marker verbatim so userspace can + * render it as [FTRACE TRAMPOLINE]; pass everything else + * through trace_adjust_address() so the binary export + * follows the same address-adjustment rules as the text + * export. + */ + if (ip == FTRACE_TRAMPOLINE_MARKER) + e->ips[i] = (u64)FTRACE_TRAMPOLINE_MARKER; + else + e->ips[i] = (u64)trace_adjust_address(iter->smap->tr, ip); + } + + seq_write(m, e, struct_size(e, ips, nr)); + return 0; +} + +static const struct seq_operations stackmap_bin_seq_ops = { + .start = stackmap_bin_seq_start, + .next = stackmap_bin_seq_next, + .stop = stackmap_bin_seq_stop, + .show = stackmap_bin_seq_show, +}; + +static int stackmap_bin_open(struct inode *inode, struct file *file) +{ + struct ftrace_stackmap *smap = inode->i_private; + struct stackmap_bin_iter *iter; + struct seq_file *m; + u32 nr = 0; + loff_t i; + int ret; + + if (!smap) + return -ENODEV; + + ret = tracing_check_open_get_tr(smap->tr); + if (ret) + return ret; + + ret = seq_open_private(file, &stackmap_bin_seq_ops, sizeof(*iter)); + if (ret) + goto put_tr; + + m = file->private_data; + iter = m->private; + iter->smap = smap; + iter->slots = kvcalloc(BITS_TO_LONGS(smap->map_size), + sizeof(*iter->slots), GFP_KERNEL_ACCOUNT); + if (!iter->slots) { + ret = -ENOMEM; + goto release_seq; + } + + /* + * Select the populated slots up front so header.nr_stacks exactly + * frames the records emitted by this fd. The bitmap copies only + * membership, not stack payloads; at the maximum map size it uses + * 64 KiB. Inserts after this scan are deliberately excluded. + */ + down_read(&smap->reader_sem); + iter->generation = smap->generation; + for (i = 0; i < smap->map_size; i++) { + if (READ_ONCE(smap->entries[i].key) && + stackmap_load_elt(&smap->entries[i])) { + __set_bit(i, iter->slots); + nr++; + } + } + up_read(&smap->reader_sem); + + iter->nr_stacks = nr; + return 0; + +release_seq: + seq_release_private(inode, file); +put_tr: + trace_array_put(smap->tr); + return ret; +} + +static int stackmap_bin_release(struct inode *inode, struct file *file) +{ + struct seq_file *m = file->private_data; + struct stackmap_bin_iter *iter = m->private; + struct ftrace_stackmap *smap = iter->smap; + int ret; + + kvfree(iter->slots); + ret = seq_release_private(inode, file); + trace_array_put(smap->tr); + return ret; +} + +const struct file_operations ftrace_stackmap_bin_fops = { + .open = stackmap_bin_open, + .read = seq_read, + .llseek = seq_lseek, + .release = stackmap_bin_release, +}; diff --git a/kernel/trace/trace_stackmap.h b/kernel/trace/trace_stackmap.h index 7615e346dfa6..80415b985627 100644 --- a/kernel/trace/trace_stackmap.h +++ b/kernel/trace/trace_stackmap.h @@ -7,6 +7,25 @@ #define FTRACE_STACKMAP_MAX_DEPTH 64 +/* Binary export format */ +#define FTRACE_STACKMAP_BIN_MAGIC 0x46534D42 /* 'FSMB' */ +#define FTRACE_STACKMAP_BIN_VERSION 1 + +struct ftrace_stackmap_bin_header { + u32 magic; + u32 version; + u32 nr_stacks; + u32 reserved; +}; + +struct ftrace_stackmap_bin_entry { + u32 stack_id; + u32 nr; + u32 ref_count; + u32 reserved; + u64 ips[]; /* nr entries follow the header */ +}; + struct trace_array; #ifdef CONFIG_FTRACE_STACKMAP @@ -20,6 +39,7 @@ int ftrace_stackmap_get_id(struct ftrace_stackmap *smap, extern const struct file_operations ftrace_stackmap_fops; extern const struct file_operations ftrace_stackmap_stat_fops; +extern const struct file_operations ftrace_stackmap_bin_fops; #else -- 2.34.1
