vprintk_store() is using a single static buffer as a temporary sprint buffer for the message text. This will not work once @logbuf_lock is removed. Replace the single static buffer with a pool of buffers.
The pool is large enough to support a worse case of 2 contexts per CPU (non-NMI and NMI). Signed-off-by: John Ogness <[email protected]> --- kernel/printk/printk.c | 142 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 137 insertions(+), 5 deletions(-) diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index cff13b33e926..763494d1d6b3 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -1872,6 +1872,112 @@ static void call_console_drivers(const char *ext_text, size_t ext_len, } } +/* + * The sprint buffers are used with interrupts disabled, so each CPU + * only requires 2 buffers: for non-NMI and NMI contexts. Recursive + * printk() calls are handled by the safe buffers. + */ +#define SPRINT_CTX_DEPTH 2 + +/* Static sprint buffers for early boot (only 1 CPU). */ +static DECLARE_BITMAP(sprint_static_textbuf_map, SPRINT_CTX_DEPTH); +static char sprint_static_textbuf[SPRINT_CTX_DEPTH * LOG_LINE_MAX]; + +/* Dynamically allocated sprint buffers. */ +static unsigned int sprint_dynamic_textbuf_count; +static unsigned long *sprint_dynamic_textbuf_map; +static char *sprint_dynamic_textbuf; + +/* + * Acquire an unused buffer, returning its index. If no buffer is + * available, @count is returned. + */ +static int _get_sprint_buf(unsigned long *map, int count) +{ + int index; + + do { + index = find_first_zero_bit(map, count); + if (index == count) + break; + /* + * Guarantee map changes are ordered for the other CPUs. + * Pairs with clear_bit() in _put_sprint_buf(). + */ + } while (test_and_set_bit(index, map)); + + return index; +} + +/* Mark the buffer @index as unused. */ +static void _put_sprint_buf(unsigned long *map, unsigned int count, unsigned int index) +{ + /* + * Guarantee map changes are ordered for the other CPUs. + * Pairs with test_and_set_bit() in _get_sprint_buf(). + */ + clear_bit(index, map); +} + +/* + * Get a buffer sized LOG_LINE_MAX for sprinting. On success, @id is set and + * interrupts are disabled. @id is used to put back the buffer. + * + * @id is non-negative for static buffers, negative for dynamic buffers. + */ +static char *get_sprint_buf(int *id, unsigned long *flags) +{ + unsigned int index; + + local_irq_save(*flags); + + /* First try with static pool. */ + index = _get_sprint_buf(sprint_static_textbuf_map, SPRINT_CTX_DEPTH); + if (index != SPRINT_CTX_DEPTH) { + *id = index; + return &sprint_static_textbuf[index * LOG_LINE_MAX]; + } + + /* + * Fallback to dynamic pool, if available. + * + * Guarantee the buffer is loaded before loading the map and count. + * Pairs with smp_store_release() in console_init(). + */ + if (smp_load_acquire(&sprint_dynamic_textbuf)) { + index = _get_sprint_buf(sprint_dynamic_textbuf_map, + sprint_dynamic_textbuf_count); + if (index != sprint_dynamic_textbuf_count) { + /* Convert to dynamic buffer representation. */ + *id = -index - 1; + return &sprint_dynamic_textbuf[index * LOG_LINE_MAX]; + } + } + + /* Failed to get a buffer. */ + local_irq_restore(*flags); + return NULL; +} + +/* Put back an sprint buffer and restore interrupts. */ +static void put_sprint_buf(int id, unsigned long flags) +{ + unsigned int index; + + if (id >= 0) { + index = id; + _put_sprint_buf(sprint_static_textbuf_map, + SPRINT_CTX_DEPTH, index); + } else { + /* Convert from dynamic buffer representation. */ + index = -id - 1; + _put_sprint_buf(sprint_dynamic_textbuf_map, + sprint_dynamic_textbuf_count, index); + } + + local_irq_restore(flags); +} + int printk_delay_msec __read_mostly; static inline void printk_delay(void) @@ -1921,21 +2027,26 @@ static size_t log_output(int facility, int level, enum log_flags lflags, dev_info, text, text_len); } -/* Must be called under logbuf_lock. */ int vprintk_store(int facility, int level, const struct dev_printk_info *dev_info, const char *fmt, va_list args) { - static char textbuf[LOG_LINE_MAX]; - char *text = textbuf; size_t text_len; enum log_flags lflags = 0; + unsigned long irqflags; + int sprint_id; + char *text; + int ret; + + text = get_sprint_buf(&sprint_id, &irqflags); + if (WARN_ON_ONCE(!text)) + return 0; /* * The printf needs to come first; we need the syslog * prefix which might be passed-in as a parameter. */ - text_len = vscnprintf(text, sizeof(textbuf), fmt, args); + text_len = vscnprintf(text, LOG_LINE_MAX, fmt, args); /* mark and strip a trailing newline */ if (text_len && text[text_len-1] == '\n') { @@ -1968,7 +2079,11 @@ int vprintk_store(int facility, int level, if (dev_info) lflags |= LOG_NEWLINE; - return log_output(facility, level, lflags, dev_info, text, text_len); + ret = log_output(facility, level, lflags, dev_info, text, text_len); + + put_sprint_buf(sprint_id, irqflags); + + return ret; } asmlinkage int vprintk_emit(int facility, int level, @@ -2920,6 +3035,23 @@ void __init console_init(void) initcall_t call; initcall_entry_t *ce; + sprint_dynamic_textbuf_count = num_possible_cpus() * SPRINT_CTX_DEPTH; + sprint_dynamic_textbuf_map = bitmap_alloc(sprint_dynamic_textbuf_count, GFP_KERNEL); + if (sprint_dynamic_textbuf_map) { + /* + * Guarantee the buffer is stored after storing the + * map and count. Pairs with smp_load_acquire() in + * get_sprint_buf(). + */ + smp_store_release(&sprint_dynamic_textbuf, + kmalloc(sprint_dynamic_textbuf_count * LOG_LINE_MAX, GFP_KERNEL)); + } + if (!sprint_dynamic_textbuf) { + bitmap_free(sprint_dynamic_textbuf_map); + pr_info("sprint_dynamic_textbuf: %u buffers not available\n", + sprint_dynamic_textbuf_count); + } + /* Setup the default TTY line discipline. */ n_tty_init(); -- 2.20.1
