In preparation to introducing atomic replace, introduce iterators for klp_func and klp_object, such that objects and functions can be dynmically allocated (needed for atomic replace). Note that this patch is careful, not to grow the size of klp_func as that's the most common data structure. This patch is intended to effectively be a no-op until atomic replace is introduced.
Signed-off-by: Jason Baron <[email protected]> Cc: Josh Poimboeuf <[email protected]> Cc: Jessica Yu <[email protected]> Cc: Jiri Kosina <[email protected]> Cc: Miroslav Benes <[email protected]> Cc: Petr Mladek <[email protected]> --- include/linux/livepatch.h | 106 ++++++++++++++++++++++++++++++++++++++++-- kernel/livepatch/core.c | 25 +++++++--- kernel/livepatch/patch.c | 9 ++-- kernel/livepatch/transition.c | 18 ++++--- 4 files changed, 137 insertions(+), 21 deletions(-) diff --git a/include/linux/livepatch.h b/include/linux/livepatch.h index 194991e..5038337 100644 --- a/include/linux/livepatch.h +++ b/include/linux/livepatch.h @@ -24,6 +24,7 @@ #include <linux/module.h> #include <linux/ftrace.h> #include <linux/completion.h> +#include <linux/list.h> #if IS_ENABLED(CONFIG_LIVEPATCH) @@ -88,10 +89,23 @@ struct klp_func { }; /** + * struct klp_func_no_op - internal object used to link no_op functions, which + avoids the need to bloat struct klp_func + * @orig_func: embeds struct klp_func + * @func_entry: used link struct klp_func_no_op to struct klp_object + */ +struct klp_func_no_op { + struct klp_func orig_func; + struct list_head func_entry; +}; + +/** * struct klp_object - kernel object structure for live patching * @name: module name (or NULL for vmlinux) * @funcs: function entries for functions to be patched in the object * @kobj: kobject for sysfs resources + * @func_list: head of list for struct klp_func_no_op + * @obj_entry: used to link struct klp_object to struct klp_patch * @mod: kernel module associated with the patched object * (NULL for vmlinux) * @patched: the object's funcs have been added to the klp_ops list @@ -103,6 +117,8 @@ struct klp_object { /* internal */ struct kobject kobj; + struct list_head func_list; + struct list_head obj_entry; struct module *mod; bool patched; }; @@ -114,6 +130,7 @@ struct klp_object { * @immediate: patch all funcs immediately, bypassing safety mechanisms * @list: list node for global list of registered patches * @kobj: kobject for sysfs resources + * @obj_list: head of list for dynamically allocated struct klp_object * @enabled: the patch is enabled (but operation may be incomplete) * @finish: for waiting till it is safe to remove the patch module */ @@ -126,17 +143,96 @@ struct klp_patch { /* internal */ struct list_head list; struct kobject kobj; + struct list_head obj_list; bool enabled; struct completion finish; }; -#define klp_for_each_object(patch, obj) \ +struct obj_iter { + struct klp_object *obj; + struct list_head *obj_list_head; + struct list_head *obj_list_pos; +}; + +static inline struct klp_object *obj_iter_next(struct obj_iter *iter) +{ + struct klp_object *obj; + + if (iter->obj->funcs || iter->obj->name) { + obj = iter->obj; + iter->obj++; + } else { + if (iter->obj_list_pos == iter->obj_list_head) { + obj = NULL; + } else { + obj = list_entry(iter->obj_list_pos, struct klp_object, + obj_entry); + iter->obj_list_pos = iter->obj_list_pos->next; + } + } + + return obj; +} + +static inline struct klp_object *obj_iter_init(struct klp_patch *patch, + struct obj_iter *iter) +{ + iter->obj = patch->objs; + iter->obj_list_head = &patch->obj_list; + iter->obj_list_pos = iter->obj_list_head->next; + + return obj_iter_next(iter); +} + +#define klp_for_each_object(patch, obj, iter) \ + for (obj = obj_iter_init(patch, iter); obj; obj = obj_iter_next(iter)) + +#define klp_for_each_object_core(patch, obj) \ for (obj = patch->objs; obj->funcs || obj->name; obj++) -#define klp_for_each_func(obj, func) \ - for (func = obj->funcs; \ - func->old_name || func->new_func || func->old_sympos; \ - func++) +struct func_iter { + struct klp_func *func; + struct list_head *func_list_head; + struct list_head *func_list_pos; +}; + +static inline struct klp_func *func_iter_next(struct func_iter *iter) +{ + struct klp_func *func; + struct klp_func_no_op *func_no_op; + + if (iter->func->old_name || iter->func->new_func || + iter->func->old_sympos) { + func = iter->func; + iter->func++; + } else { + if (iter->func_list_pos == iter->func_list_head) { + func = NULL; + } else { + func_no_op = list_entry(iter->func_list_pos, + struct klp_func_no_op, + func_entry); + func = &func_no_op->orig_func; + iter->func_list_pos = iter->func_list_pos->next; + } + } + + return func; +} + +static inline struct klp_func *func_iter_init(struct klp_object *obj, + struct func_iter *iter) +{ + iter->func = obj->funcs; + iter->func_list_head = &obj->func_list; + iter->func_list_pos = iter->func_list_head->next; + + return func_iter_next(iter); +} + +#define klp_for_each_func(obj, func, iter) \ + for (func = func_iter_init(obj, iter); func; \ + func = func_iter_next(iter)) int klp_register_patch(struct klp_patch *); int klp_unregister_patch(struct klp_patch *); diff --git a/kernel/livepatch/core.c b/kernel/livepatch/core.c index b9628e4..e63f478 100644 --- a/kernel/livepatch/core.c +++ b/kernel/livepatch/core.c @@ -346,6 +346,7 @@ EXPORT_SYMBOL_GPL(klp_disable_patch); static int __klp_enable_patch(struct klp_patch *patch) { struct klp_object *obj; + struct obj_iter o_iter; int ret; if (klp_transition_patch) @@ -384,7 +385,7 @@ static int __klp_enable_patch(struct klp_patch *patch) */ smp_wmb(); - klp_for_each_object(patch, obj) { + klp_for_each_object(patch, obj, &o_iter) { if (!klp_is_object_loaded(obj)) continue; @@ -571,10 +572,11 @@ static void klp_free_funcs_limited(struct klp_object *obj, static void klp_free_object_loaded(struct klp_object *obj) { struct klp_func *func; + struct func_iter f_iter; obj->mod = NULL; - klp_for_each_func(obj, func) + klp_for_each_func(obj, func, &f_iter) func->old_addr = 0; } @@ -630,6 +632,7 @@ static int klp_init_object_loaded(struct klp_patch *patch, struct klp_object *obj) { struct klp_func *func; + struct func_iter f_iter; int ret; module_disable_ro(patch->mod); @@ -642,7 +645,7 @@ static int klp_init_object_loaded(struct klp_patch *patch, arch_klp_init_object_loaded(patch, obj); module_enable_ro(patch->mod, true); - klp_for_each_func(obj, func) { + klp_for_each_func(obj, func, &f_iter) { ret = klp_find_object_symbol(obj->name, func->old_name, func->old_sympos, &func->old_addr); @@ -672,6 +675,7 @@ static int klp_init_object_loaded(struct klp_patch *patch, static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) { struct klp_func *func; + struct func_iter f_iter; int ret; const char *name; @@ -689,7 +693,7 @@ static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) if (ret) return ret; - klp_for_each_func(obj, func) { + klp_for_each_func(obj, func, &f_iter) { ret = klp_init_func(obj, func); if (ret) goto free; @@ -712,6 +716,7 @@ static int klp_init_object(struct klp_patch *patch, struct klp_object *obj) static int klp_init_patch(struct klp_patch *patch) { struct klp_object *obj; + struct obj_iter o_iter; int ret; if (!patch->objs) @@ -729,7 +734,11 @@ static int klp_init_patch(struct klp_patch *patch) return ret; } - klp_for_each_object(patch, obj) { + INIT_LIST_HEAD(&patch->obj_list); + klp_for_each_object_core(patch, obj) + INIT_LIST_HEAD(&obj->func_list); + + klp_for_each_object(patch, obj, &o_iter) { ret = klp_init_object(patch, obj); if (ret) goto free; @@ -835,6 +844,7 @@ int klp_module_coming(struct module *mod) int ret; struct klp_patch *patch; struct klp_object *obj; + struct obj_iter o_iter; if (WARN_ON(mod->state != MODULE_STATE_COMING)) return -EINVAL; @@ -848,7 +858,7 @@ int klp_module_coming(struct module *mod) mod->klp_alive = true; list_for_each_entry(patch, &klp_patches, list) { - klp_for_each_object(patch, obj) { + klp_for_each_object(patch, obj, &o_iter) { if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) continue; @@ -904,6 +914,7 @@ void klp_module_going(struct module *mod) { struct klp_patch *patch; struct klp_object *obj; + struct obj_iter o_iter; if (WARN_ON(mod->state != MODULE_STATE_GOING && mod->state != MODULE_STATE_COMING)) @@ -918,7 +929,7 @@ void klp_module_going(struct module *mod) mod->klp_alive = false; list_for_each_entry(patch, &klp_patches, list) { - klp_for_each_object(patch, obj) { + klp_for_each_object(patch, obj, &o_iter) { if (!klp_is_module(obj) || strcmp(obj->name, mod->name)) continue; diff --git a/kernel/livepatch/patch.c b/kernel/livepatch/patch.c index 52c4e90..1cfdabc 100644 --- a/kernel/livepatch/patch.c +++ b/kernel/livepatch/patch.c @@ -238,8 +238,9 @@ static int klp_patch_func(struct klp_func *func) void klp_unpatch_object(struct klp_object *obj) { struct klp_func *func; + struct func_iter f_iter; - klp_for_each_func(obj, func) + klp_for_each_func(obj, func, &f_iter) if (func->patched) klp_unpatch_func(func); @@ -249,12 +250,13 @@ void klp_unpatch_object(struct klp_object *obj) int klp_patch_object(struct klp_object *obj) { struct klp_func *func; + struct func_iter f_iter; int ret; if (WARN_ON(obj->patched)) return -EINVAL; - klp_for_each_func(obj, func) { + klp_for_each_func(obj, func, &f_iter) { ret = klp_patch_func(func); if (ret) { klp_unpatch_object(obj); @@ -269,8 +271,9 @@ int klp_patch_object(struct klp_object *obj) void klp_unpatch_objects(struct klp_patch *patch) { struct klp_object *obj; + struct obj_iter o_iter; - klp_for_each_object(patch, obj) + klp_for_each_object(patch, obj, &o_iter) if (obj->patched) klp_unpatch_object(obj); } diff --git a/kernel/livepatch/transition.c b/kernel/livepatch/transition.c index b004a1f..e112826 100644 --- a/kernel/livepatch/transition.c +++ b/kernel/livepatch/transition.c @@ -81,6 +81,8 @@ static void klp_complete_transition(void) struct task_struct *g, *task; unsigned int cpu; bool immediate_func = false; + struct obj_iter o_iter; + struct func_iter f_iter; if (klp_target_state == KLP_UNPATCHED) { /* @@ -101,8 +103,8 @@ static void klp_complete_transition(void) if (klp_transition_patch->immediate) goto done; - klp_for_each_object(klp_transition_patch, obj) { - klp_for_each_func(obj, func) { + klp_for_each_object(klp_transition_patch, obj, &o_iter) { + klp_for_each_func(obj, func, &f_iter) { func->transition = false; if (func->immediate) immediate_func = true; @@ -244,6 +246,8 @@ static int klp_check_stack(struct task_struct *task, char *err_buf) struct stack_trace trace; struct klp_object *obj; struct klp_func *func; + struct obj_iter o_iter; + struct func_iter f_iter; int ret; trace.skip = 0; @@ -259,10 +263,10 @@ static int klp_check_stack(struct task_struct *task, char *err_buf) return ret; } - klp_for_each_object(klp_transition_patch, obj) { + klp_for_each_object(klp_transition_patch, obj, &o_iter) { if (!obj->patched) continue; - klp_for_each_func(obj, func) { + klp_for_each_func(obj, func, &f_iter) { ret = klp_check_stack_func(func, &trace); if (ret) { snprintf(err_buf, STACK_ERR_BUF_SIZE, @@ -470,6 +474,8 @@ void klp_init_transition(struct klp_patch *patch, int state) unsigned int cpu; struct klp_object *obj; struct klp_func *func; + struct obj_iter o_iter; + struct func_iter f_iter; int initial_state = !state; WARN_ON_ONCE(klp_target_state != KLP_UNDEFINED); @@ -531,8 +537,8 @@ void klp_init_transition(struct klp_patch *patch, int state) * When unpatching, the funcs are already in the func_stack and so are * already visible to the ftrace handler. */ - klp_for_each_object(patch, obj) - klp_for_each_func(obj, func) + klp_for_each_object(patch, obj, &o_iter) + klp_for_each_func(obj, func, &f_iter) func->transition = true; } -- 2.6.1
