Unlike x86-64, arm64 places alternative replacement instructions in .text, immediately after the affected function.
So if the replacement instructions have PC-relative branches without relocations, their offsets relative to the function have to remain constant. Achieve that by cloning the function's alternative replacements immediately after cloning the function itself. Signed-off-by: Josh Poimboeuf <[email protected]> --- tools/objtool/elf.c | 9 +++-- tools/objtool/include/objtool/elf.h | 7 +++- tools/objtool/klp-diff.c | 63 ++++++++++++++++++++++++----- 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index a4d9afa3a079c..a5b2929ea0fa9 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -1413,14 +1413,15 @@ int elf_add_string(struct elf *elf, struct section *strtab, const char *str) return -1; } - data = elf_add_data(elf, strtab, str, strlen(str) + 1); + data = elf_add_data(elf, strtab, str, strlen(str) + 1, true); if (!data) return -1; return data - strtab->data->d_buf; } -void *elf_add_data(struct elf *elf, struct section *sec, const void *data, size_t size) +void *elf_add_data(struct elf *elf, struct section *sec, const void *data, + size_t size, bool align) { unsigned long offset, size_old, size_new, alloc_size_old, alloc_size_new; Elf_Scn *s; @@ -1447,7 +1448,7 @@ void *elf_add_data(struct elf *elf, struct section *sec, const void *data, size_ } size_old = sec->data->d_size; - offset = ALIGN(size_old, sec->sh.sh_addralign); + offset = ALIGN(size_old, align ? sec->sh.sh_addralign : 1); size_new = offset + size; if (!sec->data_overallocated) @@ -1590,7 +1591,7 @@ static int elf_alloc_reloc(struct elf *elf, struct section *rsec) unsigned long nr_alloc_old = 0, nr_alloc_new; struct symbol *sym; - if (!elf_add_data(elf, rsec, NULL, elf_rela_size(elf))) + if (!elf_add_data(elf, rsec, NULL, elf_rela_size(elf), true)) return -1; rsec->data->d_type = ELF_T_RELA; diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index ab1d53ed23189..fba0a0e08f8b6 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -106,6 +106,8 @@ struct symbol { u8 included : 1; u8 klp : 1; u8 dont_correlate : 1; + u8 fake : 1; + u8 unalign : 1; struct list_head pv_target; struct reloc *relocs; struct section *group_sec; @@ -186,7 +188,7 @@ struct symbol *elf_create_symbol(struct elf *elf, const char *name, struct symbol *elf_create_section_symbol(struct elf *elf, struct section *sec); void *elf_add_data(struct elf *elf, struct section *sec, const void *data, - size_t size); + size_t size, bool align); int elf_find_string(struct elf *elf, struct section *strtab, const char *str); int elf_add_string(struct elf *elf, struct section *strtab, const char *str); @@ -532,6 +534,9 @@ static inline void set_sym_next_reloc(struct reloc *reloc, struct reloc *next) #define sec_for_each_sym_from(sec, sym) \ list_for_each_entry_from(sym, &sec->symbol_list, list) +#define sec_for_each_sym_continue(sec, sym) \ + list_for_each_entry_continue(sym, &sec->symbol_list, list) + #define sec_prev_sym(sym) \ sym->sec && sym->list.prev != &sym->sec->symbol_list ? \ list_prev_entry(sym, list) : NULL diff --git a/tools/objtool/klp-diff.c b/tools/objtool/klp-diff.c index e1d4d94c9d77c..b9624bd9439b9 100644 --- a/tools/objtool/klp-diff.c +++ b/tools/objtool/klp-diff.c @@ -1027,8 +1027,9 @@ static int clone_sym_relocs(struct elfs *e, struct symbol *patched_sym); static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym, bool data_too) { - struct section *out_sec = NULL; unsigned long offset = 0, pfx_size = 0; + bool align = !patched_sym->unalign; + struct section *out_sec = NULL; struct symbol *out_sym; if (data_too && !is_undef_sym(patched_sym)) { @@ -1054,7 +1055,7 @@ static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym } if (!is_sec_sym(patched_sym)) - offset = ALIGN(sec_size(out_sec), out_sec->sh.sh_addralign); + offset = ALIGN(sec_size(out_sec), align ? out_sec->sh.sh_addralign : 1); if (patched_sym->len || is_sec_sym(patched_sym)) { void *data = NULL; @@ -1072,7 +1073,7 @@ static struct symbol *__clone_symbol(struct elf *elf, struct symbol *patched_sym else size = patched_sym->len + pfx_size; - if (!elf_add_data(elf, out_sec, data, size)) + if (!elf_add_data(elf, out_sec, data, size, align)) return NULL; offset += pfx_size; @@ -1114,6 +1115,37 @@ static const char *sym_bind(struct symbol *sym) } } +static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym, + bool data_too); + +/* + * For arm64 alternatives, the replacement instructions come immediately after + * the function. Clone any such blocks of instructions in place to preserve + * their offsets relative to the function in case they have hard-coded PC + * relative branches. + */ +static int clone_inline_alternatives(struct elfs *e, struct symbol *patched_sym) +{ + struct symbol *next; + + if (!__is_defined(ARCH_HAS_INLINE_ALTS) || !is_func_sym(patched_sym)) + return 0; + + next = patched_sym; + sec_for_each_sym_continue(patched_sym->sec, next) { + if (next->offset < (patched_sym->offset + patched_sym->len) || + is_mapping_sym(next)) + continue; + if (!next->fake) + break; + next->unalign = 1; + if (!clone_symbol(e, next, true)) + return -1; + } + + return 0; +} + /* * Copy a symbol to the output object, optionally including its data and * relocations. @@ -1138,7 +1170,13 @@ static struct symbol *clone_symbol(struct elfs *e, struct symbol *patched_sym, if (!__clone_symbol(e->out, patched_sym, data_too)) return NULL; - if (data_too && clone_sym_relocs(e, patched_sym)) + if (!data_too || is_undef_sym(patched_sym)) + return patched_sym->clone; + + if (clone_sym_relocs(e, patched_sym)) + return NULL; + + if (clone_inline_alternatives(e, patched_sym)) return NULL; return patched_sym->clone; @@ -1551,7 +1589,7 @@ static int clone_reloc_klp(struct elfs *e, struct reloc *patched_reloc, memset(&klp_reloc, 0, sizeof(klp_reloc)); klp_reloc.type = reloc_type(patched_reloc); - if (!elf_add_data(e->out, klp_relocs, &klp_reloc, sizeof(klp_reloc))) + if (!elf_add_data(e->out, klp_relocs, &klp_reloc, sizeof(klp_reloc), true)) return -1; /* klp_reloc.offset */ @@ -1714,6 +1752,7 @@ static int create_fake_symbol(struct elf *elf, struct section *sec, unsigned long offset, size_t size) { char name[SYM_NAME_LEN]; + struct symbol *sym; unsigned int type; static int ctr; char *c; @@ -1730,7 +1769,13 @@ static int create_fake_symbol(struct elf *elf, struct section *sec, * while still allowing objdump to disassemble it. */ type = is_text_sec(sec) ? STT_NOTYPE : STT_OBJECT; - return elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size) ? 0 : -1; + + sym = elf_create_symbol(elf, name, sec, STB_LOCAL, type, offset, size); + if (!sym) + return -1; + + sym->fake = 1; + return 0; } /* @@ -2095,7 +2140,7 @@ static int create_klp_sections(struct elfs *e) return -1; /* allocate klp_object_ext */ - obj_data = elf_add_data(e->out, obj_sec, NULL, obj_size); + obj_data = elf_add_data(e->out, obj_sec, NULL, obj_size, true); if (!obj_data) return -1; @@ -2130,7 +2175,7 @@ static int create_klp_sections(struct elfs *e) continue; /* allocate klp_func_ext */ - func_data = elf_add_data(e->out, funcs_sec, NULL, func_size); + func_data = elf_add_data(e->out, funcs_sec, NULL, func_size, true); if (!func_data) return -1; @@ -2276,7 +2321,7 @@ static int copy_import_ns(struct elfs *e) } } - if (!elf_add_data(e->out, out_sec, import_ns, strlen(import_ns) + 1)) + if (!elf_add_data(e->out, out_sec, import_ns, strlen(import_ns) + 1, true)) return -1; } -- 2.53.0
