[PATCH v7 10/10] x86/kernel: jump_table: use relative references

2018-01-02 Thread Ard Biesheuvel
Similar to the arm64 case, 64-bit x86 can benefit from using 32-bit
relative references rather than 64-bit absolute ones when emitting
struct jump_entry instances. Not only does this reduce the memory
footprint of the entries themselves by 50%, it also removes the need
for carrying relocation metadata on relocatable builds (i.e., for KASLR)
which saves a fair chunk of .init space as well (although the savings
are not as dramatic as on arm64)

Signed-off-by: Ard Biesheuvel 
---
 arch/x86/include/asm/jump_label.h | 35 
 arch/x86/kernel/jump_label.c  | 16 +
 tools/objtool/special.c   |  4 +--
 3 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/arch/x86/include/asm/jump_label.h 
b/arch/x86/include/asm/jump_label.h
index 009ff2699d07..35fc2c5ec846 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -36,8 +36,8 @@ static __always_inline bool arch_static_branch(struct 
static_key *key, bool bran
asm_volatile_goto("1:"
".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
".pushsection __jump_table,  \"aw\" \n\t"
-   _ASM_ALIGN "\n\t"
-   _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
+   ".balign 4\n\t"
+   ".long 1b - ., %l[l_yes] - ., %c0 + %c1 - .\n\t"
".popsection \n\t"
: :  "i" (key), "i" (branch) : : l_yes);
 
@@ -52,8 +52,8 @@ static __always_inline bool arch_static_branch_jump(struct 
static_key *key, bool
".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
"2:\n\t"
".pushsection __jump_table,  \"aw\" \n\t"
-   _ASM_ALIGN "\n\t"
-   _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
+   ".balign 4\n\t"
+   ".long 1b - ., %l[l_yes] - ., %c0 + %c1 - .\n\t"
".popsection \n\t"
: :  "i" (key), "i" (branch) : : l_yes);
 
@@ -69,19 +69,26 @@ typedef u32 jump_label_t;
 #endif
 
 struct jump_entry {
-   jump_label_t code;
-   jump_label_t target;
-   jump_label_t key;
+   s32 code;
+   s32 target;
+   s32 key;
 };
 
 static inline jump_label_t jump_entry_code(const struct jump_entry *entry)
 {
-   return entry->code;
+   return (unsigned long)>code + entry->code;
+}
+
+static inline jump_label_t jump_entry_target(const struct jump_entry *entry)
+{
+   return (unsigned long)>target + entry->target;
 }
 
 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
 {
-   return (struct static_key *)((unsigned long)entry->key & ~1UL);
+   unsigned long key = (unsigned long)>key + entry->key;
+
+   return (struct static_key *)(key & ~1UL);
 }
 
 static inline bool jump_entry_is_branch(const struct jump_entry *entry)
@@ -99,7 +106,7 @@ static inline void jump_entry_set_module_init(struct 
jump_entry *entry)
entry->code = 0;
 }
 
-#define jump_label_swapNULL
+void jump_label_swap(void *a, void *b, int size);
 
 #else  /* __ASSEMBLY__ */
 
@@ -114,8 +121,8 @@ static inline void jump_entry_set_module_init(struct 
jump_entry *entry)
.byte   STATIC_KEY_INIT_NOP
.endif
.pushsection __jump_table, "aw"
-   _ASM_ALIGN
-   _ASM_PTR.Lstatic_jump_\@, \target, \key
+   .balign 4
+   .long   .Lstatic_jump_\@ - ., \target - ., \key - .
.popsection
 .endm
 
@@ -130,8 +137,8 @@ static inline void jump_entry_set_module_init(struct 
jump_entry *entry)
 .Lstatic_jump_after_\@:
.endif
.pushsection __jump_table, "aw"
-   _ASM_ALIGN
-   _ASM_PTR.Lstatic_jump_\@, \target, \key + 1
+   .balign 4
+   .long   .Lstatic_jump_\@ - ., \target - ., \key + 1 - .
.popsection
 .endm
 
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index d64296092ef5..cc5034b42335 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -149,4 +149,20 @@ __init_or_module void 
arch_jump_label_transform_static(struct jump_entry *entry,
__jump_label_transform(entry, type, text_poke_early, 1);
 }
 
+void jump_label_swap(void *a, void *b, int size)
+{
+   long delta = (unsigned long)a - (unsigned long)b;
+   struct jump_entry *jea = a;
+   struct jump_entry *jeb = b;
+   struct jump_entry tmp = *jea;
+
+   jea->code   = jeb->code - delta;
+   jea->target = jeb->target - delta;
+   jea->key= jeb->key - delta;
+
+   jeb->code   = tmp.code + delta;
+   jeb->target = tmp.target + delta;
+   jeb->key= tmp.key + delta;
+}
+
 #endif
diff --git a/tools/objtool/special.c b/tools/objtool/special.c
index 84f001d52322..98ae55b39037 100644
--- a/tools/objtool/special.c
+++ b/tools/objtool/special.c
@@ -30,9 +30,9 @@
 #define EX_ORIG_OFFSET 0
 #define EX_NEW_OFFSET  4
 

[PATCH v7 10/10] x86/kernel: jump_table: use relative references

2018-01-02 Thread Ard Biesheuvel
Similar to the arm64 case, 64-bit x86 can benefit from using 32-bit
relative references rather than 64-bit absolute ones when emitting
struct jump_entry instances. Not only does this reduce the memory
footprint of the entries themselves by 50%, it also removes the need
for carrying relocation metadata on relocatable builds (i.e., for KASLR)
which saves a fair chunk of .init space as well (although the savings
are not as dramatic as on arm64)

Signed-off-by: Ard Biesheuvel 
---
 arch/x86/include/asm/jump_label.h | 35 
 arch/x86/kernel/jump_label.c  | 16 +
 tools/objtool/special.c   |  4 +--
 3 files changed, 39 insertions(+), 16 deletions(-)

diff --git a/arch/x86/include/asm/jump_label.h 
b/arch/x86/include/asm/jump_label.h
index 009ff2699d07..35fc2c5ec846 100644
--- a/arch/x86/include/asm/jump_label.h
+++ b/arch/x86/include/asm/jump_label.h
@@ -36,8 +36,8 @@ static __always_inline bool arch_static_branch(struct 
static_key *key, bool bran
asm_volatile_goto("1:"
".byte " __stringify(STATIC_KEY_INIT_NOP) "\n\t"
".pushsection __jump_table,  \"aw\" \n\t"
-   _ASM_ALIGN "\n\t"
-   _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
+   ".balign 4\n\t"
+   ".long 1b - ., %l[l_yes] - ., %c0 + %c1 - .\n\t"
".popsection \n\t"
: :  "i" (key), "i" (branch) : : l_yes);
 
@@ -52,8 +52,8 @@ static __always_inline bool arch_static_branch_jump(struct 
static_key *key, bool
".byte 0xe9\n\t .long %l[l_yes] - 2f\n\t"
"2:\n\t"
".pushsection __jump_table,  \"aw\" \n\t"
-   _ASM_ALIGN "\n\t"
-   _ASM_PTR "1b, %l[l_yes], %c0 + %c1 \n\t"
+   ".balign 4\n\t"
+   ".long 1b - ., %l[l_yes] - ., %c0 + %c1 - .\n\t"
".popsection \n\t"
: :  "i" (key), "i" (branch) : : l_yes);
 
@@ -69,19 +69,26 @@ typedef u32 jump_label_t;
 #endif
 
 struct jump_entry {
-   jump_label_t code;
-   jump_label_t target;
-   jump_label_t key;
+   s32 code;
+   s32 target;
+   s32 key;
 };
 
 static inline jump_label_t jump_entry_code(const struct jump_entry *entry)
 {
-   return entry->code;
+   return (unsigned long)>code + entry->code;
+}
+
+static inline jump_label_t jump_entry_target(const struct jump_entry *entry)
+{
+   return (unsigned long)>target + entry->target;
 }
 
 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)
 {
-   return (struct static_key *)((unsigned long)entry->key & ~1UL);
+   unsigned long key = (unsigned long)>key + entry->key;
+
+   return (struct static_key *)(key & ~1UL);
 }
 
 static inline bool jump_entry_is_branch(const struct jump_entry *entry)
@@ -99,7 +106,7 @@ static inline void jump_entry_set_module_init(struct 
jump_entry *entry)
entry->code = 0;
 }
 
-#define jump_label_swapNULL
+void jump_label_swap(void *a, void *b, int size);
 
 #else  /* __ASSEMBLY__ */
 
@@ -114,8 +121,8 @@ static inline void jump_entry_set_module_init(struct 
jump_entry *entry)
.byte   STATIC_KEY_INIT_NOP
.endif
.pushsection __jump_table, "aw"
-   _ASM_ALIGN
-   _ASM_PTR.Lstatic_jump_\@, \target, \key
+   .balign 4
+   .long   .Lstatic_jump_\@ - ., \target - ., \key - .
.popsection
 .endm
 
@@ -130,8 +137,8 @@ static inline void jump_entry_set_module_init(struct 
jump_entry *entry)
 .Lstatic_jump_after_\@:
.endif
.pushsection __jump_table, "aw"
-   _ASM_ALIGN
-   _ASM_PTR.Lstatic_jump_\@, \target, \key + 1
+   .balign 4
+   .long   .Lstatic_jump_\@ - ., \target - ., \key + 1 - .
.popsection
 .endm
 
diff --git a/arch/x86/kernel/jump_label.c b/arch/x86/kernel/jump_label.c
index d64296092ef5..cc5034b42335 100644
--- a/arch/x86/kernel/jump_label.c
+++ b/arch/x86/kernel/jump_label.c
@@ -149,4 +149,20 @@ __init_or_module void 
arch_jump_label_transform_static(struct jump_entry *entry,
__jump_label_transform(entry, type, text_poke_early, 1);
 }
 
+void jump_label_swap(void *a, void *b, int size)
+{
+   long delta = (unsigned long)a - (unsigned long)b;
+   struct jump_entry *jea = a;
+   struct jump_entry *jeb = b;
+   struct jump_entry tmp = *jea;
+
+   jea->code   = jeb->code - delta;
+   jea->target = jeb->target - delta;
+   jea->key= jeb->key - delta;
+
+   jeb->code   = tmp.code + delta;
+   jeb->target = tmp.target + delta;
+   jeb->key= tmp.key + delta;
+}
+
 #endif
diff --git a/tools/objtool/special.c b/tools/objtool/special.c
index 84f001d52322..98ae55b39037 100644
--- a/tools/objtool/special.c
+++ b/tools/objtool/special.c
@@ -30,9 +30,9 @@
 #define EX_ORIG_OFFSET 0
 #define EX_NEW_OFFSET  4
 
-#define JUMP_ENTRY_SIZE