Re: [RFC PATCH 01/12] riscv: Move kernel mapping outside of linear mapping
Le 1/6/21 à 1:44 AM, Anup Patel a écrit :
On Wed, Jan 6, 2021 at 12:06 PM Alex Ghiti wrote:
Hi Anup,
Le 1/5/21 à 6:40 AM, Anup Patel a écrit :
On Tue, Jan 5, 2021 at 1:29 AM Alexandre Ghiti wrote:
This is a preparatory patch for relocatable kernel and sv48 support.
The kernel used to be linked at PAGE_OFFSET address therefore we could use
the linear mapping for the kernel mapping. But the relocated kernel base
address will be different from PAGE_OFFSET and since in the linear mapping,
two different virtual addresses cannot point to the same physical address,
the kernel mapping needs to lie outside the linear mapping so that we don't
have to copy it at the same physical offset.
The kernel mapping is moved to the last 2GB of the address space and then
BPF and modules are also pushed to the same range since they have to lie
close to the kernel inside a 2GB window.
Note then that KASLR implementation will simply have to move the kernel in
this 2GB range and modify BPF/modules regions accordingly.
In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.
Awesome ! This is a good approach with no performance impact.
Suggested-by: Arnd Bergmann
Signed-off-by: Alexandre Ghiti
---
arch/riscv/boot/loader.lds.S | 3 +-
arch/riscv/include/asm/page.h| 10 -
arch/riscv/include/asm/pgtable.h | 39 +--
arch/riscv/kernel/head.S | 3 +-
arch/riscv/kernel/module.c | 4 +-
arch/riscv/kernel/vmlinux.lds.S | 3 +-
arch/riscv/mm/init.c | 65
arch/riscv/mm/physaddr.c | 2 +-
8 files changed, 94 insertions(+), 35 deletions(-)
diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
index 47a5003c2e28..62d94696a19c 100644
--- a/arch/riscv/boot/loader.lds.S
+++ b/arch/riscv/boot/loader.lds.S
@@ -1,13 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include
+#include
OUTPUT_ARCH(riscv)
ENTRY(_start)
SECTIONS
{
- . = PAGE_OFFSET;
+ . = KERNEL_LINK_ADDR;
.payload : {
*(.payload)
diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index 2d50f76efe48..98188e315e8d 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -90,18 +90,26 @@ typedef struct page *pgtable_t;
#ifdef CONFIG_MMU
extern unsigned long va_pa_offset;
+extern unsigned long va_kernel_pa_offset;
extern unsigned long pfn_base;
#define ARCH_PFN_OFFSET(pfn_base)
#else
#define va_pa_offset 0
+#define va_kernel_pa_offset0
#define ARCH_PFN_OFFSET(PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
extern unsigned long max_low_pfn;
extern unsigned long min_low_pfn;
+extern unsigned long kernel_virt_addr;
#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset))
-#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
+#define linear_mapping_va_to_pa(x) ((unsigned long)(x) - va_pa_offset)
+#define kernel_mapping_va_to_pa(x) \
+ ((unsigned long)(x) - va_kernel_pa_offset)
+#define __va_to_pa_nodebug(x) \
+ (((x) < KERNEL_LINK_ADDR) ? \
+ linear_mapping_va_to_pa(x) : kernel_mapping_va_to_pa(x))
#ifdef CONFIG_DEBUG_VIRTUAL
extern phys_addr_t __virt_to_phys(unsigned long x);
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 183f1f4b2ae6..102b728ca146 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -11,23 +11,32 @@
#include
-#ifndef __ASSEMBLY__
-
-/* Page Upper Directory not used in RISC-V */
-#include
-#include
-#include
-#include
+#ifndef CONFIG_MMU
+#define KERNEL_VIRT_ADDR PAGE_OFFSET
+#define KERNEL_LINK_ADDR PAGE_OFFSET
+#else
-#ifdef CONFIG_MMU
+#define ADDRESS_SPACE_END (UL(-1))
+/*
+ * Leave 2GB for kernel, modules and BPF at the end of the address space
+ */
+#define KERNEL_VIRT_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
+#define KERNEL_LINK_ADDR KERNEL_VIRT_ADDR
#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START(PAGE_OFFSET - VMALLOC_SIZE)
+/* KASLR should leave at least 128MB for BPF after the kernel */
#define BPF_JIT_REGION_SIZE(SZ_128M)
-#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
-#define BPF_JIT_REGION_END (VMALLOC_END)
+#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
+
+/* Modules always live before the kernel */
+#ifdef CONFIG_64BIT
+#define VMALLOC_MODULE_START (PFN_ALIGN((unsigned long)&_end) - SZ_2G)
+#define VMALLOC_MODULE_END (PFN_ALIGN((unsigned long)&_start))
+#endif
This does not look right or I am
Re: [RFC PATCH 01/12] riscv: Move kernel mapping outside of linear mapping
On Wed, Jan 6, 2021 at 12:06 PM Alex Ghiti wrote:
>
> Hi Anup,
>
> Le 1/5/21 à 6:40 AM, Anup Patel a écrit :
> > On Tue, Jan 5, 2021 at 1:29 AM Alexandre Ghiti wrote:
> >>
> >> This is a preparatory patch for relocatable kernel and sv48 support.
> >>
> >> The kernel used to be linked at PAGE_OFFSET address therefore we could use
> >> the linear mapping for the kernel mapping. But the relocated kernel base
> >> address will be different from PAGE_OFFSET and since in the linear mapping,
> >> two different virtual addresses cannot point to the same physical address,
> >> the kernel mapping needs to lie outside the linear mapping so that we don't
> >> have to copy it at the same physical offset.
> >>
> >> The kernel mapping is moved to the last 2GB of the address space and then
> >> BPF and modules are also pushed to the same range since they have to lie
> >> close to the kernel inside a 2GB window.
> >>
> >> Note then that KASLR implementation will simply have to move the kernel in
> >> this 2GB range and modify BPF/modules regions accordingly.
> >>
> >> In addition, by moving the kernel to the end of the address space, both
> >> sv39 and sv48 kernels will be exactly the same without needing to be
> >> relocated at runtime.
> >
> > Awesome ! This is a good approach with no performance impact.
> >
> >>
> >> Suggested-by: Arnd Bergmann
> >> Signed-off-by: Alexandre Ghiti
> >> ---
> >> arch/riscv/boot/loader.lds.S | 3 +-
> >> arch/riscv/include/asm/page.h| 10 -
> >> arch/riscv/include/asm/pgtable.h | 39 +--
> >> arch/riscv/kernel/head.S | 3 +-
> >> arch/riscv/kernel/module.c | 4 +-
> >> arch/riscv/kernel/vmlinux.lds.S | 3 +-
> >> arch/riscv/mm/init.c | 65
> >> arch/riscv/mm/physaddr.c | 2 +-
> >> 8 files changed, 94 insertions(+), 35 deletions(-)
> >>
> >> diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
> >> index 47a5003c2e28..62d94696a19c 100644
> >> --- a/arch/riscv/boot/loader.lds.S
> >> +++ b/arch/riscv/boot/loader.lds.S
> >> @@ -1,13 +1,14 @@
> >> /* SPDX-License-Identifier: GPL-2.0 */
> >>
> >> #include
> >> +#include
> >>
> >> OUTPUT_ARCH(riscv)
> >> ENTRY(_start)
> >>
> >> SECTIONS
> >> {
> >> - . = PAGE_OFFSET;
> >> + . = KERNEL_LINK_ADDR;
> >>
> >> .payload : {
> >> *(.payload)
> >> diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
> >> index 2d50f76efe48..98188e315e8d 100644
> >> --- a/arch/riscv/include/asm/page.h
> >> +++ b/arch/riscv/include/asm/page.h
> >> @@ -90,18 +90,26 @@ typedef struct page *pgtable_t;
> >>
> >> #ifdef CONFIG_MMU
> >> extern unsigned long va_pa_offset;
> >> +extern unsigned long va_kernel_pa_offset;
> >> extern unsigned long pfn_base;
> >> #define ARCH_PFN_OFFSET(pfn_base)
> >> #else
> >> #define va_pa_offset 0
> >> +#define va_kernel_pa_offset0
> >> #define ARCH_PFN_OFFSET(PAGE_OFFSET >> PAGE_SHIFT)
> >> #endif /* CONFIG_MMU */
> >>
> >> extern unsigned long max_low_pfn;
> >> extern unsigned long min_low_pfn;
> >> +extern unsigned long kernel_virt_addr;
> >>
> >> #define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) +
> >> va_pa_offset))
> >> -#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
> >> +#define linear_mapping_va_to_pa(x) ((unsigned long)(x) - va_pa_offset)
> >> +#define kernel_mapping_va_to_pa(x) \
> >> + ((unsigned long)(x) - va_kernel_pa_offset)
> >> +#define __va_to_pa_nodebug(x) \
> >> + (((x) < KERNEL_LINK_ADDR) ? \
> >> + linear_mapping_va_to_pa(x) : kernel_mapping_va_to_pa(x))
> >>
> >> #ifdef CONFIG_DEBUG_VIRTUAL
> >> extern phys_addr_t __virt_to_phys(unsigned long x);
> >> diff --git a/arch/riscv/include/asm/pgtable.h
> >> b/arch/riscv/include/asm/pgtable.h
> >> index 183f1f4b2ae6..102b728ca146 100644
> >> --- a/arch/riscv/include/asm/pgtable.h
> >> +++ b/arch/riscv/include/asm/pgtable.h
> >> @@ -11,23 +11,32 @@
> >>
> >> #include
> >>
> >> -#ifndef __ASSEMBLY__
> >> -
> >> -/* Page Upper Directory not used in RISC-V */
> >> -#include
> >> -#include
> >> -#include
> >> -#include
> >> +#ifndef CONFIG_MMU
> >> +#define KERNEL_VIRT_ADDR PAGE_OFFSET
> >> +#define KERNEL_LINK_ADDR PAGE_OFFSET
> >> +#else
> >>
> >> -#ifdef CONFIG_MMU
> >> +#define ADDRESS_SPACE_END (UL(-1))
> >> +/*
> >> + * Leave 2GB for kernel, modules and BPF at the end of the address space
> >> + */
> >> +#define KERNEL_VIRT_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
> >> +#define KERNEL_LINK_ADDR KERNEL_VIRT_ADDR
> >>
> >> #define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
> >> #define VMALLOC_END (PAGE_OFFSET - 1)
> >> #define VMALLOC_START(PAGE_OFFSET - VMALLOC_SIZE)
> >>
> >> +/* KASLR should leave at least 128MB for BPF after the kernel */
> >> #define
Re: [RFC PATCH 01/12] riscv: Move kernel mapping outside of linear mapping
Hi Anup,
Le 1/5/21 à 6:40 AM, Anup Patel a écrit :
On Tue, Jan 5, 2021 at 1:29 AM Alexandre Ghiti wrote:
This is a preparatory patch for relocatable kernel and sv48 support.
The kernel used to be linked at PAGE_OFFSET address therefore we could use
the linear mapping for the kernel mapping. But the relocated kernel base
address will be different from PAGE_OFFSET and since in the linear mapping,
two different virtual addresses cannot point to the same physical address,
the kernel mapping needs to lie outside the linear mapping so that we don't
have to copy it at the same physical offset.
The kernel mapping is moved to the last 2GB of the address space and then
BPF and modules are also pushed to the same range since they have to lie
close to the kernel inside a 2GB window.
Note then that KASLR implementation will simply have to move the kernel in
this 2GB range and modify BPF/modules regions accordingly.
In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.
Awesome ! This is a good approach with no performance impact.
Suggested-by: Arnd Bergmann
Signed-off-by: Alexandre Ghiti
---
arch/riscv/boot/loader.lds.S | 3 +-
arch/riscv/include/asm/page.h| 10 -
arch/riscv/include/asm/pgtable.h | 39 +--
arch/riscv/kernel/head.S | 3 +-
arch/riscv/kernel/module.c | 4 +-
arch/riscv/kernel/vmlinux.lds.S | 3 +-
arch/riscv/mm/init.c | 65
arch/riscv/mm/physaddr.c | 2 +-
8 files changed, 94 insertions(+), 35 deletions(-)
diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
index 47a5003c2e28..62d94696a19c 100644
--- a/arch/riscv/boot/loader.lds.S
+++ b/arch/riscv/boot/loader.lds.S
@@ -1,13 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include
+#include
OUTPUT_ARCH(riscv)
ENTRY(_start)
SECTIONS
{
- . = PAGE_OFFSET;
+ . = KERNEL_LINK_ADDR;
.payload : {
*(.payload)
diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index 2d50f76efe48..98188e315e8d 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -90,18 +90,26 @@ typedef struct page *pgtable_t;
#ifdef CONFIG_MMU
extern unsigned long va_pa_offset;
+extern unsigned long va_kernel_pa_offset;
extern unsigned long pfn_base;
#define ARCH_PFN_OFFSET(pfn_base)
#else
#define va_pa_offset 0
+#define va_kernel_pa_offset0
#define ARCH_PFN_OFFSET(PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
extern unsigned long max_low_pfn;
extern unsigned long min_low_pfn;
+extern unsigned long kernel_virt_addr;
#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset))
-#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
+#define linear_mapping_va_to_pa(x) ((unsigned long)(x) - va_pa_offset)
+#define kernel_mapping_va_to_pa(x) \
+ ((unsigned long)(x) - va_kernel_pa_offset)
+#define __va_to_pa_nodebug(x) \
+ (((x) < KERNEL_LINK_ADDR) ? \
+ linear_mapping_va_to_pa(x) : kernel_mapping_va_to_pa(x))
#ifdef CONFIG_DEBUG_VIRTUAL
extern phys_addr_t __virt_to_phys(unsigned long x);
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 183f1f4b2ae6..102b728ca146 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -11,23 +11,32 @@
#include
-#ifndef __ASSEMBLY__
-
-/* Page Upper Directory not used in RISC-V */
-#include
-#include
-#include
-#include
+#ifndef CONFIG_MMU
+#define KERNEL_VIRT_ADDR PAGE_OFFSET
+#define KERNEL_LINK_ADDR PAGE_OFFSET
+#else
-#ifdef CONFIG_MMU
+#define ADDRESS_SPACE_END (UL(-1))
+/*
+ * Leave 2GB for kernel, modules and BPF at the end of the address space
+ */
+#define KERNEL_VIRT_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
+#define KERNEL_LINK_ADDR KERNEL_VIRT_ADDR
#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START(PAGE_OFFSET - VMALLOC_SIZE)
+/* KASLR should leave at least 128MB for BPF after the kernel */
#define BPF_JIT_REGION_SIZE(SZ_128M)
-#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
-#define BPF_JIT_REGION_END (VMALLOC_END)
+#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
+
+/* Modules always live before the kernel */
+#ifdef CONFIG_64BIT
+#define VMALLOC_MODULE_START (PFN_ALIGN((unsigned long)&_end) - SZ_2G)
+#define VMALLOC_MODULE_END (PFN_ALIGN((unsigned long)&_start))
+#endif
This does not look right or I am missing something.
I think the VMALLOC_MODULE_START should be:
#define VMALLOC_MODULE_START (PFN_ALIGN((unsigned long)&_start) -
Re: [RFC PATCH 01/12] riscv: Move kernel mapping outside of linear mapping
On Tue, Jan 5, 2021 at 1:29 AM Alexandre Ghiti wrote:
>
> This is a preparatory patch for relocatable kernel and sv48 support.
>
> The kernel used to be linked at PAGE_OFFSET address therefore we could use
> the linear mapping for the kernel mapping. But the relocated kernel base
> address will be different from PAGE_OFFSET and since in the linear mapping,
> two different virtual addresses cannot point to the same physical address,
> the kernel mapping needs to lie outside the linear mapping so that we don't
> have to copy it at the same physical offset.
>
> The kernel mapping is moved to the last 2GB of the address space and then
> BPF and modules are also pushed to the same range since they have to lie
> close to the kernel inside a 2GB window.
>
> Note then that KASLR implementation will simply have to move the kernel in
> this 2GB range and modify BPF/modules regions accordingly.
>
> In addition, by moving the kernel to the end of the address space, both
> sv39 and sv48 kernels will be exactly the same without needing to be
> relocated at runtime.
Awesome ! This is a good approach with no performance impact.
>
> Suggested-by: Arnd Bergmann
> Signed-off-by: Alexandre Ghiti
> ---
> arch/riscv/boot/loader.lds.S | 3 +-
> arch/riscv/include/asm/page.h| 10 -
> arch/riscv/include/asm/pgtable.h | 39 +--
> arch/riscv/kernel/head.S | 3 +-
> arch/riscv/kernel/module.c | 4 +-
> arch/riscv/kernel/vmlinux.lds.S | 3 +-
> arch/riscv/mm/init.c | 65
> arch/riscv/mm/physaddr.c | 2 +-
> 8 files changed, 94 insertions(+), 35 deletions(-)
>
> diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
> index 47a5003c2e28..62d94696a19c 100644
> --- a/arch/riscv/boot/loader.lds.S
> +++ b/arch/riscv/boot/loader.lds.S
> @@ -1,13 +1,14 @@
> /* SPDX-License-Identifier: GPL-2.0 */
>
> #include
> +#include
>
> OUTPUT_ARCH(riscv)
> ENTRY(_start)
>
> SECTIONS
> {
> - . = PAGE_OFFSET;
> + . = KERNEL_LINK_ADDR;
>
> .payload : {
> *(.payload)
> diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
> index 2d50f76efe48..98188e315e8d 100644
> --- a/arch/riscv/include/asm/page.h
> +++ b/arch/riscv/include/asm/page.h
> @@ -90,18 +90,26 @@ typedef struct page *pgtable_t;
>
> #ifdef CONFIG_MMU
> extern unsigned long va_pa_offset;
> +extern unsigned long va_kernel_pa_offset;
> extern unsigned long pfn_base;
> #define ARCH_PFN_OFFSET(pfn_base)
> #else
> #define va_pa_offset 0
> +#define va_kernel_pa_offset0
> #define ARCH_PFN_OFFSET(PAGE_OFFSET >> PAGE_SHIFT)
> #endif /* CONFIG_MMU */
>
> extern unsigned long max_low_pfn;
> extern unsigned long min_low_pfn;
> +extern unsigned long kernel_virt_addr;
>
> #define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset))
> -#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
> +#define linear_mapping_va_to_pa(x) ((unsigned long)(x) - va_pa_offset)
> +#define kernel_mapping_va_to_pa(x) \
> + ((unsigned long)(x) - va_kernel_pa_offset)
> +#define __va_to_pa_nodebug(x) \
> + (((x) < KERNEL_LINK_ADDR) ? \
> + linear_mapping_va_to_pa(x) : kernel_mapping_va_to_pa(x))
>
> #ifdef CONFIG_DEBUG_VIRTUAL
> extern phys_addr_t __virt_to_phys(unsigned long x);
> diff --git a/arch/riscv/include/asm/pgtable.h
> b/arch/riscv/include/asm/pgtable.h
> index 183f1f4b2ae6..102b728ca146 100644
> --- a/arch/riscv/include/asm/pgtable.h
> +++ b/arch/riscv/include/asm/pgtable.h
> @@ -11,23 +11,32 @@
>
> #include
>
> -#ifndef __ASSEMBLY__
> -
> -/* Page Upper Directory not used in RISC-V */
> -#include
> -#include
> -#include
> -#include
> +#ifndef CONFIG_MMU
> +#define KERNEL_VIRT_ADDR PAGE_OFFSET
> +#define KERNEL_LINK_ADDR PAGE_OFFSET
> +#else
>
> -#ifdef CONFIG_MMU
> +#define ADDRESS_SPACE_END (UL(-1))
> +/*
> + * Leave 2GB for kernel, modules and BPF at the end of the address space
> + */
> +#define KERNEL_VIRT_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
> +#define KERNEL_LINK_ADDR KERNEL_VIRT_ADDR
>
> #define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
> #define VMALLOC_END (PAGE_OFFSET - 1)
> #define VMALLOC_START(PAGE_OFFSET - VMALLOC_SIZE)
>
> +/* KASLR should leave at least 128MB for BPF after the kernel */
> #define BPF_JIT_REGION_SIZE(SZ_128M)
> -#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
> -#define BPF_JIT_REGION_END (VMALLOC_END)
> +#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
> +#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
> +
> +/* Modules always live before the kernel */
> +#ifdef CONFIG_64BIT
> +#define VMALLOC_MODULE_START (PFN_ALIGN((unsigned long)&_end) - SZ_2G)
> +#define VMALLOC_MODULE_END (PFN_ALIGN((unsigned long)&_start))
> +#endif
This does not look
[RFC PATCH 01/12] riscv: Move kernel mapping outside of linear mapping
This is a preparatory patch for relocatable kernel and sv48 support.
The kernel used to be linked at PAGE_OFFSET address therefore we could use
the linear mapping for the kernel mapping. But the relocated kernel base
address will be different from PAGE_OFFSET and since in the linear mapping,
two different virtual addresses cannot point to the same physical address,
the kernel mapping needs to lie outside the linear mapping so that we don't
have to copy it at the same physical offset.
The kernel mapping is moved to the last 2GB of the address space and then
BPF and modules are also pushed to the same range since they have to lie
close to the kernel inside a 2GB window.
Note then that KASLR implementation will simply have to move the kernel in
this 2GB range and modify BPF/modules regions accordingly.
In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.
Suggested-by: Arnd Bergmann
Signed-off-by: Alexandre Ghiti
---
arch/riscv/boot/loader.lds.S | 3 +-
arch/riscv/include/asm/page.h| 10 -
arch/riscv/include/asm/pgtable.h | 39 +--
arch/riscv/kernel/head.S | 3 +-
arch/riscv/kernel/module.c | 4 +-
arch/riscv/kernel/vmlinux.lds.S | 3 +-
arch/riscv/mm/init.c | 65
arch/riscv/mm/physaddr.c | 2 +-
8 files changed, 94 insertions(+), 35 deletions(-)
diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
index 47a5003c2e28..62d94696a19c 100644
--- a/arch/riscv/boot/loader.lds.S
+++ b/arch/riscv/boot/loader.lds.S
@@ -1,13 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include
+#include
OUTPUT_ARCH(riscv)
ENTRY(_start)
SECTIONS
{
- . = PAGE_OFFSET;
+ . = KERNEL_LINK_ADDR;
.payload : {
*(.payload)
diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index 2d50f76efe48..98188e315e8d 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -90,18 +90,26 @@ typedef struct page *pgtable_t;
#ifdef CONFIG_MMU
extern unsigned long va_pa_offset;
+extern unsigned long va_kernel_pa_offset;
extern unsigned long pfn_base;
#define ARCH_PFN_OFFSET(pfn_base)
#else
#define va_pa_offset 0
+#define va_kernel_pa_offset0
#define ARCH_PFN_OFFSET(PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
extern unsigned long max_low_pfn;
extern unsigned long min_low_pfn;
+extern unsigned long kernel_virt_addr;
#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset))
-#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
+#define linear_mapping_va_to_pa(x) ((unsigned long)(x) - va_pa_offset)
+#define kernel_mapping_va_to_pa(x) \
+ ((unsigned long)(x) - va_kernel_pa_offset)
+#define __va_to_pa_nodebug(x) \
+ (((x) < KERNEL_LINK_ADDR) ? \
+ linear_mapping_va_to_pa(x) : kernel_mapping_va_to_pa(x))
#ifdef CONFIG_DEBUG_VIRTUAL
extern phys_addr_t __virt_to_phys(unsigned long x);
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index 183f1f4b2ae6..102b728ca146 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -11,23 +11,32 @@
#include
-#ifndef __ASSEMBLY__
-
-/* Page Upper Directory not used in RISC-V */
-#include
-#include
-#include
-#include
+#ifndef CONFIG_MMU
+#define KERNEL_VIRT_ADDR PAGE_OFFSET
+#define KERNEL_LINK_ADDR PAGE_OFFSET
+#else
-#ifdef CONFIG_MMU
+#define ADDRESS_SPACE_END (UL(-1))
+/*
+ * Leave 2GB for kernel, modules and BPF at the end of the address space
+ */
+#define KERNEL_VIRT_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
+#define KERNEL_LINK_ADDR KERNEL_VIRT_ADDR
#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START(PAGE_OFFSET - VMALLOC_SIZE)
+/* KASLR should leave at least 128MB for BPF after the kernel */
#define BPF_JIT_REGION_SIZE(SZ_128M)
-#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
-#define BPF_JIT_REGION_END (VMALLOC_END)
+#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
+
+/* Modules always live before the kernel */
+#ifdef CONFIG_64BIT
+#define VMALLOC_MODULE_START (PFN_ALIGN((unsigned long)&_end) - SZ_2G)
+#define VMALLOC_MODULE_END (PFN_ALIGN((unsigned long)&_start))
+#endif
/*
* Roughly size the vmemmap space to be large enough to fit enough
@@ -57,9 +66,16 @@
#define FIXADDR_SIZE PGDIR_SIZE
#endif
#define FIXADDR_START(FIXADDR_TOP - FIXADDR_SIZE)
-
#endif
+#ifndef __ASSEMBLY__
+
+/* Page Upper Directory not used in RISC-V */
+#include
+#include
+#include
+#include
+
#ifdef CONFIG_64BIT
#include
#else
@@
