Re: [PATCH v5 1/3] riscv: Move kernel mapping outside of linear mapping
Hi Palmer,
Le 4/15/21 à 2:00 PM, Alex Ghiti a écrit :
Le 4/15/21 à 12:54 AM, Alex Ghiti a écrit :
Le 4/15/21 à 12:20 AM, Palmer Dabbelt a écrit :
On Sun, 11 Apr 2021 09:41:44 PDT (-0700), a...@ghiti.fr wrote:
This is a preparatory patch for relocatable kernel and sv48 support.
The kernel used to be linked at PAGE_OFFSET address therefore we
could use
the linear mapping for the kernel mapping. But the relocated kernel
base
address will be different from PAGE_OFFSET and since in the linear
mapping,
two different virtual addresses cannot point to the same physical
address,
the kernel mapping needs to lie outside the linear mapping so that
we don't
have to copy it at the same physical offset.
The kernel mapping is moved to the last 2GB of the address space, BPF
is now always after the kernel and modules use the 2GB memory range
right
before the kernel, so BPF and modules regions do not overlap. KASLR
implementation will simply have to move the kernel in the last 2GB
range
and just take care of leaving enough space for BPF.
In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.
Suggested-by: Arnd Bergmann
Signed-off-by: Alexandre Ghiti
---
arch/riscv/boot/loader.lds.S | 3 +-
arch/riscv/include/asm/page.h | 17 +-
arch/riscv/include/asm/pgtable.h | 37
arch/riscv/include/asm/set_memory.h | 1 +
arch/riscv/kernel/head.S | 3 +-
arch/riscv/kernel/module.c | 6 +-
arch/riscv/kernel/setup.c | 5 ++
arch/riscv/kernel/vmlinux.lds.S | 3 +-
arch/riscv/mm/fault.c | 13 +
arch/riscv/mm/init.c | 87 ++---
arch/riscv/mm/kasan_init.c | 9 +++
arch/riscv/mm/physaddr.c | 2 +-
12 files changed, 146 insertions(+), 40 deletions(-)
diff --git a/arch/riscv/boot/loader.lds.S
b/arch/riscv/boot/loader.lds.S
index 47a5003c2e28..62d94696a19c 100644
--- a/arch/riscv/boot/loader.lds.S
+++ b/arch/riscv/boot/loader.lds.S
@@ -1,13 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include
+#include
OUTPUT_ARCH(riscv)
ENTRY(_start)
SECTIONS
{
- . = PAGE_OFFSET;
+ . = KERNEL_LINK_ADDR;
.payload : {
*(.payload)
diff --git a/arch/riscv/include/asm/page.h
b/arch/riscv/include/asm/page.h
index adc9d26f3d75..22cfb2be60dc 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -90,15 +90,28 @@ typedef struct page *pgtable_t;
#ifdef CONFIG_MMU
extern unsigned long va_pa_offset;
+extern unsigned long va_kernel_pa_offset;
extern unsigned long pfn_base;
#define ARCH_PFN_OFFSET (pfn_base)
#else
#define va_pa_offset 0
+#define va_kernel_pa_offset 0
#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
-#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) +
va_pa_offset))
-#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
+extern unsigned long kernel_virt_addr;
+
+#define linear_mapping_pa_to_va(x) ((void *)((unsigned long)(x)
+ va_pa_offset))
+#define kernel_mapping_pa_to_va(x) ((void *)((unsigned long)(x)
+ va_kernel_pa_offset))
+#define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x)
+
+#define linear_mapping_va_to_pa(x) ((unsigned long)(x) -
va_pa_offset)
+#define kernel_mapping_va_to_pa(x) ((unsigned long)(x) -
va_kernel_pa_offset)
+#define __va_to_pa_nodebug(x) ({ \
+ unsigned long _x = x; \
+ (_x < kernel_virt_addr) ? \
+ linear_mapping_va_to_pa(_x) :
kernel_mapping_va_to_pa(_x); \
+ })
#ifdef CONFIG_DEBUG_VIRTUAL
extern phys_addr_t __virt_to_phys(unsigned long x);
diff --git a/arch/riscv/include/asm/pgtable.h
b/arch/riscv/include/asm/pgtable.h
index ebf817c1bdf4..80e63a93e903 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -11,23 +11,30 @@
#include
-#ifndef __ASSEMBLY__
-
-/* Page Upper Directory not used in RISC-V */
-#include
-#include
-#include
-#include
+#ifndef CONFIG_MMU
+#define KERNEL_LINK_ADDR PAGE_OFFSET
+#else
-#ifdef CONFIG_MMU
+#define ADDRESS_SPACE_END (UL(-1))
+/*
+ * Leave 2GB for kernel and BPF at the end of the address space
+ */
+#define KERNEL_LINK_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
#define VMALLOC_SIZE (KERN_VIRT_SIZE >>1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE)
+/* KASLR should leave at least 128MB for BPF after the kernel */
#define BPF_JIT_REGION_SIZE (SZ_128M)
-#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
-#define BPF_JIT_REGION_END (VMALLOC_END)
+#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START +
BPF_JIT_REGION_SIZE)
+
+/* Modules always live before the
Re: [PATCH v5 1/3] riscv: Move kernel mapping outside of linear mapping
Le 4/15/21 à 12:54 AM, Alex Ghiti a écrit :
Le 4/15/21 à 12:20 AM, Palmer Dabbelt a écrit :
On Sun, 11 Apr 2021 09:41:44 PDT (-0700), a...@ghiti.fr wrote:
This is a preparatory patch for relocatable kernel and sv48 support.
The kernel used to be linked at PAGE_OFFSET address therefore we
could use
the linear mapping for the kernel mapping. But the relocated kernel base
address will be different from PAGE_OFFSET and since in the linear
mapping,
two different virtual addresses cannot point to the same physical
address,
the kernel mapping needs to lie outside the linear mapping so that we
don't
have to copy it at the same physical offset.
The kernel mapping is moved to the last 2GB of the address space, BPF
is now always after the kernel and modules use the 2GB memory range
right
before the kernel, so BPF and modules regions do not overlap. KASLR
implementation will simply have to move the kernel in the last 2GB range
and just take care of leaving enough space for BPF.
In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.
Suggested-by: Arnd Bergmann
Signed-off-by: Alexandre Ghiti
---
arch/riscv/boot/loader.lds.S | 3 +-
arch/riscv/include/asm/page.h | 17 +-
arch/riscv/include/asm/pgtable.h | 37
arch/riscv/include/asm/set_memory.h | 1 +
arch/riscv/kernel/head.S | 3 +-
arch/riscv/kernel/module.c | 6 +-
arch/riscv/kernel/setup.c | 5 ++
arch/riscv/kernel/vmlinux.lds.S | 3 +-
arch/riscv/mm/fault.c | 13 +
arch/riscv/mm/init.c | 87 ++---
arch/riscv/mm/kasan_init.c | 9 +++
arch/riscv/mm/physaddr.c | 2 +-
12 files changed, 146 insertions(+), 40 deletions(-)
diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
index 47a5003c2e28..62d94696a19c 100644
--- a/arch/riscv/boot/loader.lds.S
+++ b/arch/riscv/boot/loader.lds.S
@@ -1,13 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include
+#include
OUTPUT_ARCH(riscv)
ENTRY(_start)
SECTIONS
{
- . = PAGE_OFFSET;
+ . = KERNEL_LINK_ADDR;
.payload : {
*(.payload)
diff --git a/arch/riscv/include/asm/page.h
b/arch/riscv/include/asm/page.h
index adc9d26f3d75..22cfb2be60dc 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -90,15 +90,28 @@ typedef struct page *pgtable_t;
#ifdef CONFIG_MMU
extern unsigned long va_pa_offset;
+extern unsigned long va_kernel_pa_offset;
extern unsigned long pfn_base;
#define ARCH_PFN_OFFSET (pfn_base)
#else
#define va_pa_offset 0
+#define va_kernel_pa_offset 0
#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
-#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) +
va_pa_offset))
-#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
+extern unsigned long kernel_virt_addr;
+
+#define linear_mapping_pa_to_va(x) ((void *)((unsigned long)(x) +
va_pa_offset))
+#define kernel_mapping_pa_to_va(x) ((void *)((unsigned long)(x) +
va_kernel_pa_offset))
+#define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x)
+
+#define linear_mapping_va_to_pa(x) ((unsigned long)(x) -
va_pa_offset)
+#define kernel_mapping_va_to_pa(x) ((unsigned long)(x) -
va_kernel_pa_offset)
+#define __va_to_pa_nodebug(x) ({ \
+ unsigned long _x = x; \
+ (_x < kernel_virt_addr) ? \
+ linear_mapping_va_to_pa(_x) : kernel_mapping_va_to_pa(_x); \
+ })
#ifdef CONFIG_DEBUG_VIRTUAL
extern phys_addr_t __virt_to_phys(unsigned long x);
diff --git a/arch/riscv/include/asm/pgtable.h
b/arch/riscv/include/asm/pgtable.h
index ebf817c1bdf4..80e63a93e903 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -11,23 +11,30 @@
#include
-#ifndef __ASSEMBLY__
-
-/* Page Upper Directory not used in RISC-V */
-#include
-#include
-#include
-#include
+#ifndef CONFIG_MMU
+#define KERNEL_LINK_ADDR PAGE_OFFSET
+#else
-#ifdef CONFIG_MMU
+#define ADDRESS_SPACE_END (UL(-1))
+/*
+ * Leave 2GB for kernel and BPF at the end of the address space
+ */
+#define KERNEL_LINK_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE)
+/* KASLR should leave at least 128MB for BPF after the kernel */
#define BPF_JIT_REGION_SIZE (SZ_128M)
-#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
-#define BPF_JIT_REGION_END (VMALLOC_END)
+#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START +
BPF_JIT_REGION_SIZE)
+
+/* Modules always live before the kernel */
+#ifdef CONFIG_64BIT
+#define MODULES_VADDR
Re: [PATCH v5 1/3] riscv: Move kernel mapping outside of linear mapping
Le 4/15/21 à 12:20 AM, Palmer Dabbelt a écrit :
On Sun, 11 Apr 2021 09:41:44 PDT (-0700), a...@ghiti.fr wrote:
This is a preparatory patch for relocatable kernel and sv48 support.
The kernel used to be linked at PAGE_OFFSET address therefore we could
use
the linear mapping for the kernel mapping. But the relocated kernel base
address will be different from PAGE_OFFSET and since in the linear
mapping,
two different virtual addresses cannot point to the same physical
address,
the kernel mapping needs to lie outside the linear mapping so that we
don't
have to copy it at the same physical offset.
The kernel mapping is moved to the last 2GB of the address space, BPF
is now always after the kernel and modules use the 2GB memory range right
before the kernel, so BPF and modules regions do not overlap. KASLR
implementation will simply have to move the kernel in the last 2GB range
and just take care of leaving enough space for BPF.
In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.
Suggested-by: Arnd Bergmann
Signed-off-by: Alexandre Ghiti
---
arch/riscv/boot/loader.lds.S | 3 +-
arch/riscv/include/asm/page.h | 17 +-
arch/riscv/include/asm/pgtable.h | 37
arch/riscv/include/asm/set_memory.h | 1 +
arch/riscv/kernel/head.S | 3 +-
arch/riscv/kernel/module.c | 6 +-
arch/riscv/kernel/setup.c | 5 ++
arch/riscv/kernel/vmlinux.lds.S | 3 +-
arch/riscv/mm/fault.c | 13 +
arch/riscv/mm/init.c | 87 ++---
arch/riscv/mm/kasan_init.c | 9 +++
arch/riscv/mm/physaddr.c | 2 +-
12 files changed, 146 insertions(+), 40 deletions(-)
diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
index 47a5003c2e28..62d94696a19c 100644
--- a/arch/riscv/boot/loader.lds.S
+++ b/arch/riscv/boot/loader.lds.S
@@ -1,13 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include
+#include
OUTPUT_ARCH(riscv)
ENTRY(_start)
SECTIONS
{
- . = PAGE_OFFSET;
+ . = KERNEL_LINK_ADDR;
.payload : {
*(.payload)
diff --git a/arch/riscv/include/asm/page.h
b/arch/riscv/include/asm/page.h
index adc9d26f3d75..22cfb2be60dc 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -90,15 +90,28 @@ typedef struct page *pgtable_t;
#ifdef CONFIG_MMU
extern unsigned long va_pa_offset;
+extern unsigned long va_kernel_pa_offset;
extern unsigned long pfn_base;
#define ARCH_PFN_OFFSET (pfn_base)
#else
#define va_pa_offset 0
+#define va_kernel_pa_offset 0
#define ARCH_PFN_OFFSET (PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
-#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) +
va_pa_offset))
-#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
+extern unsigned long kernel_virt_addr;
+
+#define linear_mapping_pa_to_va(x) ((void *)((unsigned long)(x) +
va_pa_offset))
+#define kernel_mapping_pa_to_va(x) ((void *)((unsigned long)(x) +
va_kernel_pa_offset))
+#define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x)
+
+#define linear_mapping_va_to_pa(x) ((unsigned long)(x) -
va_pa_offset)
+#define kernel_mapping_va_to_pa(x) ((unsigned long)(x) -
va_kernel_pa_offset)
+#define __va_to_pa_nodebug(x) ({ \
+ unsigned long _x = x; \
+ (_x < kernel_virt_addr) ? \
+ linear_mapping_va_to_pa(_x) : kernel_mapping_va_to_pa(_x); \
+ })
#ifdef CONFIG_DEBUG_VIRTUAL
extern phys_addr_t __virt_to_phys(unsigned long x);
diff --git a/arch/riscv/include/asm/pgtable.h
b/arch/riscv/include/asm/pgtable.h
index ebf817c1bdf4..80e63a93e903 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -11,23 +11,30 @@
#include
-#ifndef __ASSEMBLY__
-
-/* Page Upper Directory not used in RISC-V */
-#include
-#include
-#include
-#include
+#ifndef CONFIG_MMU
+#define KERNEL_LINK_ADDR PAGE_OFFSET
+#else
-#ifdef CONFIG_MMU
+#define ADDRESS_SPACE_END (UL(-1))
+/*
+ * Leave 2GB for kernel and BPF at the end of the address space
+ */
+#define KERNEL_LINK_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START (PAGE_OFFSET - VMALLOC_SIZE)
+/* KASLR should leave at least 128MB for BPF after the kernel */
#define BPF_JIT_REGION_SIZE (SZ_128M)
-#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
-#define BPF_JIT_REGION_END (VMALLOC_END)
+#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START +
BPF_JIT_REGION_SIZE)
+
+/* Modules always live before the kernel */
+#ifdef CONFIG_64BIT
+#define MODULES_VADDR (PFN_ALIGN((unsigned long)&_end) - SZ_2G)
Re: [PATCH v5 1/3] riscv: Move kernel mapping outside of linear mapping
On Sun, 11 Apr 2021 09:41:44 PDT (-0700), a...@ghiti.fr wrote:
This is a preparatory patch for relocatable kernel and sv48 support.
The kernel used to be linked at PAGE_OFFSET address therefore we could use
the linear mapping for the kernel mapping. But the relocated kernel base
address will be different from PAGE_OFFSET and since in the linear mapping,
two different virtual addresses cannot point to the same physical address,
the kernel mapping needs to lie outside the linear mapping so that we don't
have to copy it at the same physical offset.
The kernel mapping is moved to the last 2GB of the address space, BPF
is now always after the kernel and modules use the 2GB memory range right
before the kernel, so BPF and modules regions do not overlap. KASLR
implementation will simply have to move the kernel in the last 2GB range
and just take care of leaving enough space for BPF.
In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.
Suggested-by: Arnd Bergmann
Signed-off-by: Alexandre Ghiti
---
arch/riscv/boot/loader.lds.S| 3 +-
arch/riscv/include/asm/page.h | 17 +-
arch/riscv/include/asm/pgtable.h| 37
arch/riscv/include/asm/set_memory.h | 1 +
arch/riscv/kernel/head.S| 3 +-
arch/riscv/kernel/module.c | 6 +-
arch/riscv/kernel/setup.c | 5 ++
arch/riscv/kernel/vmlinux.lds.S | 3 +-
arch/riscv/mm/fault.c | 13 +
arch/riscv/mm/init.c| 87 ++---
arch/riscv/mm/kasan_init.c | 9 +++
arch/riscv/mm/physaddr.c| 2 +-
12 files changed, 146 insertions(+), 40 deletions(-)
diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
index 47a5003c2e28..62d94696a19c 100644
--- a/arch/riscv/boot/loader.lds.S
+++ b/arch/riscv/boot/loader.lds.S
@@ -1,13 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include
+#include
OUTPUT_ARCH(riscv)
ENTRY(_start)
SECTIONS
{
- . = PAGE_OFFSET;
+ . = KERNEL_LINK_ADDR;
.payload : {
*(.payload)
diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index adc9d26f3d75..22cfb2be60dc 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -90,15 +90,28 @@ typedef struct page *pgtable_t;
#ifdef CONFIG_MMU
extern unsigned long va_pa_offset;
+extern unsigned long va_kernel_pa_offset;
extern unsigned long pfn_base;
#define ARCH_PFN_OFFSET(pfn_base)
#else
#define va_pa_offset 0
+#define va_kernel_pa_offset0
#define ARCH_PFN_OFFSET(PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
-#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset))
-#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
+extern unsigned long kernel_virt_addr;
+
+#define linear_mapping_pa_to_va(x) ((void *)((unsigned long)(x) +
va_pa_offset))
+#define kernel_mapping_pa_to_va(x) ((void *)((unsigned long)(x) +
va_kernel_pa_offset))
+#define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x)
+
+#define linear_mapping_va_to_pa(x) ((unsigned long)(x) - va_pa_offset)
+#define kernel_mapping_va_to_pa(x) ((unsigned long)(x) -
va_kernel_pa_offset)
+#define __va_to_pa_nodebug(x) ({
\
+ unsigned long _x = x;
\
+ (_x < kernel_virt_addr) ?\
+ linear_mapping_va_to_pa(_x) : kernel_mapping_va_to_pa(_x);
\
+ })
#ifdef CONFIG_DEBUG_VIRTUAL
extern phys_addr_t __virt_to_phys(unsigned long x);
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index ebf817c1bdf4..80e63a93e903 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -11,23 +11,30 @@
#include
-#ifndef __ASSEMBLY__
-
-/* Page Upper Directory not used in RISC-V */
-#include
-#include
-#include
-#include
+#ifndef CONFIG_MMU
+#define KERNEL_LINK_ADDR PAGE_OFFSET
+#else
-#ifdef CONFIG_MMU
+#define ADDRESS_SPACE_END (UL(-1))
+/*
+ * Leave 2GB for kernel and BPF at the end of the address space
+ */
+#define KERNEL_LINK_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START(PAGE_OFFSET - VMALLOC_SIZE)
+/* KASLR should leave at least 128MB for BPF after the kernel */
#define BPF_JIT_REGION_SIZE(SZ_128M)
-#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
-#define BPF_JIT_REGION_END (VMALLOC_END)
+#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
+
+/* Modules always live before the kernel */
+#ifdef CONFIG_64BIT
+#define
[PATCH v5 1/3] riscv: Move kernel mapping outside of linear mapping
This is a preparatory patch for relocatable kernel and sv48 support.
The kernel used to be linked at PAGE_OFFSET address therefore we could use
the linear mapping for the kernel mapping. But the relocated kernel base
address will be different from PAGE_OFFSET and since in the linear mapping,
two different virtual addresses cannot point to the same physical address,
the kernel mapping needs to lie outside the linear mapping so that we don't
have to copy it at the same physical offset.
The kernel mapping is moved to the last 2GB of the address space, BPF
is now always after the kernel and modules use the 2GB memory range right
before the kernel, so BPF and modules regions do not overlap. KASLR
implementation will simply have to move the kernel in the last 2GB range
and just take care of leaving enough space for BPF.
In addition, by moving the kernel to the end of the address space, both
sv39 and sv48 kernels will be exactly the same without needing to be
relocated at runtime.
Suggested-by: Arnd Bergmann
Signed-off-by: Alexandre Ghiti
---
arch/riscv/boot/loader.lds.S| 3 +-
arch/riscv/include/asm/page.h | 17 +-
arch/riscv/include/asm/pgtable.h| 37
arch/riscv/include/asm/set_memory.h | 1 +
arch/riscv/kernel/head.S| 3 +-
arch/riscv/kernel/module.c | 6 +-
arch/riscv/kernel/setup.c | 5 ++
arch/riscv/kernel/vmlinux.lds.S | 3 +-
arch/riscv/mm/fault.c | 13 +
arch/riscv/mm/init.c| 87 ++---
arch/riscv/mm/kasan_init.c | 9 +++
arch/riscv/mm/physaddr.c| 2 +-
12 files changed, 146 insertions(+), 40 deletions(-)
diff --git a/arch/riscv/boot/loader.lds.S b/arch/riscv/boot/loader.lds.S
index 47a5003c2e28..62d94696a19c 100644
--- a/arch/riscv/boot/loader.lds.S
+++ b/arch/riscv/boot/loader.lds.S
@@ -1,13 +1,14 @@
/* SPDX-License-Identifier: GPL-2.0 */
#include
+#include
OUTPUT_ARCH(riscv)
ENTRY(_start)
SECTIONS
{
- . = PAGE_OFFSET;
+ . = KERNEL_LINK_ADDR;
.payload : {
*(.payload)
diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h
index adc9d26f3d75..22cfb2be60dc 100644
--- a/arch/riscv/include/asm/page.h
+++ b/arch/riscv/include/asm/page.h
@@ -90,15 +90,28 @@ typedef struct page *pgtable_t;
#ifdef CONFIG_MMU
extern unsigned long va_pa_offset;
+extern unsigned long va_kernel_pa_offset;
extern unsigned long pfn_base;
#define ARCH_PFN_OFFSET(pfn_base)
#else
#define va_pa_offset 0
+#define va_kernel_pa_offset0
#define ARCH_PFN_OFFSET(PAGE_OFFSET >> PAGE_SHIFT)
#endif /* CONFIG_MMU */
-#define __pa_to_va_nodebug(x) ((void *)((unsigned long) (x) + va_pa_offset))
-#define __va_to_pa_nodebug(x) ((unsigned long)(x) - va_pa_offset)
+extern unsigned long kernel_virt_addr;
+
+#define linear_mapping_pa_to_va(x) ((void *)((unsigned long)(x) +
va_pa_offset))
+#define kernel_mapping_pa_to_va(x) ((void *)((unsigned long)(x) +
va_kernel_pa_offset))
+#define __pa_to_va_nodebug(x) linear_mapping_pa_to_va(x)
+
+#define linear_mapping_va_to_pa(x) ((unsigned long)(x) - va_pa_offset)
+#define kernel_mapping_va_to_pa(x) ((unsigned long)(x) -
va_kernel_pa_offset)
+#define __va_to_pa_nodebug(x) ({
\
+ unsigned long _x = x;
\
+ (_x < kernel_virt_addr) ?
\
+ linear_mapping_va_to_pa(_x) : kernel_mapping_va_to_pa(_x);
\
+ })
#ifdef CONFIG_DEBUG_VIRTUAL
extern phys_addr_t __virt_to_phys(unsigned long x);
diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h
index ebf817c1bdf4..80e63a93e903 100644
--- a/arch/riscv/include/asm/pgtable.h
+++ b/arch/riscv/include/asm/pgtable.h
@@ -11,23 +11,30 @@
#include
-#ifndef __ASSEMBLY__
-
-/* Page Upper Directory not used in RISC-V */
-#include
-#include
-#include
-#include
+#ifndef CONFIG_MMU
+#define KERNEL_LINK_ADDR PAGE_OFFSET
+#else
-#ifdef CONFIG_MMU
+#define ADDRESS_SPACE_END (UL(-1))
+/*
+ * Leave 2GB for kernel and BPF at the end of the address space
+ */
+#define KERNEL_LINK_ADDR (ADDRESS_SPACE_END - SZ_2G + 1)
#define VMALLOC_SIZE (KERN_VIRT_SIZE >> 1)
#define VMALLOC_END (PAGE_OFFSET - 1)
#define VMALLOC_START(PAGE_OFFSET - VMALLOC_SIZE)
+/* KASLR should leave at least 128MB for BPF after the kernel */
#define BPF_JIT_REGION_SIZE(SZ_128M)
-#define BPF_JIT_REGION_START (PAGE_OFFSET - BPF_JIT_REGION_SIZE)
-#define BPF_JIT_REGION_END (VMALLOC_END)
+#define BPF_JIT_REGION_START PFN_ALIGN((unsigned long)&_end)
+#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE)
+
+/* Modules always live before the kernel */
+#ifdef CONFIG_64BIT
+#define MODULES_VADDR (PFN_ALIGN((unsigned long)&_end) -
