Re: [PATCH 1/2] x86/mm: add .data..decrypted section to hold shared variables
On 08/27/2018 05:11 PM, Tom Lendacky wrote:
On 08/27/2018 06:24 AM, Brijesh Singh wrote:
kvmclock defines few static variables which are shared with hypervisor
during the kvmclock initialization.
When SEV is active, memory is encrypted with a guest-specific key, and
if guest OS wants to share the memory region with hypervisor then it must
clear the C-bit before sharing it.
The '__decrypted' can be used to define a shared variables; the variables
will be put in the .data.decryption section. This section is mapped with
C=0 early in the boot, we also ensure that the initialized values are
updated to match with C=0 (i.e peform an in-place decryption). The
.data..decrypted section is PMD aligned and sized so that we avoid the
need for spliting the pages when map with C=0.
This should probably be broken into a few smaller patches. Maybe a
patch that adds the section and the attribute, a patch that re-arranges
the mapping setup and then the in-place decryption and clearing of the
encryption bit for the area.
OK, I will break the patch. Probably will create a separate patch which
just re-arranges the mapping setup without making any logical changes.
Signed-off-by: Brijesh Singh
Fixes: 368a540e0232 ("x86/kvmclock: Remove memblock dependency")
Cc: sta...@vger.kernel.org
Cc: Tom Lendacky
Cc: k...@vger.kernel.org
Cc: Thomas Gleixner
Cc: Borislav Petkov
Cc: "H. Peter Anvin"
Cc: linux-kernel@vger.kernel.org
Cc: Paolo Bonzini
Cc: Sean Christopherson
Cc: "Radim Krčmář"
---
arch/x86/include/asm/mem_encrypt.h | 4 +
arch/x86/kernel/head64.c | 12 ++
arch/x86/kernel/vmlinux.lds.S | 18 +++
arch/x86/mm/mem_encrypt_identity.c | 220 +++--
4 files changed, 197 insertions(+), 57 deletions(-)
diff --git a/arch/x86/include/asm/mem_encrypt.h
b/arch/x86/include/asm/mem_encrypt.h
index c064383..3f7d9d3 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -52,6 +52,8 @@ void __init mem_encrypt_init(void);
bool sme_active(void);
bool sev_active(void);
+#define __decrypted __attribute__((__section__(".data..decrypted")))
+
#else /* !CONFIG_AMD_MEM_ENCRYPT */
#define sme_me_mask 0ULL
@@ -77,6 +79,8 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long
size) { return 0;
static inline int __init
early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return
0; }
+#define __decrypted
+
#endif/* CONFIG_AMD_MEM_ENCRYPT */
/*
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 8047379..6a18297 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -43,6 +43,9 @@ extern pmd_t
early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
static unsigned int __initdata next_early_pgt;
pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
+/* To clear memory encryption mask from the decrypted section */
+extern char __start_data_decrypted[], __end_data_decrypted[];
+
Should find a header for these rather than defining them here.
OK, will move then in mem_encrypt.h. Will that work ?
#ifdef CONFIG_X86_5LEVEL
unsigned int __pgtable_l5_enabled __ro_after_init;
unsigned int pgdir_shift __ro_after_init = 39;
@@ -112,6 +115,7 @@ static bool __head check_la57_support(unsigned long
physaddr)
unsigned long __head __startup_64(unsigned long physaddr,
struct boot_params *bp)
{
+ unsigned long vaddr, vaddr_end;
unsigned long load_delta, *p;
unsigned long pgtable_flags;
pgdval_t *pgd;
@@ -234,6 +238,14 @@ unsigned long __head __startup_64(unsigned long physaddr,
/* Encrypt the kernel and related (if SME is active) */
sme_encrypt_kernel(bp);
+ /* Clear the memory encryption mask from the decrypted section */
+ vaddr = (unsigned long)__start_data_decrypted;
+ vaddr_end = (unsigned long)__end_data_decrypted;
+ for (; vaddr < vaddr_end; vaddr += PMD_SIZE) {
+ i = pmd_index(vaddr);
+ pmd[i] -= sme_get_me_mask();
+ }
+
/*
* Return the SME encryption mask (if SME is active) to be used as a
* modifier for the initial pgdir entry programmed into CR3.
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 8bde0a4..511b875 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -89,6 +89,22 @@ PHDRS {
note PT_NOTE FLAGS(0); /* ___ */
}
+/*
+ * This section contains data which will be mapped as decrypted. Memory
+ * encryption operates on a page basis. But we make this section a pmd
+ * aligned to avoid spliting the pages while mapping the section early.
+ *
+ * Note: We use a separate section so that only this section gets
+ * decrypted to avoid exposing more than we wish.
+ */
+#define DATA_DECRYPTED_SECTION \
+ . =
Re: [PATCH 1/2] x86/mm: add .data..decrypted section to hold shared variables
On 08/27/2018 05:11 PM, Tom Lendacky wrote:
On 08/27/2018 06:24 AM, Brijesh Singh wrote:
kvmclock defines few static variables which are shared with hypervisor
during the kvmclock initialization.
When SEV is active, memory is encrypted with a guest-specific key, and
if guest OS wants to share the memory region with hypervisor then it must
clear the C-bit before sharing it.
The '__decrypted' can be used to define a shared variables; the variables
will be put in the .data.decryption section. This section is mapped with
C=0 early in the boot, we also ensure that the initialized values are
updated to match with C=0 (i.e peform an in-place decryption). The
.data..decrypted section is PMD aligned and sized so that we avoid the
need for spliting the pages when map with C=0.
This should probably be broken into a few smaller patches. Maybe a
patch that adds the section and the attribute, a patch that re-arranges
the mapping setup and then the in-place decryption and clearing of the
encryption bit for the area.
OK, I will break the patch. Probably will create a separate patch which
just re-arranges the mapping setup without making any logical changes.
Signed-off-by: Brijesh Singh
Fixes: 368a540e0232 ("x86/kvmclock: Remove memblock dependency")
Cc: sta...@vger.kernel.org
Cc: Tom Lendacky
Cc: k...@vger.kernel.org
Cc: Thomas Gleixner
Cc: Borislav Petkov
Cc: "H. Peter Anvin"
Cc: linux-kernel@vger.kernel.org
Cc: Paolo Bonzini
Cc: Sean Christopherson
Cc: "Radim Krčmář"
---
arch/x86/include/asm/mem_encrypt.h | 4 +
arch/x86/kernel/head64.c | 12 ++
arch/x86/kernel/vmlinux.lds.S | 18 +++
arch/x86/mm/mem_encrypt_identity.c | 220 +++--
4 files changed, 197 insertions(+), 57 deletions(-)
diff --git a/arch/x86/include/asm/mem_encrypt.h
b/arch/x86/include/asm/mem_encrypt.h
index c064383..3f7d9d3 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -52,6 +52,8 @@ void __init mem_encrypt_init(void);
bool sme_active(void);
bool sev_active(void);
+#define __decrypted __attribute__((__section__(".data..decrypted")))
+
#else /* !CONFIG_AMD_MEM_ENCRYPT */
#define sme_me_mask 0ULL
@@ -77,6 +79,8 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long
size) { return 0;
static inline int __init
early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return
0; }
+#define __decrypted
+
#endif/* CONFIG_AMD_MEM_ENCRYPT */
/*
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 8047379..6a18297 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -43,6 +43,9 @@ extern pmd_t
early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
static unsigned int __initdata next_early_pgt;
pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
+/* To clear memory encryption mask from the decrypted section */
+extern char __start_data_decrypted[], __end_data_decrypted[];
+
Should find a header for these rather than defining them here.
OK, will move then in mem_encrypt.h. Will that work ?
#ifdef CONFIG_X86_5LEVEL
unsigned int __pgtable_l5_enabled __ro_after_init;
unsigned int pgdir_shift __ro_after_init = 39;
@@ -112,6 +115,7 @@ static bool __head check_la57_support(unsigned long
physaddr)
unsigned long __head __startup_64(unsigned long physaddr,
struct boot_params *bp)
{
+ unsigned long vaddr, vaddr_end;
unsigned long load_delta, *p;
unsigned long pgtable_flags;
pgdval_t *pgd;
@@ -234,6 +238,14 @@ unsigned long __head __startup_64(unsigned long physaddr,
/* Encrypt the kernel and related (if SME is active) */
sme_encrypt_kernel(bp);
+ /* Clear the memory encryption mask from the decrypted section */
+ vaddr = (unsigned long)__start_data_decrypted;
+ vaddr_end = (unsigned long)__end_data_decrypted;
+ for (; vaddr < vaddr_end; vaddr += PMD_SIZE) {
+ i = pmd_index(vaddr);
+ pmd[i] -= sme_get_me_mask();
+ }
+
/*
* Return the SME encryption mask (if SME is active) to be used as a
* modifier for the initial pgdir entry programmed into CR3.
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 8bde0a4..511b875 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -89,6 +89,22 @@ PHDRS {
note PT_NOTE FLAGS(0); /* ___ */
}
+/*
+ * This section contains data which will be mapped as decrypted. Memory
+ * encryption operates on a page basis. But we make this section a pmd
+ * aligned to avoid spliting the pages while mapping the section early.
+ *
+ * Note: We use a separate section so that only this section gets
+ * decrypted to avoid exposing more than we wish.
+ */
+#define DATA_DECRYPTED_SECTION \
+ . =
Re: [PATCH 1/2] x86/mm: add .data..decrypted section to hold shared variables
On 08/27/2018 06:24 AM, Brijesh Singh wrote:
> kvmclock defines few static variables which are shared with hypervisor
> during the kvmclock initialization.
>
> When SEV is active, memory is encrypted with a guest-specific key, and
> if guest OS wants to share the memory region with hypervisor then it must
> clear the C-bit before sharing it.
>
> The '__decrypted' can be used to define a shared variables; the variables
> will be put in the .data.decryption section. This section is mapped with
> C=0 early in the boot, we also ensure that the initialized values are
> updated to match with C=0 (i.e peform an in-place decryption). The
> .data..decrypted section is PMD aligned and sized so that we avoid the
> need for spliting the pages when map with C=0.
This should probably be broken into a few smaller patches. Maybe a
patch that adds the section and the attribute, a patch that re-arranges
the mapping setup and then the in-place decryption and clearing of the
encryption bit for the area.
>
> Signed-off-by: Brijesh Singh
> Fixes: 368a540e0232 ("x86/kvmclock: Remove memblock dependency")
> Cc: sta...@vger.kernel.org
> Cc: Tom Lendacky
> Cc: k...@vger.kernel.org
> Cc: Thomas Gleixner
> Cc: Borislav Petkov
> Cc: "H. Peter Anvin"
> Cc: linux-kernel@vger.kernel.org
> Cc: Paolo Bonzini
> Cc: Sean Christopherson
> Cc: "Radim Krčmář"
> ---
> arch/x86/include/asm/mem_encrypt.h | 4 +
> arch/x86/kernel/head64.c | 12 ++
> arch/x86/kernel/vmlinux.lds.S | 18 +++
> arch/x86/mm/mem_encrypt_identity.c | 220
> +++--
> 4 files changed, 197 insertions(+), 57 deletions(-)
>
> diff --git a/arch/x86/include/asm/mem_encrypt.h
> b/arch/x86/include/asm/mem_encrypt.h
> index c064383..3f7d9d3 100644
> --- a/arch/x86/include/asm/mem_encrypt.h
> +++ b/arch/x86/include/asm/mem_encrypt.h
> @@ -52,6 +52,8 @@ void __init mem_encrypt_init(void);
> bool sme_active(void);
> bool sev_active(void);
>
> +#define __decrypted __attribute__((__section__(".data..decrypted")))
> +
> #else/* !CONFIG_AMD_MEM_ENCRYPT */
>
> #define sme_me_mask 0ULL
> @@ -77,6 +79,8 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned
> long size) { return 0;
> static inline int __init
> early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return
> 0; }
>
> +#define __decrypted
> +
> #endif /* CONFIG_AMD_MEM_ENCRYPT */
>
> /*
> diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
> index 8047379..6a18297 100644
> --- a/arch/x86/kernel/head64.c
> +++ b/arch/x86/kernel/head64.c
> @@ -43,6 +43,9 @@ extern pmd_t
> early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
> static unsigned int __initdata next_early_pgt;
> pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
>
> +/* To clear memory encryption mask from the decrypted section */
> +extern char __start_data_decrypted[], __end_data_decrypted[];
> +
Should find a header for these rather than defining them here.
> #ifdef CONFIG_X86_5LEVEL
> unsigned int __pgtable_l5_enabled __ro_after_init;
> unsigned int pgdir_shift __ro_after_init = 39;
> @@ -112,6 +115,7 @@ static bool __head check_la57_support(unsigned long
> physaddr)
> unsigned long __head __startup_64(unsigned long physaddr,
> struct boot_params *bp)
> {
> + unsigned long vaddr, vaddr_end;
> unsigned long load_delta, *p;
> unsigned long pgtable_flags;
> pgdval_t *pgd;
> @@ -234,6 +238,14 @@ unsigned long __head __startup_64(unsigned long physaddr,
> /* Encrypt the kernel and related (if SME is active) */
> sme_encrypt_kernel(bp);
>
> + /* Clear the memory encryption mask from the decrypted section */
> + vaddr = (unsigned long)__start_data_decrypted;
> + vaddr_end = (unsigned long)__end_data_decrypted;
> + for (; vaddr < vaddr_end; vaddr += PMD_SIZE) {
> + i = pmd_index(vaddr);
> + pmd[i] -= sme_get_me_mask();
> + }
> +
> /*
>* Return the SME encryption mask (if SME is active) to be used as a
>* modifier for the initial pgdir entry programmed into CR3.
> diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
> index 8bde0a4..511b875 100644
> --- a/arch/x86/kernel/vmlinux.lds.S
> +++ b/arch/x86/kernel/vmlinux.lds.S
> @@ -89,6 +89,22 @@ PHDRS {
> note PT_NOTE FLAGS(0); /* ___ */
> }
>
> +/*
> + * This section contains data which will be mapped as decrypted. Memory
> + * encryption operates on a page basis. But we make this section a pmd
> + * aligned to avoid spliting the pages while mapping the section early.
> + *
> + * Note: We use a separate section so that only this section gets
> + * decrypted to avoid exposing more than we wish.
> + */
> +#define DATA_DECRYPTED_SECTION
> \
> + . = ALIGN(PMD_SIZE);\
> +
Re: [PATCH 1/2] x86/mm: add .data..decrypted section to hold shared variables
On 08/27/2018 06:24 AM, Brijesh Singh wrote:
> kvmclock defines few static variables which are shared with hypervisor
> during the kvmclock initialization.
>
> When SEV is active, memory is encrypted with a guest-specific key, and
> if guest OS wants to share the memory region with hypervisor then it must
> clear the C-bit before sharing it.
>
> The '__decrypted' can be used to define a shared variables; the variables
> will be put in the .data.decryption section. This section is mapped with
> C=0 early in the boot, we also ensure that the initialized values are
> updated to match with C=0 (i.e peform an in-place decryption). The
> .data..decrypted section is PMD aligned and sized so that we avoid the
> need for spliting the pages when map with C=0.
This should probably be broken into a few smaller patches. Maybe a
patch that adds the section and the attribute, a patch that re-arranges
the mapping setup and then the in-place decryption and clearing of the
encryption bit for the area.
>
> Signed-off-by: Brijesh Singh
> Fixes: 368a540e0232 ("x86/kvmclock: Remove memblock dependency")
> Cc: sta...@vger.kernel.org
> Cc: Tom Lendacky
> Cc: k...@vger.kernel.org
> Cc: Thomas Gleixner
> Cc: Borislav Petkov
> Cc: "H. Peter Anvin"
> Cc: linux-kernel@vger.kernel.org
> Cc: Paolo Bonzini
> Cc: Sean Christopherson
> Cc: "Radim Krčmář"
> ---
> arch/x86/include/asm/mem_encrypt.h | 4 +
> arch/x86/kernel/head64.c | 12 ++
> arch/x86/kernel/vmlinux.lds.S | 18 +++
> arch/x86/mm/mem_encrypt_identity.c | 220
> +++--
> 4 files changed, 197 insertions(+), 57 deletions(-)
>
> diff --git a/arch/x86/include/asm/mem_encrypt.h
> b/arch/x86/include/asm/mem_encrypt.h
> index c064383..3f7d9d3 100644
> --- a/arch/x86/include/asm/mem_encrypt.h
> +++ b/arch/x86/include/asm/mem_encrypt.h
> @@ -52,6 +52,8 @@ void __init mem_encrypt_init(void);
> bool sme_active(void);
> bool sev_active(void);
>
> +#define __decrypted __attribute__((__section__(".data..decrypted")))
> +
> #else/* !CONFIG_AMD_MEM_ENCRYPT */
>
> #define sme_me_mask 0ULL
> @@ -77,6 +79,8 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned
> long size) { return 0;
> static inline int __init
> early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return
> 0; }
>
> +#define __decrypted
> +
> #endif /* CONFIG_AMD_MEM_ENCRYPT */
>
> /*
> diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
> index 8047379..6a18297 100644
> --- a/arch/x86/kernel/head64.c
> +++ b/arch/x86/kernel/head64.c
> @@ -43,6 +43,9 @@ extern pmd_t
> early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
> static unsigned int __initdata next_early_pgt;
> pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
>
> +/* To clear memory encryption mask from the decrypted section */
> +extern char __start_data_decrypted[], __end_data_decrypted[];
> +
Should find a header for these rather than defining them here.
> #ifdef CONFIG_X86_5LEVEL
> unsigned int __pgtable_l5_enabled __ro_after_init;
> unsigned int pgdir_shift __ro_after_init = 39;
> @@ -112,6 +115,7 @@ static bool __head check_la57_support(unsigned long
> physaddr)
> unsigned long __head __startup_64(unsigned long physaddr,
> struct boot_params *bp)
> {
> + unsigned long vaddr, vaddr_end;
> unsigned long load_delta, *p;
> unsigned long pgtable_flags;
> pgdval_t *pgd;
> @@ -234,6 +238,14 @@ unsigned long __head __startup_64(unsigned long physaddr,
> /* Encrypt the kernel and related (if SME is active) */
> sme_encrypt_kernel(bp);
>
> + /* Clear the memory encryption mask from the decrypted section */
> + vaddr = (unsigned long)__start_data_decrypted;
> + vaddr_end = (unsigned long)__end_data_decrypted;
> + for (; vaddr < vaddr_end; vaddr += PMD_SIZE) {
> + i = pmd_index(vaddr);
> + pmd[i] -= sme_get_me_mask();
> + }
> +
> /*
>* Return the SME encryption mask (if SME is active) to be used as a
>* modifier for the initial pgdir entry programmed into CR3.
> diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
> index 8bde0a4..511b875 100644
> --- a/arch/x86/kernel/vmlinux.lds.S
> +++ b/arch/x86/kernel/vmlinux.lds.S
> @@ -89,6 +89,22 @@ PHDRS {
> note PT_NOTE FLAGS(0); /* ___ */
> }
>
> +/*
> + * This section contains data which will be mapped as decrypted. Memory
> + * encryption operates on a page basis. But we make this section a pmd
> + * aligned to avoid spliting the pages while mapping the section early.
> + *
> + * Note: We use a separate section so that only this section gets
> + * decrypted to avoid exposing more than we wish.
> + */
> +#define DATA_DECRYPTED_SECTION
> \
> + . = ALIGN(PMD_SIZE);\
> +
[PATCH 1/2] x86/mm: add .data..decrypted section to hold shared variables
kvmclock defines few static variables which are shared with hypervisor
during the kvmclock initialization.
When SEV is active, memory is encrypted with a guest-specific key, and
if guest OS wants to share the memory region with hypervisor then it must
clear the C-bit before sharing it.
The '__decrypted' can be used to define a shared variables; the variables
will be put in the .data.decryption section. This section is mapped with
C=0 early in the boot, we also ensure that the initialized values are
updated to match with C=0 (i.e peform an in-place decryption). The
.data..decrypted section is PMD aligned and sized so that we avoid the
need for spliting the pages when map with C=0.
Signed-off-by: Brijesh Singh
Fixes: 368a540e0232 ("x86/kvmclock: Remove memblock dependency")
Cc: sta...@vger.kernel.org
Cc: Tom Lendacky
Cc: k...@vger.kernel.org
Cc: Thomas Gleixner
Cc: Borislav Petkov
Cc: "H. Peter Anvin"
Cc: linux-kernel@vger.kernel.org
Cc: Paolo Bonzini
Cc: Sean Christopherson
Cc: "Radim Krčmář"
---
arch/x86/include/asm/mem_encrypt.h | 4 +
arch/x86/kernel/head64.c | 12 ++
arch/x86/kernel/vmlinux.lds.S | 18 +++
arch/x86/mm/mem_encrypt_identity.c | 220 +++--
4 files changed, 197 insertions(+), 57 deletions(-)
diff --git a/arch/x86/include/asm/mem_encrypt.h
b/arch/x86/include/asm/mem_encrypt.h
index c064383..3f7d9d3 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -52,6 +52,8 @@ void __init mem_encrypt_init(void);
bool sme_active(void);
bool sev_active(void);
+#define __decrypted __attribute__((__section__(".data..decrypted")))
+
#else /* !CONFIG_AMD_MEM_ENCRYPT */
#define sme_me_mask0ULL
@@ -77,6 +79,8 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long
size) { return 0;
static inline int __init
early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return
0; }
+#define __decrypted
+
#endif /* CONFIG_AMD_MEM_ENCRYPT */
/*
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 8047379..6a18297 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -43,6 +43,9 @@ extern pmd_t
early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
static unsigned int __initdata next_early_pgt;
pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
+/* To clear memory encryption mask from the decrypted section */
+extern char __start_data_decrypted[], __end_data_decrypted[];
+
#ifdef CONFIG_X86_5LEVEL
unsigned int __pgtable_l5_enabled __ro_after_init;
unsigned int pgdir_shift __ro_after_init = 39;
@@ -112,6 +115,7 @@ static bool __head check_la57_support(unsigned long
physaddr)
unsigned long __head __startup_64(unsigned long physaddr,
struct boot_params *bp)
{
+ unsigned long vaddr, vaddr_end;
unsigned long load_delta, *p;
unsigned long pgtable_flags;
pgdval_t *pgd;
@@ -234,6 +238,14 @@ unsigned long __head __startup_64(unsigned long physaddr,
/* Encrypt the kernel and related (if SME is active) */
sme_encrypt_kernel(bp);
+ /* Clear the memory encryption mask from the decrypted section */
+ vaddr = (unsigned long)__start_data_decrypted;
+ vaddr_end = (unsigned long)__end_data_decrypted;
+ for (; vaddr < vaddr_end; vaddr += PMD_SIZE) {
+ i = pmd_index(vaddr);
+ pmd[i] -= sme_get_me_mask();
+ }
+
/*
* Return the SME encryption mask (if SME is active) to be used as a
* modifier for the initial pgdir entry programmed into CR3.
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 8bde0a4..511b875 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -89,6 +89,22 @@ PHDRS {
note PT_NOTE FLAGS(0); /* ___ */
}
+/*
+ * This section contains data which will be mapped as decrypted. Memory
+ * encryption operates on a page basis. But we make this section a pmd
+ * aligned to avoid spliting the pages while mapping the section early.
+ *
+ * Note: We use a separate section so that only this section gets
+ * decrypted to avoid exposing more than we wish.
+ */
+#define DATA_DECRYPTED_SECTION \
+ . = ALIGN(PMD_SIZE);\
+ __start_data_decrypted = .; \
+ *(.data..decrypted);\
+ __end_data_decrypted = .; \
+ . = ALIGN(PMD_SIZE);\
+
+
SECTIONS
{
#ifdef CONFIG_X86_32
@@ -171,6 +187,8 @@ SECTIONS
/* rarely changed data like cpu maps */
READ_MOSTLY_DATA(INTERNODE_CACHE_BYTES)
+ DATA_DECRYPTED_SECTION
+
/* End of data section */
_edata =
[PATCH 1/2] x86/mm: add .data..decrypted section to hold shared variables
kvmclock defines few static variables which are shared with hypervisor
during the kvmclock initialization.
When SEV is active, memory is encrypted with a guest-specific key, and
if guest OS wants to share the memory region with hypervisor then it must
clear the C-bit before sharing it.
The '__decrypted' can be used to define a shared variables; the variables
will be put in the .data.decryption section. This section is mapped with
C=0 early in the boot, we also ensure that the initialized values are
updated to match with C=0 (i.e peform an in-place decryption). The
.data..decrypted section is PMD aligned and sized so that we avoid the
need for spliting the pages when map with C=0.
Signed-off-by: Brijesh Singh
Fixes: 368a540e0232 ("x86/kvmclock: Remove memblock dependency")
Cc: sta...@vger.kernel.org
Cc: Tom Lendacky
Cc: k...@vger.kernel.org
Cc: Thomas Gleixner
Cc: Borislav Petkov
Cc: "H. Peter Anvin"
Cc: linux-kernel@vger.kernel.org
Cc: Paolo Bonzini
Cc: Sean Christopherson
Cc: "Radim Krčmář"
---
arch/x86/include/asm/mem_encrypt.h | 4 +
arch/x86/kernel/head64.c | 12 ++
arch/x86/kernel/vmlinux.lds.S | 18 +++
arch/x86/mm/mem_encrypt_identity.c | 220 +++--
4 files changed, 197 insertions(+), 57 deletions(-)
diff --git a/arch/x86/include/asm/mem_encrypt.h
b/arch/x86/include/asm/mem_encrypt.h
index c064383..3f7d9d3 100644
--- a/arch/x86/include/asm/mem_encrypt.h
+++ b/arch/x86/include/asm/mem_encrypt.h
@@ -52,6 +52,8 @@ void __init mem_encrypt_init(void);
bool sme_active(void);
bool sev_active(void);
+#define __decrypted __attribute__((__section__(".data..decrypted")))
+
#else /* !CONFIG_AMD_MEM_ENCRYPT */
#define sme_me_mask0ULL
@@ -77,6 +79,8 @@ early_set_memory_decrypted(unsigned long vaddr, unsigned long
size) { return 0;
static inline int __init
early_set_memory_encrypted(unsigned long vaddr, unsigned long size) { return
0; }
+#define __decrypted
+
#endif /* CONFIG_AMD_MEM_ENCRYPT */
/*
diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c
index 8047379..6a18297 100644
--- a/arch/x86/kernel/head64.c
+++ b/arch/x86/kernel/head64.c
@@ -43,6 +43,9 @@ extern pmd_t
early_dynamic_pgts[EARLY_DYNAMIC_PAGE_TABLES][PTRS_PER_PMD];
static unsigned int __initdata next_early_pgt;
pmdval_t early_pmd_flags = __PAGE_KERNEL_LARGE & ~(_PAGE_GLOBAL | _PAGE_NX);
+/* To clear memory encryption mask from the decrypted section */
+extern char __start_data_decrypted[], __end_data_decrypted[];
+
#ifdef CONFIG_X86_5LEVEL
unsigned int __pgtable_l5_enabled __ro_after_init;
unsigned int pgdir_shift __ro_after_init = 39;
@@ -112,6 +115,7 @@ static bool __head check_la57_support(unsigned long
physaddr)
unsigned long __head __startup_64(unsigned long physaddr,
struct boot_params *bp)
{
+ unsigned long vaddr, vaddr_end;
unsigned long load_delta, *p;
unsigned long pgtable_flags;
pgdval_t *pgd;
@@ -234,6 +238,14 @@ unsigned long __head __startup_64(unsigned long physaddr,
/* Encrypt the kernel and related (if SME is active) */
sme_encrypt_kernel(bp);
+ /* Clear the memory encryption mask from the decrypted section */
+ vaddr = (unsigned long)__start_data_decrypted;
+ vaddr_end = (unsigned long)__end_data_decrypted;
+ for (; vaddr < vaddr_end; vaddr += PMD_SIZE) {
+ i = pmd_index(vaddr);
+ pmd[i] -= sme_get_me_mask();
+ }
+
/*
* Return the SME encryption mask (if SME is active) to be used as a
* modifier for the initial pgdir entry programmed into CR3.
diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S
index 8bde0a4..511b875 100644
--- a/arch/x86/kernel/vmlinux.lds.S
+++ b/arch/x86/kernel/vmlinux.lds.S
@@ -89,6 +89,22 @@ PHDRS {
note PT_NOTE FLAGS(0); /* ___ */
}
+/*
+ * This section contains data which will be mapped as decrypted. Memory
+ * encryption operates on a page basis. But we make this section a pmd
+ * aligned to avoid spliting the pages while mapping the section early.
+ *
+ * Note: We use a separate section so that only this section gets
+ * decrypted to avoid exposing more than we wish.
+ */
+#define DATA_DECRYPTED_SECTION \
+ . = ALIGN(PMD_SIZE);\
+ __start_data_decrypted = .; \
+ *(.data..decrypted);\
+ __end_data_decrypted = .; \
+ . = ALIGN(PMD_SIZE);\
+
+
SECTIONS
{
#ifdef CONFIG_X86_32
@@ -171,6 +187,8 @@ SECTIONS
/* rarely changed data like cpu maps */
READ_MOSTLY_DATA(INTERNODE_CACHE_BYTES)
+ DATA_DECRYPTED_SECTION
+
/* End of data section */
_edata =
