[PATCH] KVM: arm64: Support PREL/PLT relocs in EL2 code
gen-hyprel tool parses object files of the EL2 portion of KVM
and generates runtime relocation data. While only filtering for
R_AARCH64_ABS64 relocations in the input object files, it has an
allow-list of relocation types that are used for relative
addressing. Other, unexpected, relocation types are rejected and
cause the build to fail.
This allow-list did not include the position-relative relocation
types R_AARCH64_PREL64/32/16 and the recently introduced _PLT32.
While not seen used by toolchains in the wild, add them to the
allow-list for completeness.
Fixes: 8c49b5d43d4c ("KVM: arm64: Generate hyp relocation data")
Cc:
Reported-by: Will Deacon
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/gen-hyprel.c | 18 ++
1 file changed, 18 insertions(+)
diff --git a/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
b/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
index ead02c6a7628..6bc88a756cb7 100644
--- a/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
+++ b/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
@@ -50,6 +50,18 @@
#ifndef R_AARCH64_ABS64
#define R_AARCH64_ABS64257
#endif
+#ifndef R_AARCH64_PREL64
+#define R_AARCH64_PREL64 260
+#endif
+#ifndef R_AARCH64_PREL32
+#define R_AARCH64_PREL32 261
+#endif
+#ifndef R_AARCH64_PREL16
+#define R_AARCH64_PREL16 262
+#endif
+#ifndef R_AARCH64_PLT32
+#define R_AARCH64_PLT32314
+#endif
#ifndef R_AARCH64_LD_PREL_LO19
#define R_AARCH64_LD_PREL_LO19 273
#endif
@@ -371,6 +383,12 @@ static void emit_rela_section(Elf64_Shdr *sh_rela)
case R_AARCH64_ABS64:
emit_rela_abs64(rela, sh_orig_name);
break;
+ /* Allow position-relative data relocations. */
+ case R_AARCH64_PREL64:
+ case R_AARCH64_PREL32:
+ case R_AARCH64_PREL16:
+ case R_AARCH64_PLT32:
+ break;
/* Allow relocations to generate PC-relative addressing. */
case R_AARCH64_LD_PREL_LO19:
case R_AARCH64_ADR_PREL_LO21:
--
2.31.0.291.g576ba9dcdaf-goog
[PATCH pre-5.10] selinux: vsock: Set SID for socket returned by accept()
[Backport of commit 1f935e8e72ec28dddb2dc0650b3b6626a293d94b to all
stable branches from 4.4 to 5.4, inclusive]
For AF_VSOCK, accept() currently returns sockets that are unlabelled.
Other socket families derive the child's SID from the SID of the parent
and the SID of the incoming packet. This is typically done as the
connected socket is placed in the queue that accept() removes from.
Reuse the existing 'security_sk_clone' hook to copy the SID from the
parent (server) socket to the child. There is no packet SID in this
case.
Cc: sta...@vger.kernel.org
Fixes: d021c344051a ("VSOCK: Introduce VM Sockets")
Signed-off-by: David Brazdil
Signed-off-by: David S. Miller
---
net/vmw_vsock/af_vsock.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
index 5d323574d04f..c82e7b52ab1f 100644
--- a/net/vmw_vsock/af_vsock.c
+++ b/net/vmw_vsock/af_vsock.c
@@ -620,6 +620,7 @@ struct sock *__vsock_create(struct net *net,
vsk->trusted = psk->trusted;
vsk->owner = get_cred(psk->owner);
vsk->connect_timeout = psk->connect_timeout;
+ security_sk_clone(parent, sk);
} else {
vsk->trusted = ns_capable_noaudit(_user_ns, CAP_NET_ADMIN);
vsk->owner = get_current_cred();
--
2.31.0.291.g576ba9dcdaf-goog
Re: [PATCH v2] selinux: vsock: Set SID for socket returned by accept()
On Fri, Mar 19, 2021 at 01:05:41PM +, David Brazdil wrote:
> For AF_VSOCK, accept() currently returns sockets that are unlabelled.
> Other socket families derive the child's SID from the SID of the parent
> and the SID of the incoming packet. This is typically done as the
> connected socket is placed in the queue that accept() removes from.
>
> Reuse the existing 'security_sk_clone' hook to copy the SID from the
> parent (server) socket to the child. There is no packet SID in this
> case.
>
> Fixes: d021c344051a ("VSOCK: Introduce VM Sockets")
Cc:
> Signed-off-by: David Brazdil
> ---
> Tested on Android AOSP and Fedora 33 with v5.12-rc3.
> Unit test is available here:
> https://github.com/SELinuxProject/selinux-testsuite/pull/75
>
> Changes since v1:
> * reuse security_sk_clone instead of adding a new hook
>
> net/vmw_vsock/af_vsock.c | 1 +
> 1 file changed, 1 insertion(+)
>
> diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
> index 5546710d8ac1..bc7fb9bf3351 100644
> --- a/net/vmw_vsock/af_vsock.c
> +++ b/net/vmw_vsock/af_vsock.c
> @@ -755,6 +755,7 @@ static struct sock *__vsock_create(struct net *net,
> vsk->buffer_size = psk->buffer_size;
> vsk->buffer_min_size = psk->buffer_min_size;
> vsk->buffer_max_size = psk->buffer_max_size;
> + security_sk_clone(parent, sk);
> } else {
> vsk->trusted = ns_capable_noaudit(_user_ns, CAP_NET_ADMIN);
> vsk->owner = get_current_cred();
> --
> 2.31.0.rc2.261.g7f71774620-goog
>
[PATCH v2] selinux: vsock: Set SID for socket returned by accept()
For AF_VSOCK, accept() currently returns sockets that are unlabelled.
Other socket families derive the child's SID from the SID of the parent
and the SID of the incoming packet. This is typically done as the
connected socket is placed in the queue that accept() removes from.
Reuse the existing 'security_sk_clone' hook to copy the SID from the
parent (server) socket to the child. There is no packet SID in this
case.
Fixes: d021c344051a ("VSOCK: Introduce VM Sockets")
Signed-off-by: David Brazdil
---
Tested on Android AOSP and Fedora 33 with v5.12-rc3.
Unit test is available here:
https://github.com/SELinuxProject/selinux-testsuite/pull/75
Changes since v1:
* reuse security_sk_clone instead of adding a new hook
net/vmw_vsock/af_vsock.c | 1 +
1 file changed, 1 insertion(+)
diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
index 5546710d8ac1..bc7fb9bf3351 100644
--- a/net/vmw_vsock/af_vsock.c
+++ b/net/vmw_vsock/af_vsock.c
@@ -755,6 +755,7 @@ static struct sock *__vsock_create(struct net *net,
vsk->buffer_size = psk->buffer_size;
vsk->buffer_min_size = psk->buffer_min_size;
vsk->buffer_max_size = psk->buffer_max_size;
+ security_sk_clone(parent, sk);
} else {
vsk->trusted = ns_capable_noaudit(_user_ns, CAP_NET_ADMIN);
vsk->owner = get_current_cred();
--
2.31.0.rc2.261.g7f71774620-goog
Re: [PATCH] selinux: vsock: Set SID for socket returned by accept()
Hi Paul, I'll post a v2 shortly but will address your comments here. > > include/linux/lsm_hooks.h | 7 +++ > > include/linux/security.h | 5 + > > net/vmw_vsock/af_vsock.c | 1 + > > security/security.c | 5 + > > security/selinux/hooks.c | 10 ++ > > 6 files changed, 29 insertions(+) > > Additional comments below, but I think it would be a good idea for you > to test your patches on a more traditional Linux distribution as well > as Android. > No problem, I was going to add a test case into selinux-testsuite anyway. Done now (link in v2) and tested on Fedora 33 with v5.12-rc3. > > diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c > > index 5546710d8ac1..a9bf3b90cb2f 100644 > > --- a/net/vmw_vsock/af_vsock.c > > +++ b/net/vmw_vsock/af_vsock.c > > @@ -755,6 +755,7 @@ static struct sock *__vsock_create(struct net *net, > > vsk->buffer_size = psk->buffer_size; > > vsk->buffer_min_size = psk->buffer_min_size; > > vsk->buffer_max_size = psk->buffer_max_size; > > + security_vsock_sk_clone(parent, sk); > > Did you try calling the existing security_sk_clone() hook here? I > would be curious to hear why it doesn't work in this case. > > Feel free to educate me on AF_VSOCK, it's entirely possible I'm > misunderstanding something here :) > No, you're completely right. security_sk_clone does what's needed here. Adding a new hook was me trying to mimic other socket families going via selinux_conn_sid. Happy to reuse the existing hook - makes this a nice oneliner. :) Please note that I'm marking v2 with 'Fixes' for backporting. This does feel to me like a bug, an integration that was never considered. Please shout if you disagree. -David
[PATCH] selinux: vsock: Set SID for socket returned by accept()
For AF_VSOCK, accept() currently returns sockets that are unlabelled.
Other socket families derive the child's SID from the SID of the parent
and the SID of the incoming packet. This is typically done as the
connected socket is placed in the queue that accept() removes from.
Implement an LSM hook 'vsock_sk_clone' that takes the parent (server)
and child (connection) struct socks, and assigns the parent SID to the
child. There is no packet SID in this case.
Signed-off-by: David Brazdil
---
This is my first patch in this part of the kernel so please comment if I
missed anything, specifically whether there is a packet SID that should
be mixed into the child SID.
Tested on Android.
include/linux/lsm_hook_defs.h | 1 +
include/linux/lsm_hooks.h | 7 +++
include/linux/security.h | 5 +
net/vmw_vsock/af_vsock.c | 1 +
security/security.c | 5 +
security/selinux/hooks.c | 10 ++
6 files changed, 29 insertions(+)
diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h
index 477a597db013..f35e422b2b5c 100644
--- a/include/linux/lsm_hook_defs.h
+++ b/include/linux/lsm_hook_defs.h
@@ -329,6 +329,7 @@ LSM_HOOK(int, 0, sctp_bind_connect, struct sock *sk, int
optname,
struct sockaddr *address, int addrlen)
LSM_HOOK(void, LSM_RET_VOID, sctp_sk_clone, struct sctp_endpoint *ep,
struct sock *sk, struct sock *newsk)
+LSM_HOOK(void, LSM_RET_VOID, vsock_sk_clone, struct sock *sock, struct sock
*newsk)
#endif /* CONFIG_SECURITY_NETWORK */
#ifdef CONFIG_SECURITY_INFINIBAND
diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h
index fb7f3193753d..1b4e92990401 100644
--- a/include/linux/lsm_hooks.h
+++ b/include/linux/lsm_hooks.h
@@ -1033,6 +1033,13 @@
* @sk pointer to current sock structure.
* @sk pointer to new sock structure.
*
+ * Security hooks for vSockets
+ *
+ * @vsock_sk_clone:
+ * Clone SID from the server socket to a newly connected child socket.
+ * @sock contains the sock structure.
+ * @newsk contains the new sock structure.
+ *
* Security hooks for Infiniband
*
* @ib_pkey_access:
diff --git a/include/linux/security.h b/include/linux/security.h
index 8aeebd6646dc..ffac67058355 100644
--- a/include/linux/security.h
+++ b/include/linux/security.h
@@ -1404,6 +1404,7 @@ int security_sctp_bind_connect(struct sock *sk, int
optname,
struct sockaddr *address, int addrlen);
void security_sctp_sk_clone(struct sctp_endpoint *ep, struct sock *sk,
struct sock *newsk);
+void security_vsock_sk_clone(struct sock *sock, struct sock *newsk);
#else /* CONFIG_SECURITY_NETWORK */
static inline int security_unix_stream_connect(struct sock *sock,
@@ -1623,6 +1624,10 @@ static inline void security_sctp_sk_clone(struct
sctp_endpoint *ep,
struct sock *newsk)
{
}
+
+static inline void security_vsock_sk_clone(struct sock *sock, struct sock
*newsk)
+{
+}
#endif /* CONFIG_SECURITY_NETWORK */
#ifdef CONFIG_SECURITY_INFINIBAND
diff --git a/net/vmw_vsock/af_vsock.c b/net/vmw_vsock/af_vsock.c
index 5546710d8ac1..a9bf3b90cb2f 100644
--- a/net/vmw_vsock/af_vsock.c
+++ b/net/vmw_vsock/af_vsock.c
@@ -755,6 +755,7 @@ static struct sock *__vsock_create(struct net *net,
vsk->buffer_size = psk->buffer_size;
vsk->buffer_min_size = psk->buffer_min_size;
vsk->buffer_max_size = psk->buffer_max_size;
+ security_vsock_sk_clone(parent, sk);
} else {
vsk->trusted = ns_capable_noaudit(_user_ns, CAP_NET_ADMIN);
vsk->owner = get_current_cred();
diff --git a/security/security.c b/security/security.c
index 5ac96b16f8fa..050b653405e0 100644
--- a/security/security.c
+++ b/security/security.c
@@ -2335,6 +2335,11 @@ void security_sctp_sk_clone(struct sctp_endpoint *ep,
struct sock *sk,
}
EXPORT_SYMBOL(security_sctp_sk_clone);
+void security_vsock_sk_clone(struct sock *sock, struct sock *newsk)
+{
+ call_void_hook(vsock_sk_clone, sock, newsk);
+}
+EXPORT_SYMBOL(security_vsock_sk_clone);
#endif /* CONFIG_SECURITY_NETWORK */
#ifdef CONFIG_SECURITY_INFINIBAND
diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c
index ddd097790d47..7b92d6f2e0fd 100644
--- a/security/selinux/hooks.c
+++ b/security/selinux/hooks.c
@@ -5616,6 +5616,15 @@ static int selinux_tun_dev_open(void *security)
return 0;
}
+static void selinux_socket_vsock_sk_clone(struct sock *sock, struct sock
*newsk)
+{
+ struct sk_security_struct *sksec_sock = sock->sk_security;
+ struct sk_security_struct *sksec_new = newsk->sk_security;
+
+ /* Always returns 0 when packet SID is SECSID_NULL. */
+ WARN_ON_ONCE(selinux_conn_sid(sksec_sock->sid, SECSID_NULL,
_new->sid));
+}
+
#ifdef CONFIG_NETFILTER
static unsigned int selinux_ip_forward(struct sk_buff *skb,
Re: [PATCH v2 4/8] KVM: arm64: Generate hyp relocation data
Thanks for writing the fix, Marc! There are no corner cases in this code so if it boots, that should be a good indicator that all BE inputs were converted. Just one little thing I noticed below, otherwise: Acked-by: David Brazdil > arch/arm64/kvm/hyp/nvhe/Makefile | 1 + > arch/arm64/kvm/hyp/nvhe/gen-hyprel.c | 57 > 2 files changed, 42 insertions(+), 16 deletions(-) > > diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile > b/arch/arm64/kvm/hyp/nvhe/Makefile > index 268be1376f74..09d04dd50eb8 100644 > --- a/arch/arm64/kvm/hyp/nvhe/Makefile > +++ b/arch/arm64/kvm/hyp/nvhe/Makefile > @@ -7,6 +7,7 @@ asflags-y := -D__KVM_NVHE_HYPERVISOR__ > ccflags-y := -D__KVM_NVHE_HYPERVISOR__ > > hostprogs := gen-hyprel > +HOST_EXTRACFLAGS += -I$(srctree)/include This should be $(objtree), autoconf.h is generated. David
Re: [PATCH v3 2/9] KVM: arm64: Add a buffer that can pass UBSan data from hyp/nVHE to kernel
On Fri, Jan 15, 2021 at 05:18:23PM +, Elena Petrova wrote: > From: George Popescu > > Share a buffer between the kernel and the hyp/nVHE code by using the > macros from kvm_debug_buffer.h. > The hyp/nVHE code requires a write index which counts how many elements > have been writtens inside the buffer and the kernel requires a read > index which counts how many elements have been read from the buffer. > The write index and the buffer are shared with the kernel in read-only. > > The kvm_debug_buffer_ind returns the reading and writing points of the > circular buffer and updates the reading index. > > Data collected from UBSan handlers inside hyp/nVHE is stored in the > kvm_ubsan_buffer. > This buffer stores only UBSan data because it should not be preoccupied > by other mechanisms data structures and functionalities. > > Also, for the moment the buffer is mapped inside .bss, where both the kernel > and the hyp/nVHE code have Read/Write rights, but in the future this will > change > and the kernel will not be able to acess hyp/nVHE's .bss. At that point the > buffer > will only need to be mapped in order for this patch to work. > > Signed-off-by: Elena Petrova > --- > arch/arm64/include/asm/assembler.h| 11 +++ > arch/arm64/include/asm/kvm_debug_buffer.h | 36 > arch/arm64/include/asm/kvm_host.h | 8 - > arch/arm64/include/asm/kvm_ubsan.h| 14 > arch/arm64/kvm/Makefile | 2 ++ > arch/arm64/kvm/arm.c | 9 + > arch/arm64/kvm/hyp/nvhe/host.S| 4 +++ > arch/arm64/kvm/hyp/nvhe/ubsan.c | 23 + > arch/arm64/kvm/kvm_ubsan_buffer.c | 40 +++ > 9 files changed, 146 insertions(+), 1 deletion(-) > create mode 100644 arch/arm64/include/asm/kvm_debug_buffer.h > create mode 100644 arch/arm64/include/asm/kvm_ubsan.h > create mode 100644 arch/arm64/kvm/kvm_ubsan_buffer.c > > diff --git a/arch/arm64/include/asm/assembler.h > b/arch/arm64/include/asm/assembler.h > index bf125c591116..ebc18a8a0e1f 100644 > --- a/arch/arm64/include/asm/assembler.h > +++ b/arch/arm64/include/asm/assembler.h > @@ -258,6 +258,17 @@ alternative_endif > ldr \dst, [\dst, \tmp] > .endm > > + /* > + * @sym: The name of the per-cpu variable > + * @reg: value to store > + * @tmp1: scratch register > + * @tmp2: scratch register > + */ > +.macro str_this_cpu sym, reg, tmp1, tmp2 > +adr_this_cpu \tmp1, \sym, \tmp2 > +str \reg, [\tmp1] > +.endm > + > /* > * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) > */ > diff --git a/arch/arm64/include/asm/kvm_debug_buffer.h > b/arch/arm64/include/asm/kvm_debug_buffer.h > new file mode 100644 > index ..e5375c2cff1a > --- /dev/null > +++ b/arch/arm64/include/asm/kvm_debug_buffer.h > @@ -0,0 +1,36 @@ > +/* SPDX-License-Identifier: GPL-2.0-only */ > +/* > + * Copyright 2020 Google LLC > + * Author: George Popescu > + */ > + > +#include > + > + > +#define KVM_DEBUG_BUFFER_SIZE 1000 nit: unused? > + > +#ifdef __KVM_NVHE_HYPERVISOR__ > +#define DEFINE_KVM_DEBUG_BUFFER(type_name, buffer_name, write_ind, size)\ > + DEFINE_PER_CPU(type_name, buffer_name)[size]; \ > + DEFINE_PER_CPU(unsigned long, write_ind) = 0; This is subjective but I would: * stop calling the second variable a "write index" and instead call it "cursor" - I think this is a leftover from George's early two-cursor implementation * wrap these two variables in a struct; that is a bit tricky given that each instantiation defines its own size (the macros would have to generate that struct), so bar that I would at least generate the name of the cursor variable as buffer_name##suffix. > + > +#define DECLARE_KVM_DEBUG_BUFFER(type_name, buffer_name, write_ind, size)\ > + DECLARE_PER_CPU(type_name, buffer_name)[size]; \ > + DECLARE_PER_CPU(unsigned long, write_ind); > +#else > +#define DECLARE_KVM_DEBUG_BUFFER(type_name, buffer_name, write_ind, size)\ > + DECLARE_KVM_NVHE_PER_CPU(type_name, buffer_name)[size]; \ > +DECLARE_KVM_NVHE_PER_CPU(unsigned long, write_ind); > +#endif //__KVM_NVHE_HYPERVISOR__ > + > +#ifdef __ASSEMBLY__ > +#include > + > +.macro clear_buffer tmp1, tmp2, tmp3 > +mov \tmp1, 0 > +#ifdef CONFIG_UBSAN > +str_this_cpu kvm_ubsan_buff_wr_ind, \tmp1, \tmp2, \tmp3 > +#endif //CONFIG_UBSAN > +.endm I find this macro odd. Everything else in this file is about a generic buffer data structure, yet this macro enumerates all instantiations of it (currently just one). Please keep this generic and pass the name of the buffer cursor as a parameter. > + > +#endif > \ No newline at end of file > diff --git a/arch/arm64/include/asm/kvm_host.h > b/arch/arm64/include/asm/kvm_host.h > index 8fcfab0c2567..905ca0d7c52c 100644 > ---
Re: [PATCH v3 9/9] KVM: arm64: Add UBSan tests for PKVM.
On Fri, Jan 15, 2021 at 05:18:30PM +, Elena Petrova wrote:
> From: George-Aurelian Popescu
>
> Test the UBsan functionality inside hyp/nVHE.
> Because modules are not supported inside of hyp/nVHE code, the default
> testing module for UBSan can not be used.
> New functions have to be defined inside of hyp/nVHE.
> They are called in kvm_get_mdcr_el2, to test UBSAN whenever a VM starts.
>
> Signed-off-by: Elena Petrova
> ---
> arch/arm64/include/asm/assembler.h | 17 ++-
> arch/arm64/include/asm/kvm_debug_buffer.h | 10 +-
> arch/arm64/include/asm/kvm_ubsan.h | 2 +-
> arch/arm64/kvm/hyp/include/hyp/test_ubsan.h | 112
> arch/arm64/kvm/hyp/nvhe/hyp-main.c | 3 +
> arch/arm64/kvm/kvm_ubsan_buffer.c | 1 -
> 6 files changed, 128 insertions(+), 17 deletions(-)
> create mode 100644 arch/arm64/kvm/hyp/include/hyp/test_ubsan.h
>
> diff --git a/arch/arm64/include/asm/assembler.h
> b/arch/arm64/include/asm/assembler.h
> index ebc18a8a0e1f..8422b0d925e8 100644
> --- a/arch/arm64/include/asm/assembler.h
> +++ b/arch/arm64/include/asm/assembler.h
> @@ -259,16 +259,15 @@ alternative_endif
> .endm
>
> /*
> - * @sym: The name of the per-cpu variable
> - * @reg: value to store
> - * @tmp1: scratch register
> - * @tmp2: scratch register
> - */
> -.macro str_this_cpu sym, reg, tmp1, tmp2
> -adr_this_cpu \tmp1, \sym, \tmp2
> + * @sym: The name of the per-cpu variable
> + * @reg: value to store
> + * @tmp1: scratch register
> + * @tmp2: scratch register
> + */
> + .macro str_this_cpu sym, reg, tmp1, tmp2
> + adr_this_cpu \tmp1, \sym, \tmp2
> str \reg, [\tmp1]
> -.endm
> -
> + .endm
> /*
> * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm)
> */
> diff --git a/arch/arm64/include/asm/kvm_debug_buffer.h
> b/arch/arm64/include/asm/kvm_debug_buffer.h
> index e5375c2cff1a..361b473bb004 100644
> --- a/arch/arm64/include/asm/kvm_debug_buffer.h
> +++ b/arch/arm64/include/asm/kvm_debug_buffer.h
> @@ -3,10 +3,8 @@
> * Copyright 2020 Google LLC
> * Author: George Popescu
> */
> -
> #include
>
> -
> #define KVM_DEBUG_BUFFER_SIZE 1000
>
> #ifdef __KVM_NVHE_HYPERVISOR__
> @@ -20,17 +18,17 @@
> #else
> #define DECLARE_KVM_DEBUG_BUFFER(type_name, buffer_name, write_ind, size)\
> DECLARE_KVM_NVHE_PER_CPU(type_name, buffer_name)[size]; \
> -DECLARE_KVM_NVHE_PER_CPU(unsigned long, write_ind);
> + DECLARE_KVM_NVHE_PER_CPU(unsigned long, write_ind);
> #endif //__KVM_NVHE_HYPERVISOR__
>
> #ifdef __ASSEMBLY__
> #include
>
> .macro clear_buffer tmp1, tmp2, tmp3
> -mov \tmp1, 0
> + mov \tmp1, 0
> #ifdef CONFIG_UBSAN
> -str_this_cpu kvm_ubsan_buff_wr_ind, \tmp1, \tmp2, \tmp3
> + str_this_cpu kvm_ubsan_buff_wr_ind, \tmp1, \tmp2, \tmp3
> #endif //CONFIG_UBSAN
> .endm
Are these fixing formatting? If so, move it to the patch that introduced this.
>
> -#endif
> \ No newline at end of file
> +#endif
> diff --git a/arch/arm64/include/asm/kvm_ubsan.h
> b/arch/arm64/include/asm/kvm_ubsan.h
> index da4a3b4e28e0..0b8bed08d48e 100644
> --- a/arch/arm64/include/asm/kvm_ubsan.h
> +++ b/arch/arm64/include/asm/kvm_ubsan.h
> @@ -9,7 +9,6 @@
> #define UBSAN_MAX_TYPE 6
> #define KVM_UBSAN_BUFFER_SIZE 1000
>
> -
ditto
> struct ubsan_values {
> void *lval;
> void *rval;
> @@ -18,6 +17,7 @@ struct ubsan_values {
>
> struct kvm_ubsan_info {
> enum {
> + UBSAN_NONE,
This also looks like it should have been in a previous patch. The code assumes
that 'type == 0' means 'empty slot'. So presumably this is fixing a bug?
> UBSAN_OUT_OF_BOUNDS,
> UBSAN_UNREACHABLE_DATA,
> UBSAN_SHIFT_OUT_OF_BOUNDS,
> diff --git a/arch/arm64/kvm/hyp/include/hyp/test_ubsan.h
> b/arch/arm64/kvm/hyp/include/hyp/test_ubsan.h
> new file mode 100644
> index ..07759c0d1e0e
> --- /dev/null
> +++ b/arch/arm64/kvm/hyp/include/hyp/test_ubsan.h
> @@ -0,0 +1,112 @@
> +/* SPDX-License-Identifier: GPL-2.0-only */
> +
> +#include
> +
> +typedef void(*test_ubsan_fp)(void);
> +
> +static void test_ubsan_add_overflow(void)
> +{
> + volatile int val = INT_MAX;
> +
> + val += 2;
> +}
> +
> +static void test_ubsan_sub_overflow(void)
> +{
> + volatile int val = INT_MIN;
> + volatile int val2 = 2;
> +
> + val -= val2;
> +}
> +
> +static void test_ubsan_mul_overflow(void)
> +{
> + volatile int val = INT_MAX / 2;
> +
> + val *= 3;
> +}
> +
> +static void test_ubsan_negate_overflow(void)
> +{
> + volatile int val = INT_MIN;
> +
> + val = -val;
> +}
> +
> +static void test_ubsan_divrem_overflow(void)
> +{
> + volatile int val = 16;
> + volatile int val2 = 0;
> +
> + val /= val2;
> +}
> +
> +static void test_ubsan_shift_out_of_bounds(void)
> +{
> + volatile int val = -1;
> + int val2
Re: [PATCH v4 00/21] arm64: Early CPU feature override, and applications to VHE, BTI and PAuth
On Mon, Jan 18, 2021 at 09:45:12AM +, Marc Zyngier wrote: > It recently came to light that there is a need to be able to override > some CPU features very early on, before the kernel is fully up and > running. The reasons for this range from specific feature support > (such as using Protected KVM on VHE HW, which is the main motivation > for this work) to errata workaround (a feature is broken on a CPU and > needs to be turned off, or rather not enabled). > > This series tries to offer a limited framework for this kind of > problems, by allowing a set of options to be passed on the > command-line and altering the feature set that the cpufeature > subsystem exposes to the rest of the kernel. Note that this doesn't > change anything for code that directly uses the CPU ID registers. > > The series completely changes the way a VHE-capable system boots, by > *always* booting non-VHE first, and then upgrading to VHE when deemed > capable. Although it sounds scary, this is actually simple to > implement (and I wish I had done that five years ago). The "upgrade to > VHE" path is then conditioned on the VHE feature not being disabled > from the command-line. > > Said command-line parsing borrows a lot from the kaslr code, and > subsequently allows the "nokaslr" option to be moved to the new > infrastructure (though it all looks a bit... odd). > > Further patches now add support for disabling BTI and PAuth, the > latter being based on an initial series by Srinivas Ramana[0]. There > is some ongoing discussions about being able to disable MTE, but no > clear resolution on that subject yet. > > This has been tested on multiple VHE and non-VHE systems. > > * From v3 [3]: > - Fixed the VHE_RESTART stub (duh!) > - Switched to using arm64_ftr_safe_value() instead of the user > provided value > - Per-feature override warning > > * From v2 [2]: > - Simplify the VHE_RESTART stub > - Fixed a number of spelling mistakes, and hopefully introduced a > few more > - Override features in __read_sysreg_by_encoding() > - Allow both BTI and PAuth to be overridden on the command line > - Rebased on -rc3 > > * From v1 [1]: > - Fix SPE init on VHE when EL2 doesn't own SPE > - Fix re-init when KASLR is used > - Handle the resume path > - Rebased to 5.11-rc2 > > [0] > https://lore.kernel.org/r/1610152163-16554-1-git-send-email-sram...@codeaurora.org > [1] https://lore.kernel.org/r/20201228104958.1848833-1-...@kernel.org > [2] https://lore.kernel.org/r/20210104135011.2063104-1-...@kernel.org > [3] https://lore.kernel.org/r/2021032811.2455113-1-...@kernel.org Pretty cool! Left a few minor comments here and there, other than that: Acked-by: David Brazdil
Re: [PATCH v4 18/21] arm64: Move "nokaslr" over to the early cpufeature infrastructure
On Mon, Jan 18, 2021 at 09:45:30AM +, Marc Zyngier wrote:
> Given that the early cpufeature infrastructure has borrowed quite
> a lot of code from the kaslr implementation, let's reimplement
> the matching of the "nokaslr" option with it.
>
> Signed-off-by: Marc Zyngier
Acked-by: David Brazdil
> ---
> arch/arm64/kernel/idreg-override.c | 17 ++
> arch/arm64/kernel/kaslr.c | 37 +++---
> 2 files changed, 20 insertions(+), 34 deletions(-)
>
> diff --git a/arch/arm64/kernel/idreg-override.c
> b/arch/arm64/kernel/idreg-override.c
> index 1db54878b2c4..143fe7b8e3ce 100644
> --- a/arch/arm64/kernel/idreg-override.c
> +++ b/arch/arm64/kernel/idreg-override.c
> @@ -33,8 +33,24 @@ static const struct reg_desc mmfr1 __initdata = {
> },
> };
>
> +extern u64 kaslr_feature_val;
> +extern u64 kaslr_feature_mask;
> +
> +static const struct reg_desc kaslr __initdata = {
> + .name = "kaslr",
> +#ifdef CONFIG_RANDOMIZE_BASE
> + .val= _feature_val,
> + .mask = _feature_mask,
> +#endif
> + .fields = {
> + { "disabled", 0 },
> + {}
> + },
> +};
> +
> static const struct reg_desc * const regs[] __initdata = {
> ,
> + ,
> };
>
> static const struct {
> @@ -43,6 +59,7 @@ static const struct {
> } aliases[] __initdata = {
> { "kvm-arm.mode=nvhe", "id_aa64mmfr1.vh=0" },
> { "kvm-arm.mode=protected", "id_aa64mmfr1.vh=0" },
> + { "nokaslr","kaslr.disabled=1" },
> };
>
> static int __init find_field(const char *cmdline, const struct reg_desc *reg,
> diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c
> index 5fc86e7d01a1..dcc4a5aadbe2 100644
> --- a/arch/arm64/kernel/kaslr.c
> +++ b/arch/arm64/kernel/kaslr.c
> @@ -51,39 +51,8 @@ static __init u64 get_kaslr_seed(void *fdt)
> return ret;
> }
>
> -static __init bool cmdline_contains_nokaslr(const u8 *cmdline)
> -{
> - const u8 *str;
> -
> - str = strstr(cmdline, "nokaslr");
> - return str == cmdline || (str > cmdline && *(str - 1) == ' ');
> -}
> -
> -static __init bool is_kaslr_disabled_cmdline(void *fdt)
> -{
> - if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) {
> - int node;
> - const u8 *prop;
> -
> - node = fdt_path_offset(fdt, "/chosen");
> - if (node < 0)
> - goto out;
> -
> - prop = fdt_getprop(fdt, node, "bootargs", NULL);
> - if (!prop)
> - goto out;
> -
> - if (cmdline_contains_nokaslr(prop))
> - return true;
> -
> - if (IS_ENABLED(CONFIG_CMDLINE_EXTEND))
> - goto out;
> -
> - return false;
> - }
> -out:
> - return cmdline_contains_nokaslr(CONFIG_CMDLINE);
> -}
> +u64 kaslr_feature_val __initdata;
> +u64 kaslr_feature_mask __initdata;
>
> /*
> * This routine will be executed with the kernel mapped at its default
> virtual
> @@ -126,7 +95,7 @@ u64 __init kaslr_early_init(void)
>* Check if 'nokaslr' appears on the command line, and
>* return 0 if that is the case.
>*/
> - if (is_kaslr_disabled_cmdline(fdt)) {
> + if (kaslr_feature_val & kaslr_feature_mask & 0xf) {
nit: Isn't the 0xf redundant here? You don't re-mask for VH either.
> kaslr_status = KASLR_DISABLED_CMDLINE;
> return 0;
> }
> --
> 2.29.2
>
Re: [PATCH v4 17/21] KVM: arm64: Document HVC_VHE_RESTART stub hypercall
On Mon, Jan 18, 2021 at 09:45:29AM +, Marc Zyngier wrote:
> For completeness, let's document the HVC_VHE_RESTART stub.
>
> Signed-off-by: Marc Zyngier
Acked-by: David Brazdil
> ---
> Documentation/virt/kvm/arm/hyp-abi.rst | 9 +
> 1 file changed, 9 insertions(+)
>
> diff --git a/Documentation/virt/kvm/arm/hyp-abi.rst
> b/Documentation/virt/kvm/arm/hyp-abi.rst
> index 83cadd8186fa..1ba628baf11b 100644
> --- a/Documentation/virt/kvm/arm/hyp-abi.rst
> +++ b/Documentation/virt/kvm/arm/hyp-abi.rst
> @@ -58,6 +58,15 @@ these functions (see arch/arm{,64}/include/asm/virt.h):
>into place (arm64 only), and jump to the restart address while at HYP/EL2.
>This hypercall is not expected to return to its caller.
>
> +* ::
> +
> +x0 = HVC_VHE_RESTART (arm64 only)
> +
> + Attempt to upgrade the kernel's exception level from EL1 to EL2 by enabling
> + the VHE mode. This is conditioned by the CPU supporting VHE, the EL2 MMU
> + being off, and VHE not being disabled by any other mean (comment line
> option,
'means' (both singular and plural), and 'command line'
> + for example).
> +
> Any other value of r0/x0 triggers a hypervisor-specific handling,
> which is not documented here.
>
> --
> 2.29.2
>
Re: [PATCH v4 15/21] arm64: Add an aliasing facility for the idreg override
On Mon, Jan 18, 2021 at 09:45:27AM +, Marc Zyngier wrote:
> In order to map the override of idregs to options that a user
> can easily understand, let's introduce yet another option
> array, which maps an option to the corresponding idreg options.
>
> Signed-off-by: Marc Zyngier
Acked-by: David Brazdil
> ---
> arch/arm64/kernel/idreg-override.c | 20
> 1 file changed, 20 insertions(+)
>
> diff --git a/arch/arm64/kernel/idreg-override.c
> b/arch/arm64/kernel/idreg-override.c
> index 75d9845f489b..16bc8b3b93ae 100644
> --- a/arch/arm64/kernel/idreg-override.c
> +++ b/arch/arm64/kernel/idreg-override.c
> @@ -37,6 +37,12 @@ static const struct reg_desc * const regs[] __initdata = {
> ,
> };
>
> +static const struct {
> + const char * const alias;
> + const char * const feature;
> +} aliases[] __initdata = {
> +};
> +
> static int __init find_field(const char *cmdline, const struct reg_desc *reg,
>int f, u64 *v)
> {
> @@ -80,6 +86,18 @@ static void __init match_options(const char *cmdline)
> }
> }
>
> +static __init void match_aliases(const char *cmdline)
> +{
> + int i;
> +
> + for (i = 0; i < ARRAY_SIZE(aliases); i++) {
> + char *str = strstr(cmdline, aliases[i].alias);
> +
> + if ((str == cmdline || (str > cmdline && *(str - 1) == ' ')))
nit: Extract to a 'cmdline_contains' helper? Took me a good few seconds to
parse this in the previous patch. Giving it a name would help, and now it's
also shared.
> + match_options(aliases[i].feature);
> + }
> +}
> +
> static __init void parse_cmdline(void)
> {
> if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) {
> @@ -100,6 +118,7 @@ static __init void parse_cmdline(void)
> goto out;
>
> match_options(prop);
> + match_aliases(prop);
>
> if (!IS_ENABLED(CONFIG_CMDLINE_EXTEND))
> return;
> @@ -107,6 +126,7 @@ static __init void parse_cmdline(void)
>
> out:
> match_options(CONFIG_CMDLINE);
> + match_aliases(CONFIG_CMDLINE);
> }
>
> void __init init_shadow_regs(void)
> --
> 2.29.2
>
Re: [PATCH v4 14/21] arm64: Honor VHE being disabled from the command-line
On Mon, Jan 18, 2021 at 09:45:26AM +, Marc Zyngier wrote: > Finally we can check whether VHE is disabled on the command line, > and not enable it if that's the user's wish. > > Signed-off-by: Marc Zyngier Acked-by: David Brazdil > --- > arch/arm64/kernel/hyp-stub.S | 17 ++--- > 1 file changed, 14 insertions(+), 3 deletions(-) > > diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S > index 59820f9b8522..bbab2148a2a2 100644 > --- a/arch/arm64/kernel/hyp-stub.S > +++ b/arch/arm64/kernel/hyp-stub.S > @@ -77,13 +77,24 @@ SYM_CODE_END(el1_sync) > SYM_CODE_START_LOCAL(mutate_to_vhe) > // Sanity check: MMU *must* be off > mrs x0, sctlr_el2 > - tbnzx0, #0, 1f > + tbnzx0, #0, 2f > > // Needs to be VHE capable, obviously > mrs x0, id_aa64mmfr1_el1 > ubfxx0, x0, #ID_AA64MMFR1_VHE_SHIFT, #4 > - cbz x0, 1f > + cbz x0, 2f > > + // Check whether VHE is disabled from the command line > + adr_l x1, id_aa64mmfr1_val > + ldr x0, [x1] > + adr_l x1, id_aa64mmfr1_mask > + ldr x1, [x1] super nit: There's a ldr_l macro > + ubfxx0, x0, #ID_AA64MMFR1_VHE_SHIFT, #4 > + ubfxx1, x1, #ID_AA64MMFR1_VHE_SHIFT, #4 > + cbz x1, 1f > + and x0, x0, x1 > + cbz x0, 2f > +1: > // Engage the VHE magic! > mov_q x0, HCR_HOST_VHE_FLAGS > msr hcr_el2, x0 > @@ -152,7 +163,7 @@ skip_spe: > orr x0, x0, x1 > msr spsr_el1, x0 > > -1: eret > +2: eret > SYM_CODE_END(mutate_to_vhe) > > .macro invalid_vectorlabel > -- > 2.29.2 >
Re: [PATCH v4 12/21] arm64: cpufeature: Add an early command-line cpufeature override facility
On Mon, Jan 18, 2021 at 09:45:24AM +, Marc Zyngier wrote:
> In order to be able to override CPU features at boot time,
> let's add a command line parser that matches options of the
> form "cpureg.feature=value", and store the corresponding
> value into the override val/mask pair.
>
> No features are currently defined, so no expected change in
> functionality.
>
> Signed-off-by: Marc Zyngier
Acked-by: David Brazdil
> ---
> arch/arm64/kernel/Makefile | 2 +-
> arch/arm64/kernel/head.S | 1 +
> arch/arm64/kernel/idreg-override.c | 119 +
> 3 files changed, 121 insertions(+), 1 deletion(-)
> create mode 100644 arch/arm64/kernel/idreg-override.c
>
> diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile
> index 86364ab6f13f..2262f0392857 100644
> --- a/arch/arm64/kernel/Makefile
> +++ b/arch/arm64/kernel/Makefile
> @@ -17,7 +17,7 @@ obj-y := debug-monitors.o entry.o
> irq.o fpsimd.o \
> return_address.o cpuinfo.o cpu_errata.o
> \
> cpufeature.o alternative.o cacheinfo.o
> \
> smp.o smp_spin_table.o topology.o smccc-call.o
> \
> -syscall.o proton-pack.o
> +syscall.o proton-pack.o idreg-override.o
>
> targets += efi-entry.o
>
> diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
> index d74e5f84042e..b3c4dd04f74b 100644
> --- a/arch/arm64/kernel/head.S
> +++ b/arch/arm64/kernel/head.S
> @@ -435,6 +435,7 @@ SYM_FUNC_START_LOCAL(__primary_switched)
>
> mov x0, x21 // pass FDT address in x0
> bl early_fdt_map // Try mapping the FDT early
> + bl init_shadow_regs
> bl switch_to_vhe
> #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
> bl kasan_early_init
> diff --git a/arch/arm64/kernel/idreg-override.c
> b/arch/arm64/kernel/idreg-override.c
> new file mode 100644
> index ..392f93b67103
> --- /dev/null
> +++ b/arch/arm64/kernel/idreg-override.c
> @@ -0,0 +1,119 @@
> +// SPDX-License-Identifier: GPL-2.0
> +/*
> + * Early cpufeature override framework
> + *
> + * Copyright (C) 2020 Google LLC
> + * Author: Marc Zyngier
> + */
> +
> +#include
> +#include
> +
> +#include
> +#include
> +
> +struct reg_desc {
> + const char * const name;
> + u64 * const val;
> + u64 * const mask;
> + struct {
> + const char * const name;
> + u8 shift;
nit: There's an extra space before `shift`.
> + } fields[];
> +};
> +
> +static const struct reg_desc * const regs[] __initdata = {
> +};
> +
> +static int __init find_field(const char *cmdline, const struct reg_desc *reg,
> + int f, u64 *v)
> +{
> + char buf[256], *str;
> + size_t len;
> +
> + snprintf(buf, ARRAY_SIZE(buf), "%s.%s=", reg->name,
> reg->fields[f].name);
> +
> + str = strstr(cmdline, buf);
> + if (!(str == cmdline || (str > cmdline && *(str - 1) == ' ')))
> + return -1;
> +
> + str += strlen(buf);
> + len = strcspn(str, " ");
> + len = min(len, ARRAY_SIZE(buf) - 1);
> + strncpy(buf, str, len);
> + buf[len] = 0;
> +
> + return kstrtou64(buf, 0, v);
> +}
> +
> +static void __init match_options(const char *cmdline)
> +{
> + int i;
> +
> + for (i = 0; i < ARRAY_SIZE(regs); i++) {
> + int f;
> +
> + if (!regs[i]->val || !regs[i]->mask)
> + continue;
> +
> + for (f = 0; regs[i]->fields[f].name; f++) {
> + u64 v;
> +
> + if (find_field(cmdline, regs[i], f, ))
> + continue;
> +
> + *regs[i]->val |= (v & 0xf) << regs[i]->fields[f].shift;
> + *regs[i]->mask |= 0xfUL << regs[i]->fields[f].shift;
> + }
> + }
> +}
> +
> +static __init void parse_cmdline(void)
> +{
> + if (!IS_ENABLED(CONFIG_CMDLINE_FORCE)) {
> + const u8 *prop;
> + void *fdt;
> + int node;
> +
> + fdt = get_early_fdt_ptr();
> + if (!fdt)
> + goto out;
> +
> + node = fdt_path_offset(fdt, "/chosen");
Re: [PATCH v4 04/21] arm64: Provide an 'upgrade to VHE' stub hypercall
On Mon, Jan 18, 2021 at 09:45:16AM +, Marc Zyngier wrote: > As we are about to change the way a VHE system boots, let's > provide the core helper, in the form of a stub hypercall that > enables VHE and replicates the full EL1 context at EL2, thanks > to EL1 and VHE-EL2 being extremely similar. > > On exception return, the kernel carries on at EL2. Fancy! > > Nothing calls this new hypercall yet, so no functional change. > > Signed-off-by: Marc Zyngier > --- > arch/arm64/include/asm/virt.h | 7 +++- > arch/arm64/kernel/hyp-stub.S | 67 +-- > 2 files changed, 71 insertions(+), 3 deletions(-) > > diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h > index ee6a48df89d9..7379f35ae2c6 100644 > --- a/arch/arm64/include/asm/virt.h > +++ b/arch/arm64/include/asm/virt.h > @@ -35,8 +35,13 @@ > */ > #define HVC_RESET_VECTORS 2 > > +/* > + * HVC_VHE_RESTART - Upgrade the CPU from EL1 to EL2, if possible > + */ > +#define HVC_VHE_RESTART 3 > + > /* Max number of HYP stub hypercalls */ > -#define HVC_STUB_HCALL_NR 3 > +#define HVC_STUB_HCALL_NR 4 > > /* Error returned when an invalid stub number is passed into x0 */ > #define HVC_STUB_ERR 0xbadca11 > diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S > index 160f5881a0b7..fb12398b5c28 100644 > --- a/arch/arm64/kernel/hyp-stub.S > +++ b/arch/arm64/kernel/hyp-stub.S > @@ -8,9 +8,9 @@ > > #include > #include > -#include > > #include > +#include > #include > #include > #include > @@ -47,10 +47,13 @@ SYM_CODE_END(__hyp_stub_vectors) > > SYM_CODE_START_LOCAL(el1_sync) > cmp x0, #HVC_SET_VECTORS > - b.ne2f > + b.ne1f > msr vbar_el2, x1 > b 9f > > +1: cmp x0, #HVC_VHE_RESTART > + b.eqmutate_to_vhe > + > 2: cmp x0, #HVC_SOFT_RESTART > b.ne3f > mov x0, x2 > @@ -70,6 +73,66 @@ SYM_CODE_START_LOCAL(el1_sync) > eret > SYM_CODE_END(el1_sync) > > +// nVHE? No way! Give me the real thing! > +SYM_CODE_START_LOCAL(mutate_to_vhe) > + // Sanity check: MMU *must* be off > + mrs x0, sctlr_el2 > + tbnzx0, #0, 1f > + > + // Needs to be VHE capable, obviously > + mrs x0, id_aa64mmfr1_el1 > + ubfxx0, x0, #ID_AA64MMFR1_VHE_SHIFT, #4 > + cbz x0, 1f nit: There is a HVC_STUB_ERR that you could return if these sanity checks fail. The documentation also states that it should be returned on error. > + > + // Engage the VHE magic! > + mov_q x0, HCR_HOST_VHE_FLAGS > + msr hcr_el2, x0 > + isb > + > + // Doesn't do much on VHE, but still, worth a shot > + init_el2_state vhe > + > + // Use the EL1 allocated stack, per-cpu offset > + mrs x0, sp_el1 > + mov sp, x0 > + mrs x0, tpidr_el1 > + msr tpidr_el2, x0 > + > + // FP configuration, vectors > + mrs_s x0, SYS_CPACR_EL12 > + msr cpacr_el1, x0 > + mrs_s x0, SYS_VBAR_EL12 > + msr vbar_el1, x0 > + > + // Transfer the MM state from EL1 to EL2 > + mrs_s x0, SYS_TCR_EL12 > + msr tcr_el1, x0 > + mrs_s x0, SYS_TTBR0_EL12 > + msr ttbr0_el1, x0 > + mrs_s x0, SYS_TTBR1_EL12 > + msr ttbr1_el1, x0 > + mrs_s x0, SYS_MAIR_EL12 > + msr mair_el1, x0 > + isb > + > + // Invalidate TLBs before enabling the MMU > + tlbivmalle1 > + dsb nsh > + > + // Enable the EL2 S1 MMU, as set up from EL1 > + mrs_s x0, SYS_SCTLR_EL12 > + set_sctlr_el1 x0 > + > + // Hack the exception return to stay at EL2 > + mrs x0, spsr_el1 > + and x0, x0, #~PSR_MODE_MASK > + mov x1, #PSR_MODE_EL2h > + orr x0, x0, x1 > + msr spsr_el1, x0 > + > +1: eret > +SYM_CODE_END(mutate_to_vhe) > + > .macro invalid_vectorlabel > SYM_CODE_START_LOCAL(\label) > b \label > -- > 2.29.2 >
Re: [PATCH v4 01/21] arm64: Fix labels in el2_setup macros
On Mon, Jan 18, 2021 at 09:45:13AM +, Marc Zyngier wrote: > If someone happens to write the following code: > > b 1f > init_el2_state vhe > 1: > [...] > > they will be in for a long debugging session, as the label "1f" > will be resolved *inside* the init_el2_state macro instead of > after it. Not really what one expects. > > Instead, rewite the EL2 setup macros to use unambiguous labels, > thanks to the usual macro counter trick. > > Acked-by: Catalin Marinas > Signed-off-by: Marc Zyngier > --- > arch/arm64/include/asm/el2_setup.h | 24 > 1 file changed, 12 insertions(+), 12 deletions(-) Acked-by: David Brazdil
Re: [PATCH v3 3/9] KVM: arm64: Enable UBSAN_BOUNDS for the both the kernel and hyp/nVHE
On Fri, Jan 15, 2021 at 05:18:24PM +, Elena Petrova wrote:
> From: George Popescu
>
> If an out of bounds happens inside the hyp/nVHE code, the ubsan_out_of_bounds
> handler stores the logging data inside the kvm_ubsan_buffer. The one
> responsible
> for printing is the kernel ubsan_out_of_bounds handler. The process of
> decapsulating the data from the buffer is straightforward.
>
> Signed-off-by: George Popescu
> Signed-off-by: Elena Petrova
> ---
> arch/arm64/include/asm/kvm_ubsan.h | 19 ++-
> arch/arm64/kvm/hyp/nvhe/ubsan.c| 14 --
> arch/arm64/kvm/kvm_ubsan_buffer.c | 10 ++
> 3 files changed, 40 insertions(+), 3 deletions(-)
>
> diff --git a/arch/arm64/include/asm/kvm_ubsan.h
> b/arch/arm64/include/asm/kvm_ubsan.h
> index fb32c7fd65d4..4f471acb88b0 100644
> --- a/arch/arm64/include/asm/kvm_ubsan.h
> +++ b/arch/arm64/include/asm/kvm_ubsan.h
> @@ -9,6 +9,23 @@
> #define UBSAN_MAX_TYPE 6
> #define KVM_UBSAN_BUFFER_SIZE 1000
>
> +
> +struct ubsan_values {
> + void *lval;
> + void *rval;
> + char op;
> +};
> +
> struct kvm_ubsan_info {
> - int type;
> + enum {
> + UBSAN_OUT_OF_BOUNDS,
> + } type;
> + union {
> + struct out_of_bounds_data out_of_bounds_data;
> + };
> + union {
> + struct ubsan_values u_val;
> + };
I don't see this second union having more members later in the series.
Remove it? Even the 'struct ubsan_values' seems redundant and we could just
have those three fields directly here.
> };
> +
> +void __ubsan_handle_out_of_bounds(void *_data, void *index);
> diff --git a/arch/arm64/kvm/hyp/nvhe/ubsan.c b/arch/arm64/kvm/hyp/nvhe/ubsan.c
> index 8a194fb1f6cf..55a8f6db8555 100644
> --- a/arch/arm64/kvm/hyp/nvhe/ubsan.c
> +++ b/arch/arm64/kvm/hyp/nvhe/ubsan.c
> @@ -13,7 +13,6 @@
> #include
> #include
> #include
> -#include
>
> DEFINE_KVM_DEBUG_BUFFER(struct kvm_ubsan_info, kvm_ubsan_buffer,
> kvm_ubsan_buff_wr_ind, KVM_UBSAN_BUFFER_SIZE);
> @@ -44,7 +43,18 @@ void __ubsan_handle_type_mismatch(struct
> type_mismatch_data *data, void *ptr) {}
>
> void __ubsan_handle_type_mismatch_v1(void *_data, void *ptr) {}
>
> -void __ubsan_handle_out_of_bounds(void *_data, void *index) {}
> +void __ubsan_handle_out_of_bounds(void *_data, void *index)
> +{
> + struct kvm_ubsan_info *slot;
> + struct out_of_bounds_data *data = _data;
> +
> + slot = kvm_ubsan_buffer_next_slot();
> + if (slot) {
> + slot->type = UBSAN_OUT_OF_BOUNDS;
> + slot->out_of_bounds_data = *data;
> + slot->u_val.lval = index;
> + }
> +}
>
> void __ubsan_handle_shift_out_of_bounds(void *_data, void *lhs, void *rhs) {}
>
> diff --git a/arch/arm64/kvm/kvm_ubsan_buffer.c
> b/arch/arm64/kvm/kvm_ubsan_buffer.c
> index 4a1959ba9f68..a1523f86be3c 100644
> --- a/arch/arm64/kvm/kvm_ubsan_buffer.c
> +++ b/arch/arm64/kvm/kvm_ubsan_buffer.c
> @@ -17,6 +17,15 @@
> DECLARE_KVM_DEBUG_BUFFER(struct kvm_ubsan_info, kvm_ubsan_buffer,
> kvm_ubsan_buff_wr_ind, KVM_UBSAN_BUFFER_SIZE);
>
> +void __kvm_check_ubsan_data(struct kvm_ubsan_info *slot)
> +{
> + switch (slot->type) {
> + case UBSAN_OUT_OF_BOUNDS:
> + __ubsan_handle_out_of_bounds(>out_of_bounds_data,
> + slot->u_val.lval);
> + break;
> + }
> +}
>
> void iterate_kvm_ubsan_buffer(unsigned long left, unsigned long right)
> {
> @@ -26,6 +35,7 @@ void iterate_kvm_ubsan_buffer(unsigned long left, unsigned
> long right)
> slot = (struct kvm_ubsan_info *)
> this_cpu_ptr_nvhe_sym(kvm_ubsan_buffer);
> for (i = left; i < right; ++i) {
> /* check ubsan data */
> + __kvm_check_ubsan_data(slot + i);
Not sure why this is breaking out into another function. The code will not
be shared with any other user.
> slot[i].type = 0;
This invalidation is redundant. The buffer's cursor will be reset on next
hypercall, which will implicitly invalidate all entries.
> }
> }
> --
> 2.30.0.296.g2bfb1c46d8-goog
>
> ___
> kvmarm mailing list
> kvm...@lists.cs.columbia.edu
> https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH v3 1/9] KVM: arm64: Enable UBSan instrumentation in nVHE hyp code
On Fri, Jan 15, 2021 at 05:18:22PM +, Elena Petrova wrote:
> From: George-Aurelian Popescu
>
> Implement UBSan handlers inside nVHe hyp code, as empty functions for the
> moment, so the undefined behaviours, that are triggered there, will be
> linked to them, not to the ones defined in kernel-proper lib/ubsan.c.
>
> In this way, enabling UBSAN_MISC won't cause a link error.
The commit message needs to be updated - UBSAN_MISC does not exist any longer.
Also "nVHe" is a typo.
Other than that:
Acked-by: David Brazdil
>
> Signed-off-by: Elena Petrova
> ---
> arch/arm64/kvm/hyp/nvhe/Makefile | 3 ++-
> arch/arm64/kvm/hyp/nvhe/ubsan.c | 30 ++
> 2 files changed, 32 insertions(+), 1 deletion(-)
> create mode 100644 arch/arm64/kvm/hyp/nvhe/ubsan.c
>
> diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile
> b/arch/arm64/kvm/hyp/nvhe/Makefile
> index 1f1e351c5fe2..2a683e7c6c5b 100644
> --- a/arch/arm64/kvm/hyp/nvhe/Makefile
> +++ b/arch/arm64/kvm/hyp/nvhe/Makefile
> @@ -10,6 +10,8 @@ obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o
> hyp-init.o host.o \
>hyp-main.o hyp-smp.o psci-relay.o
> obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
>../fpsimd.o ../hyp-entry.o ../exception.o
> +obj-$(CONFIG_UBSAN) += ubsan.o
> +CFLAGS_ubsan.nvhe.o += -I $(srctree)/lib/
>
> ##
> ## Build rules for compiling nVHE hyp code
> @@ -61,7 +63,6 @@ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_FTRACE)
> $(CC_FLAGS_SCS), $(KBUILD_CFLAG
> # cause crashes. Just disable it.
> GCOV_PROFILE := n
> KASAN_SANITIZE := n
> -UBSAN_SANITIZE := n
> KCOV_INSTRUMENT := n
>
> # Skip objtool checking for this directory because nVHE code is compiled with
> diff --git a/arch/arm64/kvm/hyp/nvhe/ubsan.c b/arch/arm64/kvm/hyp/nvhe/ubsan.c
> new file mode 100644
> index ..a5db6b61ceb2
> --- /dev/null
> +++ b/arch/arm64/kvm/hyp/nvhe/ubsan.c
> @@ -0,0 +1,30 @@
> +// SPDX-License-Identifier: GPL-2.0-only
> +/*
> + * Copyright 2020 Google LLC
> + * Author: George Popescu
> + */
> +#include
> +#include
> +#include
> +
> +void __ubsan_handle_add_overflow(void *_data, void *lhs, void *rhs) {}
> +
> +void __ubsan_handle_sub_overflow(void *_data, void *lhs, void *rhs) {}
> +
> +void __ubsan_handle_mul_overflow(void *_data, void *lhs, void *rhs) {}
> +
> +void __ubsan_handle_negate_overflow(void *_data, void *old_val) {}
> +
> +void __ubsan_handle_divrem_overflow(void *_data, void *lhs, void *rhs) {}
> +
> +void __ubsan_handle_type_mismatch(struct type_mismatch_data *data, void
> *ptr) {}
> +
> +void __ubsan_handle_type_mismatch_v1(void *_data, void *ptr) {}
> +
> +void __ubsan_handle_out_of_bounds(void *_data, void *index) {}
> +
> +void __ubsan_handle_shift_out_of_bounds(void *_data, void *lhs, void *rhs) {}
> +
> +void __ubsan_handle_builtin_unreachable(void *_data) {}
> +
> +void __ubsan_handle_load_invalid_value(void *_data, void *val) {}
> --
> 2.30.0.296.g2bfb1c46d8-goog
>
> ___
> kvmarm mailing list
> kvm...@lists.cs.columbia.edu
> https://lists.cs.columbia.edu/mailman/listinfo/kvmarm
Re: [PATCH] arm64: Remove unused variable in arch_show_interrupts
On Tue, Jan 05, 2021 at 06:08:35PM +, Catalin Marinas wrote:
> On Tue, Jan 05, 2021 at 06:03:14PM +0000, David Brazdil wrote:
> > Commit 5089bc51f81f ("arm64/smp: Use irq_desc_kstat_cpu() in
> > arch_show_interrupts()") removed the only user of variable `irq`.
> > Remove the unused variable.
> >
> > Signed-off-by: David Brazdil
>
> Please search the list for similar patches. I can count 5-6 and the
> number is growing ;) (and I queued one already).
Sorry, my bad. Need to get my email client out of the kvmarm list once in a
while. O:)
[PATCH v2 6/8] KVM: arm64: Fix constant-pool users in hyp
Hyp code uses absolute addressing to obtain a kimg VA of a small number of kernel symbols. Since the kernel now converts constant pool addresses to hyp VAs, this trick does not work anymore. Change the helpers to convert from hyp VA back to kimg VA or PA, as needed and rework the callers accordingly. Signed-off-by: David Brazdil --- arch/arm64/include/asm/kvm_mmu.h | 42 -- arch/arm64/kvm/hyp/nvhe/host.S | 29 +++-- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 2 -- 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 6bbb44011c84..adadc468cc71 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -73,49 +73,39 @@ alternative_cb_end .endm /* - * Convert a kernel image address to a PA - * reg: kernel address to be converted in place + * Convert a hypervisor VA to a PA + * reg: hypervisor address to be converted in place * tmp: temporary register - * - * The actual code generation takes place in kvm_get_kimage_voffset, and - * the instructions below are only there to reserve the space and - * perform the register allocation (kvm_get_kimage_voffset uses the - * specific registers encoded in the instructions). */ -.macro kimg_pa reg, tmp -alternative_cb kvm_get_kimage_voffset - movz\tmp, #0 - movk\tmp, #0, lsl #16 - movk\tmp, #0, lsl #32 - movk\tmp, #0, lsl #48 -alternative_cb_end - - /* reg = __pa(reg) */ - sub \reg, \reg, \tmp +.macro hyp_pa reg, tmp + ldr_l \tmp, hyp_physvirt_offset + add \reg, \reg, \tmp .endm /* - * Convert a kernel image address to a hyp VA - * reg: kernel address to be converted in place + * Convert a hypervisor VA to a kernel image address + * reg: hypervisor address to be converted in place * tmp: temporary register * * The actual code generation takes place in kvm_get_kimage_voffset, and * the instructions below are only there to reserve the space and - * perform the register allocation (kvm_update_kimg_phys_offset uses the + * perform the register allocation (kvm_get_kimage_voffset uses the * specific registers encoded in the instructions). */ -.macro kimg_hyp_va reg, tmp -alternative_cb kvm_update_kimg_phys_offset +.macro hyp_kimg_va reg, tmp + /* Convert hyp VA -> PA. */ + hyp_pa \reg, \tmp + + /* Load kimage_voffset. */ +alternative_cb kvm_get_kimage_voffset movz\tmp, #0 movk\tmp, #0, lsl #16 movk\tmp, #0, lsl #32 movk\tmp, #0, lsl #48 alternative_cb_end - sub \reg, \reg, \tmp - mov_q \tmp, PAGE_OFFSET - orr \reg, \reg, \tmp - kern_hyp_va \reg + /* Convert PA -> kimg VA. */ + add \reg, \reg, \tmp .endm #else diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index a820dfdc9c25..6585a7cbbc56 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -74,27 +74,28 @@ SYM_FUNC_END(__host_enter) * void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par); */ SYM_FUNC_START(__hyp_do_panic) - /* Load the format arguments into x1-7 */ - mov x6, x3 - get_vcpu_ptr x7, x3 - - mrs x3, esr_el2 - mrs x4, far_el2 - mrs x5, hpfar_el2 - /* Prepare and exit to the host's panic funciton. */ mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ PSR_MODE_EL1h) msr spsr_el2, lr ldr lr, =panic + hyp_kimg_va lr, x6 msr elr_el2, lr - /* -* Set the panic format string and enter the host, conditionally -* restoring the host context. -*/ + /* Set the panic format string. Use the, now free, LR as scratch. */ + ldr lr, =__hyp_panic_string + hyp_kimg_va lr, x6 + + /* Load the format arguments into x1-7. */ + mov x6, x3 + get_vcpu_ptr x7, x3 + mrs x3, esr_el2 + mrs x4, far_el2 + mrs x5, hpfar_el2 + + /* Enter the host, conditionally restoring the host context. */ cmp x0, xzr - ldr x0, =__hyp_panic_string + mov x0, lr b.eq__host_enter_without_restoring b __host_enter_for_panic SYM_FUNC_END(__hyp_do_panic) @@ -124,7 +125,7 @@ SYM_FUNC_END(__hyp_do_panic) * Preserve x0-x4, which may contain stub parameters. */ ldr x5, =__kvm_handle_stub_hvc - kimg_pa x5, x6 + hyp_pa x5, x6 br x5 .L__vect_end\@: .if ((.L__vect_end\@ - .L__vect_start\@) > 0x80) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 68fd64f2313e..99b408fe09ee 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -139,7 +139,6 @@ alternative_else_
[PATCH v2 8/8] KVM: arm64: Remove hyp_symbol_addr
Hyp code used the hyp_symbol_addr helper to force PC-relative addressing
because absolute addressing results in kernel VAs due to the way hyp
code is linked. This is not true anymore, so remove the helper and
update all of its users.
Acked-by: Ard Biesheuvel
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_asm.h | 26
arch/arm64/kvm/hyp/include/hyp/switch.h | 4 ++--
arch/arm64/kvm/hyp/nvhe/hyp-smp.c| 4 ++--
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 24 +++---
arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c | 2 +-
5 files changed, 17 insertions(+), 43 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 8a33d83ea843..22d933e9b59e 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -199,32 +199,6 @@ extern void __vgic_v3_init_lrs(void);
extern u32 __kvm_get_mdcr_el2(void);
-#if defined(GCC_VERSION) && GCC_VERSION < 5
-#define SYM_CONSTRAINT "i"
-#else
-#define SYM_CONSTRAINT "S"
-#endif
-
-/*
- * Obtain the PC-relative address of a kernel symbol
- * s: symbol
- *
- * The goal of this macro is to return a symbol's address based on a
- * PC-relative computation, as opposed to a loading the VA from a
- * constant pool or something similar. This works well for HYP, as an
- * absolute VA is guaranteed to be wrong. Only use this if trying to
- * obtain the address of a symbol (i.e. not something you obtained by
- * following a pointer).
- */
-#define hyp_symbol_addr(s) \
- ({ \
- typeof(s) *addr;\
- asm("adrp %0, %1\n" \
- "add%0, %0, :lo12:%1\n" \
- : "=r" (addr) : SYM_CONSTRAINT ()); \
- addr; \
- })
-
#define __KVM_EXTABLE(from, to)
\
" .pushsection__kvm_ex_table, \"a\"\n"\
" .align 3\n"\
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h
b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 84473574c2e7..54f4860cd87c 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -505,8 +505,8 @@ static inline void __kvm_unexpected_el2_exception(void)
struct exception_table_entry *entry, *end;
unsigned long elr_el2 = read_sysreg(elr_el2);
- entry = hyp_symbol_addr(__start___kvm_ex_table);
- end = hyp_symbol_addr(__stop___kvm_ex_table);
+ entry = &__start___kvm_ex_table;
+ end = &__stop___kvm_ex_table;
while (entry < end) {
addr = (unsigned long)>insn + entry->insn;
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
index 2997aa156d8e..879559057dee 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
@@ -33,8 +33,8 @@ unsigned long __hyp_per_cpu_offset(unsigned int cpu)
if (cpu >= ARRAY_SIZE(kvm_arm_hyp_percpu_base))
hyp_panic();
- cpu_base_array = (unsigned long
*)hyp_symbol_addr(kvm_arm_hyp_percpu_base);
+ cpu_base_array = (unsigned long *)_arm_hyp_percpu_base;
this_cpu_base = kern_hyp_va(cpu_base_array[cpu]);
- elf_base = (unsigned long)hyp_symbol_addr(__per_cpu_start);
+ elf_base = (unsigned long)&__per_cpu_start;
return this_cpu_base - elf_base;
}
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index e3947846ffcb..f254a425cb3a 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -134,8 +134,8 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context
*host_ctxt)
if (cpu_id == INVALID_CPU_ID)
return PSCI_RET_INVALID_PARAMS;
- boot_args = per_cpu_ptr(hyp_symbol_addr(cpu_on_args), cpu_id);
- init_params = per_cpu_ptr(hyp_symbol_addr(kvm_init_params), cpu_id);
+ boot_args = per_cpu_ptr(_on_args, cpu_id);
+ init_params = per_cpu_ptr(_init_params, cpu_id);
/* Check if the target CPU is already being booted. */
if (!try_acquire_boot_args(boot_args))
@@ -146,7 +146,7 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context
*host_ctxt)
wmb();
ret = psci_call(func_id, mpidr,
- __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_entry)),
+ __hyp_pa(_hyp_cpu_entry),
__hyp_pa(init_params));
/* If successful, the lock will be released by the target CPU. */
@@ -165,8 +
[PATCH v2 5/8] KVM: arm64: Apply hyp relocations at runtime
KVM nVHE code runs under a different VA mapping than the kernel, hence
so far it avoided using absolute addressing because the VA in a constant
pool is relocated by the linker to a kernel VA (see hyp_symbol_addr).
Now the kernel has access to a list of positions that contain a kimg VA
but will be accessed only in hyp execution context. These are generated
by the gen-hyprel build-time tool and stored in .hyp.reloc.
Add early boot pass over the entries and convert the kimg VAs to hyp VAs.
Note that this requires for .hyp* ELF sections to be mapped read-write
at that point.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_mmu.h | 1 +
arch/arm64/include/asm/sections.h | 1 +
arch/arm64/kernel/smp.c | 4 +++-
arch/arm64/kvm/va_layout.c| 28
4 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index e52d82aeadca..6bbb44011c84 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -129,6 +129,7 @@ alternative_cb_end
void kvm_update_va_mask(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst);
void kvm_compute_layout(void);
+void kvm_apply_hyp_relocations(void);
static __always_inline unsigned long __kern_hyp_va(unsigned long v)
{
diff --git a/arch/arm64/include/asm/sections.h
b/arch/arm64/include/asm/sections.h
index a6f3557d1ab2..2f36b16a5b5d 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -12,6 +12,7 @@ extern char __hibernate_exit_text_start[],
__hibernate_exit_text_end[];
extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
extern char __hyp_text_start[], __hyp_text_end[];
extern char __hyp_rodata_start[], __hyp_rodata_end[];
+extern char __hyp_reloc_begin[], __hyp_reloc_end[];
extern char __idmap_text_start[], __idmap_text_end[];
extern char __initdata_begin[], __initdata_end[];
extern char __inittext_begin[], __inittext_end[];
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index d08948c6979b..006f61a86438 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -434,8 +434,10 @@ static void __init hyp_mode_check(void)
"CPU: CPUs started in inconsistent modes");
else
pr_info("CPU: All CPU(s) started at EL1\n");
- if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode())
+ if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode()) {
kvm_compute_layout();
+ kvm_apply_hyp_relocations();
+ }
}
void __init smp_cpus_done(unsigned int max_cpus)
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index 70fcd6a12fe1..fee7dcd95d73 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -81,6 +81,34 @@ __init void kvm_compute_layout(void)
init_hyp_physvirt_offset();
}
+/*
+ * The .hyp.reloc ELF section contains a list of kimg positions that
+ * contains kimg VAs but will be accessed only in hyp execution context.
+ * Convert them to hyp VAs. See gen-hyprel.c for more details.
+ */
+__init void kvm_apply_hyp_relocations(void)
+{
+ int32_t *rel;
+ int32_t *begin = (int32_t *)__hyp_reloc_begin;
+ int32_t *end = (int32_t *)__hyp_reloc_end;
+
+ for (rel = begin; rel < end; ++rel) {
+ uintptr_t *ptr, kimg_va;
+
+ /*
+* Each entry contains a 32-bit relative offset from itself
+* to a kimg VA position.
+*/
+ ptr = (uintptr_t *)lm_alias((char *)rel + *rel);
+
+ /* Read the kimg VA value at the relocation address. */
+ kimg_va = *ptr;
+
+ /* Convert to hyp VA and store back to the relocation address.
*/
+ *ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));
+ }
+}
+
static u32 compute_instruction(int n, u32 rd, u32 rn)
{
u32 insn = AARCH64_BREAK_FAULT;
--
2.29.2.729.g45daf8777d-goog
[PATCH v2 7/8] KVM: arm64: Remove patching of fn pointers in hyp
Storing a function pointer in hyp now generates relocation information
used at early boot to convert the address to hyp VA. The existing
alternative-based conversion mechanism is therefore obsolete. Remove it
and simplify its users.
Acked-by: Ard Biesheuvel
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_mmu.h | 18 --
arch/arm64/kernel/image-vars.h | 1 -
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 11 ---
arch/arm64/kvm/va_layout.c | 6 --
4 files changed, 4 insertions(+), 32 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index adadc468cc71..90873851f677 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -135,24 +135,6 @@ static __always_inline unsigned long
__kern_hyp_va(unsigned long v)
#define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v
-static __always_inline unsigned long __kimg_hyp_va(unsigned long v)
-{
- unsigned long offset;
-
- asm volatile(ALTERNATIVE_CB("movz %0, #0\n"
- "movk %0, #0, lsl #16\n"
- "movk %0, #0, lsl #32\n"
- "movk %0, #0, lsl #48\n",
- kvm_update_kimg_phys_offset)
-: "=r" (offset));
-
- return __kern_hyp_va((v - offset) | PAGE_OFFSET);
-}
-
-#define kimg_fn_hyp_va(v) ((typeof(*v))(__kimg_hyp_va((unsigned
long)(v
-
-#define kimg_fn_ptr(x) (typeof(x) **)(x)
-
/*
* We currently support using a VM-specified IPA size. For backward
* compatibility, the default IPA size is fixed to 40bits.
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index f676243abac6..23f1a557bd9f 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -64,7 +64,6 @@ __efistub__ctype = _ctype;
/* Alternative callbacks for init-time patching of nVHE hyp code. */
KVM_NVHE_ALIAS(kvm_patch_vector_branch);
KVM_NVHE_ALIAS(kvm_update_va_mask);
-KVM_NVHE_ALIAS(kvm_update_kimg_phys_offset);
KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
/* Global kernel state accessed by nVHE hyp code. */
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index a906f9e2ff34..f012f8665ecc 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -108,9 +108,9 @@ static void handle___vgic_v3_restore_aprs(struct
kvm_cpu_context *host_ctxt)
typedef void (*hcall_t)(struct kvm_cpu_context *);
-#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = kimg_fn_ptr(handle_##x)
+#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
-static const hcall_t *host_hcall[] = {
+static const hcall_t host_hcall[] = {
HANDLE_FUNC(__kvm_vcpu_run),
HANDLE_FUNC(__kvm_flush_vm_context),
HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa),
@@ -130,7 +130,6 @@ static const hcall_t *host_hcall[] = {
static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(unsigned long, id, host_ctxt, 0);
- const hcall_t *kfn;
hcall_t hfn;
id -= KVM_HOST_SMCCC_ID(0);
@@ -138,13 +137,11 @@ static void handle_host_hcall(struct kvm_cpu_context
*host_ctxt)
if (unlikely(id >= ARRAY_SIZE(host_hcall)))
goto inval;
- kfn = host_hcall[id];
- if (unlikely(!kfn))
+ hfn = host_hcall[id];
+ if (unlikely(!hfn))
goto inval;
cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS;
-
- hfn = kimg_fn_hyp_va(kfn);
hfn(host_ctxt);
return;
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index fee7dcd95d73..978301392d67 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -283,12 +283,6 @@ static void generate_mov_q(u64 val, __le32 *origptr,
__le32 *updptr, int nr_inst
*updptr++ = cpu_to_le32(insn);
}
-void kvm_update_kimg_phys_offset(struct alt_instr *alt,
-__le32 *origptr, __le32 *updptr, int nr_inst)
-{
- generate_mov_q(kimage_voffset + PHYS_OFFSET, origptr, updptr, nr_inst);
-}
-
void kvm_get_kimage_voffset(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst)
{
--
2.29.2.729.g45daf8777d-goog
[PATCH v2 3/8] KVM: arm64: Add symbol at the beginning of each hyp section
Generating hyp relocations will require referencing positions at a given
offset from the beginning of hyp sections. Since the final layout will
not be determined until the linking of `vmlinux`, modify the hyp linker
script to insert a symbol at the first byte of each hyp section to use
as an anchor. The linker of `vmlinux` will place the symbols together
with the sections.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/hyp_image.h | 29 +++--
arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 4 ++--
2 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/hyp_image.h
b/arch/arm64/include/asm/hyp_image.h
index daa1a1da539e..f97b774b58f4 100644
--- a/arch/arm64/include/asm/hyp_image.h
+++ b/arch/arm64/include/asm/hyp_image.h
@@ -7,6 +7,9 @@
#ifndef __ARM64_HYP_IMAGE_H__
#define __ARM64_HYP_IMAGE_H__
+#define HYP_CONCAT(a, b) __HYP_CONCAT(a, b)
+#define __HYP_CONCAT(a, b) a ## b
+
/*
* KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_,
* to separate it from the kernel proper.
@@ -21,9 +24,31 @@
*/
#define HYP_SECTION_NAME(NAME) .hyp##NAME
+/* Symbol defined at the beginning of each hyp section. */
+#define HYP_SECTION_SYMBOL_NAME(NAME) \
+ HYP_CONCAT(__hyp_section_, HYP_SECTION_NAME(NAME))
+
+/*
+ * Helper to generate linker script statements starting a hyp section.
+ *
+ * A symbol with a well-known name is defined at the first byte. This
+ * is used as a base for hyp relocations (see gen-hyprel.c). It must
+ * be defined inside the section so the linker of `vmlinux` cannot
+ * separate it from the section data.
+ */
+#define BEGIN_HYP_SECTION(NAME)\
+ HYP_SECTION_NAME(NAME) : { \
+ HYP_SECTION_SYMBOL_NAME(NAME) = .;
+
+/* Helper to generate linker script statements ending a hyp section. */
+#define END_HYP_SECTION\
+ }
+
/* Defines an ELF hyp section from input section @NAME and its subsections. */
-#define HYP_SECTION(NAME) \
- HYP_SECTION_NAME(NAME) : { *(NAME NAME##.*) }
+#define HYP_SECTION(NAME) \
+ BEGIN_HYP_SECTION(NAME) \
+ *(NAME NAME##.*)\
+ END_HYP_SECTION
/*
* Defines a linker script alias of a kernel-proper symbol referenced by
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index cfdc59b4329b..cd119d82d8e3 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -22,7 +22,7 @@ SECTIONS {
* alignment for when linking into vmlinux.
*/
. = ALIGN(PAGE_SIZE);
- HYP_SECTION_NAME(.data..percpu) : {
+ BEGIN_HYP_SECTION(.data..percpu)
PERCPU_INPUT(L1_CACHE_BYTES)
- }
+ END_HYP_SECTION
}
--
2.29.2.729.g45daf8777d-goog
[PATCH v2 2/8] KVM: arm64: Set up .hyp.rodata ELF section
We will need to recognize pointers in .rodata specific to hyp, so
establish a .hyp.rodata ELF section. Merge it with the existing
.hyp.data..ro_after_init as they are treated the same at runtime.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/sections.h | 2 +-
arch/arm64/kernel/vmlinux.lds.S | 7 ---
arch/arm64/kvm/arm.c | 7 +++
arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 4 +++-
4 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/sections.h
b/arch/arm64/include/asm/sections.h
index 8ff579361731..a6f3557d1ab2 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -11,7 +11,7 @@ extern char __alt_instructions[], __alt_instructions_end[];
extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[];
extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
extern char __hyp_text_start[], __hyp_text_end[];
-extern char __hyp_data_ro_after_init_start[], __hyp_data_ro_after_init_end[];
+extern char __hyp_rodata_start[], __hyp_rodata_end[];
extern char __idmap_text_start[], __idmap_text_end[];
extern char __initdata_begin[], __initdata_end[];
extern char __inittext_begin[], __inittext_end[];
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 4c0b0c89ad59..9672b54bba7c 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -31,10 +31,11 @@ jiffies = jiffies_64;
__stop___kvm_ex_table = .;
#define HYPERVISOR_DATA_SECTIONS \
- HYP_SECTION_NAME(.data..ro_after_init) : { \
- __hyp_data_ro_after_init_start = .; \
+ HYP_SECTION_NAME(.rodata) : { \
+ __hyp_rodata_start = .; \
*(HYP_SECTION_NAME(.data..ro_after_init)) \
- __hyp_data_ro_after_init_end = .; \
+ *(HYP_SECTION_NAME(.rodata))\
+ __hyp_rodata_end = .; \
}
#define HYPERVISOR_PERCPU_SECTION \
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 04c44853b103..de1af4052780 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1749,11 +1749,10 @@ static int init_hyp_mode(void)
goto out_err;
}
- err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_ro_after_init_start),
- kvm_ksym_ref(__hyp_data_ro_after_init_end),
- PAGE_HYP_RO);
+ err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
+ kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
if (err) {
- kvm_err("Cannot map .hyp.data..ro_after_init section\n");
+ kvm_err("Cannot map .hyp.rodata section\n");
goto out_err;
}
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index 70ac48ccede7..cfdc59b4329b 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -14,6 +14,9 @@
SECTIONS {
HYP_SECTION(.idmap.text)
HYP_SECTION(.text)
+ HYP_SECTION(.data..ro_after_init)
+ HYP_SECTION(.rodata)
+
/*
* .hyp..data..percpu needs to be page aligned to maintain the same
* alignment for when linking into vmlinux.
@@ -22,5 +25,4 @@ SECTIONS {
HYP_SECTION_NAME(.data..percpu) : {
PERCPU_INPUT(L1_CACHE_BYTES)
}
- HYP_SECTION(.data..ro_after_init)
}
--
2.29.2.729.g45daf8777d-goog
[PATCH v2 0/8] arm64: Relocate absolute hyp VAs
nVHE hyp code is linked into the same kernel binary but executes under different memory mappings. If the compiler of hyp code chooses absolute addressing for accessing a symbol, the kernel linker will relocate that address to a kernel image virtual address, causing a runtime exception. So far the strategy has been to force PC-relative addressing by wrapping all symbol references with the hyp_symbol_addr macro. This is error prone and developer unfriendly. The series adds a new build-time step for nVHE hyp object file where positions targeted by R_AARCH64_ABS64 relocations are enumerated and the information stored in a separate ELF section in the kernel image. At runtime, the kernel first relocates all absolute addresses to their actual virtual offset (eg. for KASLR), and then addresses listed in this section are converted to hyp VAs. The RFC of this series did not have a build-time step and instead relied on filtering dynamic relocations at runtime. That approach does not work if the kernel is built with !CONFIG_RELOCATABLE, hence an always-present set of relocation positions was added. The series is based on 5.11-rc2 + kvmarm/next and structured as follows: * patches 1-2 make sure that all sections referred to by hyp code are handled by the hyp linker script and prefixed with .hyp so they can be identified by the build-time tool * patches 3-5 contain the actual changes to identify and relocate VAs * patches 6-7 fix existing code that assumes kernel VAs * patch 8 removes the (now redundant) hyp_symbol_addr The series is also available at: https://android-kvm.googlesource.com/linux topic/hyp-reloc_v2 Changes since v1: * fix for older linkers: declare hyp section symbols in hyp-reloc.S * fix for older host glibc: define R_AARCH64_ constants if missing * add generated files to .gitignore -David David Brazdil (8): KVM: arm64: Rename .idmap.text in hyp linker script KVM: arm64: Set up .hyp.rodata ELF section KVM: arm64: Add symbol at the beginning of each hyp section KVM: arm64: Generate hyp relocation data KVM: arm64: Apply hyp relocations at runtime KVM: arm64: Fix constant-pool users in hyp KVM: arm64: Remove patching of fn pointers in hyp KVM: arm64: Remove hyp_symbol_addr arch/arm64/include/asm/hyp_image.h | 29 +- arch/arm64/include/asm/kvm_asm.h | 26 -- arch/arm64/include/asm/kvm_mmu.h | 61 +--- arch/arm64/include/asm/sections.h| 3 +- arch/arm64/kernel/image-vars.h | 1 - arch/arm64/kernel/smp.c | 4 +- arch/arm64/kernel/vmlinux.lds.S | 18 +- arch/arm64/kvm/arm.c | 7 +- arch/arm64/kvm/hyp/include/hyp/switch.h | 4 +- arch/arm64/kvm/hyp/nvhe/.gitignore | 2 + arch/arm64/kvm/hyp/nvhe/Makefile | 28 +- arch/arm64/kvm/hyp/nvhe/gen-hyprel.c | 413 +++ arch/arm64/kvm/hyp/nvhe/host.S | 29 +- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 4 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 11 +- arch/arm64/kvm/hyp/nvhe/hyp-smp.c| 4 +- arch/arm64/kvm/hyp/nvhe/hyp.lds.S| 9 +- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 24 +- arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c | 2 +- arch/arm64/kvm/va_layout.c | 34 +- 20 files changed, 578 insertions(+), 135 deletions(-) create mode 100644 arch/arm64/kvm/hyp/nvhe/gen-hyprel.c -- 2.29.2.729.g45daf8777d-goog
[PATCH v2 1/8] KVM: arm64: Rename .idmap.text in hyp linker script
So far hyp-init.S created a .hyp.idmap.text section directly, without
relying on the hyp linker script to prefix its name. Change it to create
.idmap.text and add a HYP_SECTION entry to hyp.lds.S. This way all .hyp*
sections go through the linker script and can be instrumented there.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 2 +-
arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index 31b060a44045..68fd64f2313e 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -18,7 +18,7 @@
#include
.text
- .pushsection.hyp.idmap.text, "ax"
+ .pushsection.idmap.text, "ax"
.align 11
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index 1206d0d754d5..70ac48ccede7 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -12,6 +12,7 @@
#include
SECTIONS {
+ HYP_SECTION(.idmap.text)
HYP_SECTION(.text)
/*
* .hyp..data..percpu needs to be page aligned to maintain the same
--
2.29.2.729.g45daf8777d-goog
[PATCH v2 4/8] KVM: arm64: Generate hyp relocation data
Add a post-processing step to compilation of KVM nVHE hyp code which
calls a custom host tool (gen-hyprel) on the partially linked object
file (hyp sections' names prefixed).
The tool lists all R_AARCH64_ABS64 data relocations targeting hyp
sections and generates an assembly file that will form a new section
.hyp.reloc in the kernel binary. The new section contains an array of
32-bit offsets to the positions targeted by these relocations.
Since these addresses of those positions will not be determined until
linking of `vmlinux`, each 32-bit entry carries a R_AARCH64_PREL32
relocation with addend + . The linker of
`vmlinux` will therefore fill the slot accordingly.
This relocation data will be used at runtime to convert the kernel VAs
at those positions to hyp VAs.
Signed-off-by: David Brazdil
---
arch/arm64/kernel/vmlinux.lds.S | 11 +
arch/arm64/kvm/hyp/nvhe/.gitignore | 2 +
arch/arm64/kvm/hyp/nvhe/Makefile | 28 +-
arch/arm64/kvm/hyp/nvhe/gen-hyprel.c | 413 +++
4 files changed, 451 insertions(+), 3 deletions(-)
create mode 100644 arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 9672b54bba7c..636ca45aa1d4 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -43,10 +43,19 @@ jiffies = jiffies_64;
HYP_SECTION_NAME(.data..percpu) : { \
*(HYP_SECTION_NAME(.data..percpu)) \
}
+
+#define HYPERVISOR_RELOC_SECTION \
+ .hyp.reloc : ALIGN(4) { \
+ __hyp_reloc_begin = .; \
+ *(.hyp.reloc) \
+ __hyp_reloc_end = .;\
+ }
+
#else /* CONFIG_KVM */
#define HYPERVISOR_EXTABLE
#define HYPERVISOR_DATA_SECTIONS
#define HYPERVISOR_PERCPU_SECTION
+#define HYPERVISOR_RELOC_SECTION
#endif
#define HYPERVISOR_TEXT\
@@ -217,6 +226,8 @@ SECTIONS
PERCPU_SECTION(L1_CACHE_BYTES)
HYPERVISOR_PERCPU_SECTION
+ HYPERVISOR_RELOC_SECTION
+
.rela.dyn : ALIGN(8) {
*(.rela .rela*)
}
diff --git a/arch/arm64/kvm/hyp/nvhe/.gitignore
b/arch/arm64/kvm/hyp/nvhe/.gitignore
index 695d73d0249e..5b6c43cc96f8 100644
--- a/arch/arm64/kvm/hyp/nvhe/.gitignore
+++ b/arch/arm64/kvm/hyp/nvhe/.gitignore
@@ -1,2 +1,4 @@
# SPDX-License-Identifier: GPL-2.0-only
+gen-hyprel
hyp.lds
+hyp-reloc.S
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index 1f1e351c5fe2..268be1376f74 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -6,6 +6,8 @@
asflags-y := -D__KVM_NVHE_HYPERVISOR__
ccflags-y := -D__KVM_NVHE_HYPERVISOR__
+hostprogs := gen-hyprel
+
obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
hyp-main.o hyp-smp.o psci-relay.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
@@ -19,7 +21,7 @@ obj-y += ../vgic-v3-sr.o ../aarch32.o
../vgic-v2-cpuif-proxy.o ../entry.o \
hyp-obj := $(patsubst %.o,%.nvhe.o,$(obj-y))
obj-y := kvm_nvhe.o
-extra-y := $(hyp-obj) kvm_nvhe.tmp.o hyp.lds
+extra-y := $(hyp-obj) kvm_nvhe.tmp.o kvm_nvhe.rel.o hyp.lds hyp-reloc.S
hyp-reloc.o
# 1) Compile all source files to `.nvhe.o` object files. The file extension
#avoids file name clashes for files shared with VHE.
@@ -42,11 +44,31 @@ LDFLAGS_kvm_nvhe.tmp.o := -r -T
$(obj)/kvm_nvhe.tmp.o: $(obj)/hyp.lds $(addprefix $(obj)/,$(hyp-obj)) FORCE
$(call if_changed,ld)
-# 4) Produce the final 'kvm_nvhe.o', ready to be linked into 'vmlinux'.
+# 4) Generate list of hyp code/data positions that need to be relocated at
+#runtime. Because the hypervisor is part of the kernel binary, relocations
+#produce a kernel VA. We enumerate relocations targeting hyp at build time
+#and convert the kernel VAs at those positions to hyp VAs.
+$(obj)/hyp-reloc.S: $(obj)/kvm_nvhe.tmp.o $(obj)/gen-hyprel
+ $(call if_changed,hyprel)
+
+# 5) Compile hyp-reloc.S and link it into the existing partially linked object.
+#The object file now contains a section with pointers to hyp positions that
+#will contain kernel VAs at runtime. These pointers have relocations on
them
+#so that they get updated as the hyp object is linked into `vmlinux`.
+LDFLAGS_kvm_nvhe.rel.o := -r
+$(obj)/kvm_nvhe.rel.o: $(obj)/kvm_nvhe.tmp.o $(obj)/hyp-reloc.o FORCE
+ $(call if_changed,ld)
+
+# 6) Produce the final 'kvm_nvhe.o', ready to be linked into 'vmlinux'.
#Prefixes names of ELF symbols with '__kvm_nvhe_'.
-$(obj)/kvm_nvhe.o: $(obj)/kvm_nvhe.tmp.o FORCE
+$(obj)/kvm_nvhe.o: $(obj)/kvm_nvhe.rel.o FORCE
$(call if_changed,hypcopy)
+# The HYPREL command calls `gen-hyprel` to generate an assembly file with
+# a list
[PATCH] arm64: Remove unused variable in arch_show_interrupts
Commit 5089bc51f81f ("arm64/smp: Use irq_desc_kstat_cpu() in
arch_show_interrupts()") removed the only user of variable `irq`.
Remove the unused variable.
Signed-off-by: David Brazdil
---
arch/arm64/kernel/smp.c | 1 -
1 file changed, 1 deletion(-)
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 6bc3a3698c3d..376343d6f13a 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -807,7 +807,6 @@ int arch_show_interrupts(struct seq_file *p, int prec)
unsigned int cpu, i;
for (i = 0; i < NR_IPI; i++) {
- unsigned int irq = irq_desc_get_irq(ipi_desc[i]);
seq_printf(p, "%*s%u:%s", prec - 1, "IPI", i,
prec >= 4 ? " " : "");
for_each_online_cpu(cpu)
--
2.29.2.729.g45daf8777d-goog
Re: [PATCH] KVM: arm64: Allow PSCI SYSTEM_OFF/RESET to return
On Tue, Dec 29, 2020 at 05:16:41PM +, Marc Zyngier wrote:
> Hi David,
>
> On 2020-12-29 16:00, David Brazdil wrote:
> > The KVM/arm64 PSCI relay assumes that SYSTEM_OFF and SYSTEM_RESET should
> > not return, as dictated by the PSCI spec. However, there is firmware out
> > there which breaks this assumption, leading to a hyp panic. Make KVM
> > more robust to broken firmware by allowing these to return.
> >
> > Signed-off-by: David Brazdil
> > ---
> > arch/arm64/kvm/hyp/nvhe/psci-relay.c | 13 +
> > 1 file changed, 5 insertions(+), 8 deletions(-)
> >
> > diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
> > b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
> > index e3947846ffcb..8e7128cb7667 100644
> > --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
> > +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
> > @@ -77,12 +77,6 @@ static unsigned long psci_forward(struct
> > kvm_cpu_context *host_ctxt)
> > cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3));
> > }
> >
> > -static __noreturn unsigned long psci_forward_noreturn(struct
> > kvm_cpu_context *host_ctxt)
> > -{
> > - psci_forward(host_ctxt);
> > - hyp_panic(); /* unreachable */
> > -}
> > -
> > static unsigned int find_cpu_id(u64 mpidr)
> > {
> > unsigned int i;
> > @@ -251,10 +245,13 @@ static unsigned long psci_0_2_handler(u64
> > func_id, struct kvm_cpu_context *host_
> > case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
> > case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU:
> > return psci_forward(host_ctxt);
> > + /*
> > +* SYSTEM_OFF/RESET should not return according to the spec.
> > +* Allow it so as to stay robust to broken firmware.
> > +*/
> > case PSCI_0_2_FN_SYSTEM_OFF:
> > case PSCI_0_2_FN_SYSTEM_RESET:
> > - psci_forward_noreturn(host_ctxt);
> > - unreachable();
> > + return psci_forward(host_ctxt);
> > case PSCI_0_2_FN64_CPU_SUSPEND:
> > return psci_cpu_suspend(func_id, host_ctxt);
> > case PSCI_0_2_FN64_CPU_ON:
>
> Thanks for having tracked this.
>
> I wonder whether we should also taint the kernel in this case,
> because this is completely unexpected, and a major spec violation.
>
> Ideally, we'd be able to detect this case and prevent pKVM from
> getting initialised at all, but I guess there is no way to detect
> the sucker without ... calling SYSTEM_RESET?
Yeah, there are no bits to check, unfortunately. :(
David
[PATCH] KVM: arm64: Allow PSCI SYSTEM_OFF/RESET to return
The KVM/arm64 PSCI relay assumes that SYSTEM_OFF and SYSTEM_RESET should
not return, as dictated by the PSCI spec. However, there is firmware out
there which breaks this assumption, leading to a hyp panic. Make KVM
more robust to broken firmware by allowing these to return.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 13 +
1 file changed, 5 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index e3947846ffcb..8e7128cb7667 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -77,12 +77,6 @@ static unsigned long psci_forward(struct kvm_cpu_context
*host_ctxt)
cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3));
}
-static __noreturn unsigned long psci_forward_noreturn(struct kvm_cpu_context
*host_ctxt)
-{
- psci_forward(host_ctxt);
- hyp_panic(); /* unreachable */
-}
-
static unsigned int find_cpu_id(u64 mpidr)
{
unsigned int i;
@@ -251,10 +245,13 @@ static unsigned long psci_0_2_handler(u64 func_id, struct
kvm_cpu_context *host_
case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU:
return psci_forward(host_ctxt);
+ /*
+* SYSTEM_OFF/RESET should not return according to the spec.
+* Allow it so as to stay robust to broken firmware.
+*/
case PSCI_0_2_FN_SYSTEM_OFF:
case PSCI_0_2_FN_SYSTEM_RESET:
- psci_forward_noreturn(host_ctxt);
- unreachable();
+ return psci_forward(host_ctxt);
case PSCI_0_2_FN64_CPU_SUSPEND:
return psci_cpu_suspend(func_id, host_ctxt);
case PSCI_0_2_FN64_CPU_ON:
--
2.29.2.729.g45daf8777d-goog
[PATCH 6/9] KVM: arm64: Apply hyp relocations at runtime
KVM nVHE code runs under a different VA mapping than the kernel, hence
so far it avoided using absolute addressing because the VA in a constant
pool is relocated by the linker to a kernel VA (see hyp_symbol_addr).
Now the kernel has access to a list of positions that contain a kimg VA
but will be accessed only in hyp execution context. These are generated
by the gen-hyprel build-time tool and stored in .hyp.reloc.
Add early boot pass over the entries and convert the kimg VAs to hyp VAs.
Note that this requires for .hyp* ELF sections to be mapped read-write
at that point.
Signed-off-by: David Brazdil
---
arch/arm64/configs/defconfig | 1 +
arch/arm64/include/asm/kvm_mmu.h | 1 +
arch/arm64/include/asm/sections.h | 1 +
arch/arm64/kernel/smp.c | 4 +++-
arch/arm64/kvm/va_layout.c| 28
5 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/configs/defconfig b/arch/arm64/configs/defconfig
index 5cfe3cf6f2ac..73fc9f2f2661 100644
--- a/arch/arm64/configs/defconfig
+++ b/arch/arm64/configs/defconfig
@@ -1092,3 +1092,4 @@ CONFIG_DEBUG_KERNEL=y
# CONFIG_DEBUG_PREEMPT is not set
# CONFIG_FTRACE is not set
CONFIG_MEMTEST=y
+# CONFIG_ARM64_VHE is not set
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index e52d82aeadca..6bbb44011c84 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -129,6 +129,7 @@ alternative_cb_end
void kvm_update_va_mask(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst);
void kvm_compute_layout(void);
+void kvm_apply_hyp_relocations(void);
static __always_inline unsigned long __kern_hyp_va(unsigned long v)
{
diff --git a/arch/arm64/include/asm/sections.h
b/arch/arm64/include/asm/sections.h
index a6f3557d1ab2..2f36b16a5b5d 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -12,6 +12,7 @@ extern char __hibernate_exit_text_start[],
__hibernate_exit_text_end[];
extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
extern char __hyp_text_start[], __hyp_text_end[];
extern char __hyp_rodata_start[], __hyp_rodata_end[];
+extern char __hyp_reloc_begin[], __hyp_reloc_end[];
extern char __idmap_text_start[], __idmap_text_end[];
extern char __initdata_begin[], __initdata_end[];
extern char __inittext_begin[], __inittext_end[];
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 18e9727d3f64..47142395bc91 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -434,8 +434,10 @@ static void __init hyp_mode_check(void)
"CPU: CPUs started in inconsistent modes");
else
pr_info("CPU: All CPU(s) started at EL1\n");
- if (IS_ENABLED(CONFIG_KVM))
+ if (IS_ENABLED(CONFIG_KVM)) {
kvm_compute_layout();
+ kvm_apply_hyp_relocations();
+ }
}
void __init smp_cpus_done(unsigned int max_cpus)
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index d8cc51bd60bf..fb2ca02b7270 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -82,6 +82,34 @@ __init void kvm_compute_layout(void)
init_hyp_physvirt_offset();
}
+/*
+ * The .hyp.reloc ELF section contains a list of kimg positions that
+ * contains kimg VAs but will be accessed only in hyp execution context.
+ * Convert them to hyp VAs. See gen-hyprel.c for more details.
+ */
+__init void kvm_apply_hyp_relocations(void)
+{
+ int32_t *rel;
+ int32_t *begin = (int32_t*)__hyp_reloc_begin;
+ int32_t *end = (int32_t*)__hyp_reloc_end;
+
+ for (rel = begin; rel < end; ++rel) {
+ uintptr_t *ptr, kimg_va;
+
+ /*
+* Each entry contains a 32-bit relative offset from itself
+* to a kimg VA position.
+*/
+ ptr = (uintptr_t*)lm_alias((char*)rel + *rel);
+
+ /* Read the kimg VA value at the relocation address. */
+ kimg_va = *ptr;
+
+ /* Convert to hyp VA and store back to the relocation address.
*/
+ *ptr = __early_kern_hyp_va((uintptr_t)lm_alias(kimg_va));
+ }
+}
+
static u32 compute_instruction(int n, u32 rd, u32 rn)
{
u32 insn = AARCH64_BREAK_FAULT;
--
2.29.2.576.ga3fc446d84-goog
[PATCH 9/9] KVM: arm64: Remove hyp_symbol_addr
Hyp code used the hyp_symbol_addr helper to force PC-relative addressing
because absolute addressing results in kernel VAs due to the way hyp
code is linked. This is not true anymore, so remove the helper and
update all of its users.
Acked-by: Ard Biesheuvel
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_asm.h | 20
arch/arm64/kvm/hyp/include/hyp/switch.h | 4 ++--
arch/arm64/kvm/hyp/nvhe/hyp-smp.c| 4 ++--
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 24
arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c | 2 +-
5 files changed, 17 insertions(+), 37 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 7ccf770c53d9..22d933e9b59e 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -199,26 +199,6 @@ extern void __vgic_v3_init_lrs(void);
extern u32 __kvm_get_mdcr_el2(void);
-/*
- * Obtain the PC-relative address of a kernel symbol
- * s: symbol
- *
- * The goal of this macro is to return a symbol's address based on a
- * PC-relative computation, as opposed to a loading the VA from a
- * constant pool or something similar. This works well for HYP, as an
- * absolute VA is guaranteed to be wrong. Only use this if trying to
- * obtain the address of a symbol (i.e. not something you obtained by
- * following a pointer).
- */
-#define hyp_symbol_addr(s) \
- ({ \
- typeof(s) *addr;\
- asm("adrp %0, %1\n" \
- "add%0, %0, :lo12:%1\n" \
- : "=r" (addr) : "S" ()); \
- addr; \
- })
-
#define __KVM_EXTABLE(from, to)
\
" .pushsection__kvm_ex_table, \"a\"\n"\
" .align 3\n"\
diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h
b/arch/arm64/kvm/hyp/include/hyp/switch.h
index 84473574c2e7..54f4860cd87c 100644
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -505,8 +505,8 @@ static inline void __kvm_unexpected_el2_exception(void)
struct exception_table_entry *entry, *end;
unsigned long elr_el2 = read_sysreg(elr_el2);
- entry = hyp_symbol_addr(__start___kvm_ex_table);
- end = hyp_symbol_addr(__stop___kvm_ex_table);
+ entry = &__start___kvm_ex_table;
+ end = &__stop___kvm_ex_table;
while (entry < end) {
addr = (unsigned long)>insn + entry->insn;
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
index cbab0c6246e2..2048725517f8 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
@@ -33,8 +33,8 @@ unsigned long __hyp_per_cpu_offset(unsigned int cpu)
if (cpu >= ARRAY_SIZE(kvm_arm_hyp_percpu_base))
hyp_panic();
- cpu_base_array = (unsigned long
*)hyp_symbol_addr(kvm_arm_hyp_percpu_base);
+ cpu_base_array = (unsigned long *)_arm_hyp_percpu_base;
this_cpu_base = kern_hyp_va(cpu_base_array[cpu]);
- elf_base = (unsigned long)hyp_symbol_addr(__per_cpu_start);
+ elf_base = (unsigned long)&__per_cpu_start;
return this_cpu_base - elf_base;
}
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 08dc9de69314..746fb7079581 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -151,8 +151,8 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context
*host_ctxt)
if (cpu_id == INVALID_CPU_ID)
return PSCI_RET_INVALID_PARAMS;
- boot_args = per_cpu_ptr(hyp_symbol_addr(cpu_on_args), cpu_id);
- init_params = per_cpu_ptr(hyp_symbol_addr(kvm_init_params), cpu_id);
+ boot_args = per_cpu_ptr(_on_args, cpu_id);
+ init_params = per_cpu_ptr(_init_params, cpu_id);
/* Check if the target CPU is already being booted. */
if (!try_acquire_boot_args(boot_args))
@@ -163,7 +163,7 @@ static int psci_cpu_on(u64 func_id, struct kvm_cpu_context
*host_ctxt)
wmb();
ret = psci_call(func_id, mpidr,
- __hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_entry)),
+ __hyp_pa(_hyp_cpu_entry),
__hyp_pa(init_params));
/* If successful, the lock will be released by the target CPU. */
@@ -182,8 +182,8 @@ static int psci_cpu_suspend(u64 func_id, struct
kvm_cpu_context *host_ctxt)
struct psci_boot_args *boot_args;
struct kvm_
[PATCH 7/9] KVM: arm64: Fix constant-pool users in hyp
Hyp code uses absolute addressing to obtain a kimg VA of a small number of kernel symbols. Since the kernel now converts constant pool addresses to hyp VAs, this trick does not work anymore. Change the helpers to convert from hyp VA back to kimg VA or PA, as needed and rework the callers accordingly. Signed-off-by: David Brazdil --- arch/arm64/include/asm/kvm_mmu.h | 42 -- arch/arm64/kvm/hyp/nvhe/host.S | 29 +++-- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 2 -- 3 files changed, 31 insertions(+), 42 deletions(-) diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 6bbb44011c84..adadc468cc71 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -73,49 +73,39 @@ alternative_cb_end .endm /* - * Convert a kernel image address to a PA - * reg: kernel address to be converted in place + * Convert a hypervisor VA to a PA + * reg: hypervisor address to be converted in place * tmp: temporary register - * - * The actual code generation takes place in kvm_get_kimage_voffset, and - * the instructions below are only there to reserve the space and - * perform the register allocation (kvm_get_kimage_voffset uses the - * specific registers encoded in the instructions). */ -.macro kimg_pa reg, tmp -alternative_cb kvm_get_kimage_voffset - movz\tmp, #0 - movk\tmp, #0, lsl #16 - movk\tmp, #0, lsl #32 - movk\tmp, #0, lsl #48 -alternative_cb_end - - /* reg = __pa(reg) */ - sub \reg, \reg, \tmp +.macro hyp_pa reg, tmp + ldr_l \tmp, hyp_physvirt_offset + add \reg, \reg, \tmp .endm /* - * Convert a kernel image address to a hyp VA - * reg: kernel address to be converted in place + * Convert a hypervisor VA to a kernel image address + * reg: hypervisor address to be converted in place * tmp: temporary register * * The actual code generation takes place in kvm_get_kimage_voffset, and * the instructions below are only there to reserve the space and - * perform the register allocation (kvm_update_kimg_phys_offset uses the + * perform the register allocation (kvm_get_kimage_voffset uses the * specific registers encoded in the instructions). */ -.macro kimg_hyp_va reg, tmp -alternative_cb kvm_update_kimg_phys_offset +.macro hyp_kimg_va reg, tmp + /* Convert hyp VA -> PA. */ + hyp_pa \reg, \tmp + + /* Load kimage_voffset. */ +alternative_cb kvm_get_kimage_voffset movz\tmp, #0 movk\tmp, #0, lsl #16 movk\tmp, #0, lsl #32 movk\tmp, #0, lsl #48 alternative_cb_end - sub \reg, \reg, \tmp - mov_q \tmp, PAGE_OFFSET - orr \reg, \reg, \tmp - kern_hyp_va \reg + /* Convert PA -> kimg VA. */ + add \reg, \reg, \tmp .endm #else diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index a820dfdc9c25..6585a7cbbc56 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -74,27 +74,28 @@ SYM_FUNC_END(__host_enter) * void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par); */ SYM_FUNC_START(__hyp_do_panic) - /* Load the format arguments into x1-7 */ - mov x6, x3 - get_vcpu_ptr x7, x3 - - mrs x3, esr_el2 - mrs x4, far_el2 - mrs x5, hpfar_el2 - /* Prepare and exit to the host's panic funciton. */ mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ PSR_MODE_EL1h) msr spsr_el2, lr ldr lr, =panic + hyp_kimg_va lr, x6 msr elr_el2, lr - /* -* Set the panic format string and enter the host, conditionally -* restoring the host context. -*/ + /* Set the panic format string. Use the, now free, LR as scratch. */ + ldr lr, =__hyp_panic_string + hyp_kimg_va lr, x6 + + /* Load the format arguments into x1-7. */ + mov x6, x3 + get_vcpu_ptr x7, x3 + mrs x3, esr_el2 + mrs x4, far_el2 + mrs x5, hpfar_el2 + + /* Enter the host, conditionally restoring the host context. */ cmp x0, xzr - ldr x0, =__hyp_panic_string + mov x0, lr b.eq__host_enter_without_restoring b __host_enter_for_panic SYM_FUNC_END(__hyp_do_panic) @@ -124,7 +125,7 @@ SYM_FUNC_END(__hyp_do_panic) * Preserve x0-x4, which may contain stub parameters. */ ldr x5, =__kvm_handle_stub_hvc - kimg_pa x5, x6 + hyp_pa x5, x6 br x5 .L__vect_end\@: .if ((.L__vect_end\@ - .L__vect_start\@) > 0x80) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 68fd64f2313e..99b408fe09ee 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -139,7 +139,6 @@ alternative_else_
[PATCH 8/9] KVM: arm64: Remove patching of fn pointers in hyp
Storing a function pointer in hyp now generates relocation information
used at early boot to convert the address to hyp VA. The existing
alternative-based conversion mechanism is therefore obsolete. Remove it
and simplify its users.
Acked-by: Ard Biesheuvel
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_mmu.h | 18 --
arch/arm64/kernel/image-vars.h | 1 -
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 11 ---
arch/arm64/kvm/va_layout.c | 6 --
4 files changed, 4 insertions(+), 32 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index adadc468cc71..90873851f677 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -135,24 +135,6 @@ static __always_inline unsigned long
__kern_hyp_va(unsigned long v)
#define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v
-static __always_inline unsigned long __kimg_hyp_va(unsigned long v)
-{
- unsigned long offset;
-
- asm volatile(ALTERNATIVE_CB("movz %0, #0\n"
- "movk %0, #0, lsl #16\n"
- "movk %0, #0, lsl #32\n"
- "movk %0, #0, lsl #48\n",
- kvm_update_kimg_phys_offset)
-: "=r" (offset));
-
- return __kern_hyp_va((v - offset) | PAGE_OFFSET);
-}
-
-#define kimg_fn_hyp_va(v) ((typeof(*v))(__kimg_hyp_va((unsigned
long)(v
-
-#define kimg_fn_ptr(x) (typeof(x) **)(x)
-
/*
* We currently support using a VM-specified IPA size. For backward
* compatibility, the default IPA size is fixed to 40bits.
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 39289d75118d..3242502f45fa 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -64,7 +64,6 @@ __efistub__ctype = _ctype;
/* Alternative callbacks for init-time patching of nVHE hyp code. */
KVM_NVHE_ALIAS(kvm_patch_vector_branch);
KVM_NVHE_ALIAS(kvm_update_va_mask);
-KVM_NVHE_ALIAS(kvm_update_kimg_phys_offset);
KVM_NVHE_ALIAS(kvm_get_kimage_voffset);
/* Global kernel state accessed by nVHE hyp code. */
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index bde658d51404..0cf4b750a090 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -108,9 +108,9 @@ static void handle___vgic_v3_restore_aprs(struct
kvm_cpu_context *host_ctxt)
typedef void (*hcall_t)(struct kvm_cpu_context *);
-#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = kimg_fn_ptr(handle_##x)
+#define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
-static const hcall_t *host_hcall[] = {
+static const hcall_t host_hcall[] = {
HANDLE_FUNC(__kvm_vcpu_run),
HANDLE_FUNC(__kvm_flush_vm_context),
HANDLE_FUNC(__kvm_tlb_flush_vmid_ipa),
@@ -130,7 +130,6 @@ static const hcall_t *host_hcall[] = {
static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(unsigned long, id, host_ctxt, 0);
- const hcall_t *kfn;
hcall_t hfn;
id -= KVM_HOST_SMCCC_ID(0);
@@ -138,13 +137,11 @@ static void handle_host_hcall(struct kvm_cpu_context
*host_ctxt)
if (unlikely(id >= ARRAY_SIZE(host_hcall)))
goto inval;
- kfn = host_hcall[id];
- if (unlikely(!kfn))
+ hfn = host_hcall[id];
+ if (unlikely(!hfn))
goto inval;
cpu_reg(host_ctxt, 0) = SMCCC_RET_SUCCESS;
-
- hfn = kimg_fn_hyp_va(kfn);
hfn(host_ctxt);
return;
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index fb2ca02b7270..e0021ba960b5 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -284,12 +284,6 @@ static void generate_mov_q(u64 val, __le32 *origptr,
__le32 *updptr, int nr_inst
*updptr++ = cpu_to_le32(insn);
}
-void kvm_update_kimg_phys_offset(struct alt_instr *alt,
-__le32 *origptr, __le32 *updptr, int nr_inst)
-{
- generate_mov_q(kimage_voffset + PHYS_OFFSET, origptr, updptr, nr_inst);
-}
-
void kvm_get_kimage_voffset(struct alt_instr *alt,
__le32 *origptr, __le32 *updptr, int nr_inst)
{
--
2.29.2.576.ga3fc446d84-goog
[PATCH 2/9] KVM: arm64: Rename .idmap.text in hyp linker script
So far hyp-init.S created a .hyp.idmap.text section directly, without
relying on the hyp linker script to prefix its name. Change it to create
.idmap.text and add a HYP_SECTION entry to hyp.lds.S. This way all .hyp*
sections go through the linker script and can be instrumented there.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 2 +-
arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 1 +
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index 31b060a44045..68fd64f2313e 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -18,7 +18,7 @@
#include
.text
- .pushsection.hyp.idmap.text, "ax"
+ .pushsection.idmap.text, "ax"
.align 11
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index 1206d0d754d5..70ac48ccede7 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -12,6 +12,7 @@
#include
SECTIONS {
+ HYP_SECTION(.idmap.text)
HYP_SECTION(.text)
/*
* .hyp..data..percpu needs to be page aligned to maintain the same
--
2.29.2.576.ga3fc446d84-goog
[PATCH 1/9] KVM: arm64: Correctly align nVHE percpu data
From: Jamie Iles
The nVHE percpu data is partially linked but the nVHE linker script did
not align the percpu section. The PERCPU_INPUT macro would then align
the data to a page boundary:
#define PERCPU_INPUT(cacheline) \
__per_cpu_start = .;\
*(.data..percpu..first) \
. = ALIGN(PAGE_SIZE); \
*(.data..percpu..page_aligned) \
. = ALIGN(cacheline); \
*(.data..percpu..read_mostly) \
. = ALIGN(cacheline); \
*(.data..percpu)\
*(.data..percpu..shared_aligned)\
PERCPU_DECRYPTED_SECTION\
__per_cpu_end = .;
but then when the final vmlinux linking happens the hypervisor percpu
data is included after page alignment and so the offsets potentially
don't match. On my build I saw that the .hyp.data..percpu section was
at address 0x20 and then the percpu data would begin at 0x1000 (because
of the page alignment in PERCPU_INPUT), but when linked into vmlinux,
everything would be shifted down by 0x20 bytes.
This manifests as one of the CPUs getting lost when running
kvm-unit-tests or starting any VM and subsequent soft lockup on a Cortex
A72 device.
Fixes: 30c953911c43 ("kvm: arm64: Set up hyp percpu data for nVHE")
Signed-off-by: Jamie Iles
Signed-off-by: Marc Zyngier
Acked-by: David Brazdil
Cc: David Brazdil
Cc: Marc Zyngier
Cc: Will Deacon
Link: https://lore.kernel.org/r/20201113150406.14314-1-ja...@nuviainc.com
---
arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 5 +
1 file changed, 5 insertions(+)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index 5d76ff2ba63e..1206d0d754d5 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -13,6 +13,11 @@
SECTIONS {
HYP_SECTION(.text)
+ /*
+* .hyp..data..percpu needs to be page aligned to maintain the same
+* alignment for when linking into vmlinux.
+*/
+ . = ALIGN(PAGE_SIZE);
HYP_SECTION_NAME(.data..percpu) : {
PERCPU_INPUT(L1_CACHE_BYTES)
}
--
2.29.2.576.ga3fc446d84-goog
[PATCH 5/9] KVM: arm64: Generate hyp relocation data
Add a post-processing step to compilation of KVM nVHE hyp code which
calls a custom host tool (gen-hyprel) on the partially linked object
file (hyp sections' names prefixed).
The tool lists all R_AARCH64_ABS64 data relocations targeting hyp
sections and generates an assembly file that will form a new section
.hyp.reloc in the kernel binary. The new section contains an array of
32-bit offsets to the positions targeted by these relocations.
Since these addresses of those positions will not be determined until
linking of `vmlinux`, each 32-bit entry carries a R_AARCH64_PREL32
relocation with addend + . The linker of
`vmlinux` will therefore fill the slot accordingly.
This relocation data will be used at runtime to convert the kernel VAs
at those positions to hyp VAs.
Signed-off-by: David Brazdil
---
arch/arm64/kernel/vmlinux.lds.S | 11 +
arch/arm64/kvm/hyp/nvhe/Makefile | 28 ++-
arch/arm64/kvm/hyp/nvhe/gen-hyprel.c | 326 +++
3 files changed, 362 insertions(+), 3 deletions(-)
create mode 100644 arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index f294f2048955..93cef9607c0e 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -43,10 +43,19 @@ jiffies = jiffies_64;
HYP_SECTION_NAME(.data..percpu) : { \
*(HYP_SECTION_NAME(.data..percpu)) \
}
+
+#define HYPERVISOR_RELOC_SECTION \
+ .hyp.reloc : ALIGN(4) { \
+ __hyp_reloc_begin = .; \
+ *(.hyp.reloc) \
+ __hyp_reloc_end = .;\
+ }
+
#else /* CONFIG_KVM */
#define HYPERVISOR_EXTABLE
#define HYPERVISOR_DATA_SECTIONS
#define HYPERVISOR_PERCPU_SECTION
+#define HYPERVISOR_RELOC_SECTION
#endif
#define HYPERVISOR_TEXT\
@@ -219,6 +228,8 @@ SECTIONS
PERCPU_SECTION(L1_CACHE_BYTES)
HYPERVISOR_PERCPU_SECTION
+ HYPERVISOR_RELOC_SECTION
+
.rela.dyn : ALIGN(8) {
*(.rela .rela*)
}
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index 1f1e351c5fe2..268be1376f74 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -6,6 +6,8 @@
asflags-y := -D__KVM_NVHE_HYPERVISOR__
ccflags-y := -D__KVM_NVHE_HYPERVISOR__
+hostprogs := gen-hyprel
+
obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
hyp-main.o hyp-smp.o psci-relay.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
@@ -19,7 +21,7 @@ obj-y += ../vgic-v3-sr.o ../aarch32.o
../vgic-v2-cpuif-proxy.o ../entry.o \
hyp-obj := $(patsubst %.o,%.nvhe.o,$(obj-y))
obj-y := kvm_nvhe.o
-extra-y := $(hyp-obj) kvm_nvhe.tmp.o hyp.lds
+extra-y := $(hyp-obj) kvm_nvhe.tmp.o kvm_nvhe.rel.o hyp.lds hyp-reloc.S
hyp-reloc.o
# 1) Compile all source files to `.nvhe.o` object files. The file extension
#avoids file name clashes for files shared with VHE.
@@ -42,11 +44,31 @@ LDFLAGS_kvm_nvhe.tmp.o := -r -T
$(obj)/kvm_nvhe.tmp.o: $(obj)/hyp.lds $(addprefix $(obj)/,$(hyp-obj)) FORCE
$(call if_changed,ld)
-# 4) Produce the final 'kvm_nvhe.o', ready to be linked into 'vmlinux'.
+# 4) Generate list of hyp code/data positions that need to be relocated at
+#runtime. Because the hypervisor is part of the kernel binary, relocations
+#produce a kernel VA. We enumerate relocations targeting hyp at build time
+#and convert the kernel VAs at those positions to hyp VAs.
+$(obj)/hyp-reloc.S: $(obj)/kvm_nvhe.tmp.o $(obj)/gen-hyprel
+ $(call if_changed,hyprel)
+
+# 5) Compile hyp-reloc.S and link it into the existing partially linked object.
+#The object file now contains a section with pointers to hyp positions that
+#will contain kernel VAs at runtime. These pointers have relocations on
them
+#so that they get updated as the hyp object is linked into `vmlinux`.
+LDFLAGS_kvm_nvhe.rel.o := -r
+$(obj)/kvm_nvhe.rel.o: $(obj)/kvm_nvhe.tmp.o $(obj)/hyp-reloc.o FORCE
+ $(call if_changed,ld)
+
+# 6) Produce the final 'kvm_nvhe.o', ready to be linked into 'vmlinux'.
#Prefixes names of ELF symbols with '__kvm_nvhe_'.
-$(obj)/kvm_nvhe.o: $(obj)/kvm_nvhe.tmp.o FORCE
+$(obj)/kvm_nvhe.o: $(obj)/kvm_nvhe.rel.o FORCE
$(call if_changed,hypcopy)
+# The HYPREL command calls `gen-hyprel` to generate an assembly file with
+# a list of relocations targeting hyp code/data.
+quiet_cmd_hyprel = HYPREL $@
+ cmd_hyprel = $(obj)/gen-hyprel $< > $@
+
# The HYPCOPY command uses `objcopy` to prefix all ELF symbol names
# to avoid clashes with VHE code/data.
quiet_cmd_hypcopy = HYPCOPY $@
diff --git a/arch/arm64/kvm/hyp/nvhe/gen-hyprel.c
b/arch/arm64/kvm/hyp/nvhe/gen-hyprel
[PATCH 0/9] KVM: arm64: Relocate absolute hyp VAs
nVHE hyp code is linked into the same kernel binary but executes under different memory mappings. If the compiler of hyp code chooses absolute addressing for accessing a symbol, the kernel linker will relocate that address to a kernel image virtual address, causing a runtime exception. So far the strategy has been to force PC-relative addressing by wrapping all symbol references with the hyp_symbol_addr macro. This is error prone and developer unfriendly. The series adds a new build-time step for nVHE hyp object file where positions targeted by R_AARCH64_ABS64 relocations are enumerated and the information stored in a separate ELF section in the kernel image. At runtime, the kernel first relocates all absolute addresses to their actual virtual offset (eg. for KASLR), and then addresses listed in this section are converted to hyp VAs. The RFC of this series did not have a build-time step and instead relied on filtering dynamic relocations at runtime. That approach does not work if the kernel is built with !CONFIG_RELOCATABLE, hence an always-present set of relocation positions was added. The series is based on the current kvmarm/next (commit 3a514592b6) and structured as follows: * patch 1 is Jamie's fix of .hyp.data..percpu alignment; already in master, not yet in kvmarm/next; included to avoid merge conflicts * patches 2-3 make sure that all sections referred to by hyp code are handled by the hyp linker script and prefixed with .hyp so they can be identified by the build-time tool * patches 4-6 contain the actual changes to identify and relocate VAs * patches 7-8 fix existing code that assumes kernel VAs * patch 9 removes the (now redundant) hyp_symbol_addr The series is also available at: https://android-kvm.googlesource.com/linux topic/hyp-reloc_v1 -David David Brazdil (8): KVM: arm64: Rename .idmap.text in hyp linker script KVM: arm64: Set up .hyp.rodata ELF section KVM: arm64: Add symbol at the beginning of each hyp section KVM: arm64: Generate hyp relocation data KVM: arm64: Apply hyp relocations at runtime KVM: arm64: Fix constant-pool users in hyp KVM: arm64: Remove patching of fn pointers in hyp KVM: arm64: Remove hyp_symbol_addr Jamie Iles (1): KVM: arm64: Correctly align nVHE percpu data arch/arm64/configs/defconfig | 1 + arch/arm64/include/asm/hyp_image.h | 29 +- arch/arm64/include/asm/kvm_asm.h | 20 -- arch/arm64/include/asm/kvm_mmu.h | 61 ++--- arch/arm64/include/asm/sections.h| 3 +- arch/arm64/kernel/image-vars.h | 1 - arch/arm64/kernel/smp.c | 4 +- arch/arm64/kernel/vmlinux.lds.S | 18 +- arch/arm64/kvm/arm.c | 7 +- arch/arm64/kvm/hyp/include/hyp/switch.h | 4 +- arch/arm64/kvm/hyp/nvhe/Makefile | 28 +- arch/arm64/kvm/hyp/nvhe/gen-hyprel.c | 326 +++ arch/arm64/kvm/hyp/nvhe/host.S | 29 +- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 4 +- arch/arm64/kvm/hyp/nvhe/hyp-main.c | 11 +- arch/arm64/kvm/hyp/nvhe/hyp-smp.c| 4 +- arch/arm64/kvm/hyp/nvhe/hyp.lds.S| 14 +- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 24 +- arch/arm64/kvm/hyp/vgic-v2-cpuif-proxy.c | 2 +- arch/arm64/kvm/va_layout.c | 34 ++- 20 files changed, 495 insertions(+), 129 deletions(-) create mode 100644 arch/arm64/kvm/hyp/nvhe/gen-hyprel.c -- 2.29.2.576.ga3fc446d84-goog
[PATCH 3/9] KVM: arm64: Set up .hyp.rodata ELF section
We will need to recognize pointers in .rodata specific to hyp, so
establish a .hyp.rodata ELF section. Merge it with the existing
.hyp.data..ro_after_init as they are treated the same at runtime.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/sections.h | 2 +-
arch/arm64/kernel/vmlinux.lds.S | 7 ---
arch/arm64/kvm/arm.c | 7 +++
arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 4 +++-
4 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/sections.h
b/arch/arm64/include/asm/sections.h
index 8ff579361731..a6f3557d1ab2 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -11,7 +11,7 @@ extern char __alt_instructions[], __alt_instructions_end[];
extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[];
extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
extern char __hyp_text_start[], __hyp_text_end[];
-extern char __hyp_data_ro_after_init_start[], __hyp_data_ro_after_init_end[];
+extern char __hyp_rodata_start[], __hyp_rodata_end[];
extern char __idmap_text_start[], __idmap_text_end[];
extern char __initdata_begin[], __initdata_end[];
extern char __inittext_begin[], __inittext_end[];
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 43af13968dfd..f294f2048955 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -31,10 +31,11 @@ jiffies = jiffies_64;
__stop___kvm_ex_table = .;
#define HYPERVISOR_DATA_SECTIONS \
- HYP_SECTION_NAME(.data..ro_after_init) : { \
- __hyp_data_ro_after_init_start = .; \
+ HYP_SECTION_NAME(.rodata) : { \
+ __hyp_rodata_start = .; \
*(HYP_SECTION_NAME(.data..ro_after_init)) \
- __hyp_data_ro_after_init_end = .; \
+ *(HYP_SECTION_NAME(.rodata))\
+ __hyp_rodata_end = .; \
}
#define HYPERVISOR_PERCPU_SECTION \
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 6e637d2b4cfb..c244e57f9cd9 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1745,11 +1745,10 @@ static int init_hyp_mode(void)
goto out_err;
}
- err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_ro_after_init_start),
- kvm_ksym_ref(__hyp_data_ro_after_init_end),
- PAGE_HYP_RO);
+ err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
+ kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
if (err) {
- kvm_err("Cannot map .hyp.data..ro_after_init section\n");
+ kvm_err("Cannot map .hyp.rodata section\n");
goto out_err;
}
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index 70ac48ccede7..cfdc59b4329b 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -14,6 +14,9 @@
SECTIONS {
HYP_SECTION(.idmap.text)
HYP_SECTION(.text)
+ HYP_SECTION(.data..ro_after_init)
+ HYP_SECTION(.rodata)
+
/*
* .hyp..data..percpu needs to be page aligned to maintain the same
* alignment for when linking into vmlinux.
@@ -22,5 +25,4 @@ SECTIONS {
HYP_SECTION_NAME(.data..percpu) : {
PERCPU_INPUT(L1_CACHE_BYTES)
}
- HYP_SECTION(.data..ro_after_init)
}
--
2.29.2.576.ga3fc446d84-goog
[PATCH 4/9] KVM: arm64: Add symbol at the beginning of each hyp section
Generating hyp relocations will require referencing positions at a given
offset from the beginning of hyp sections. Since the final layout will
not be determined until the linking of `vmlinux`, modify the hyp linker
script to insert a symbol at the first byte of each hyp section to use
as an anchor. The linker of `vmlinux` will place the symbols together
with the sections.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/hyp_image.h | 29 +++--
arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 4 ++--
2 files changed, 29 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/include/asm/hyp_image.h
b/arch/arm64/include/asm/hyp_image.h
index daa1a1da539e..65e8008da932 100644
--- a/arch/arm64/include/asm/hyp_image.h
+++ b/arch/arm64/include/asm/hyp_image.h
@@ -7,6 +7,9 @@
#ifndef __ARM64_HYP_IMAGE_H__
#define __ARM64_HYP_IMAGE_H__
+#define HYP_CONCAT(a, b) __HYP_CONCAT(a, b)
+#define __HYP_CONCAT(a, b) a ## b
+
/*
* KVM nVHE code has its own symbol namespace prefixed with __kvm_nvhe_,
* to separate it from the kernel proper.
@@ -21,9 +24,31 @@
*/
#define HYP_SECTION_NAME(NAME) .hyp##NAME
+/* Symbol defined at the beginning of each hyp section. */
+#define HYP_SECTION_SYMBOL_NAME(NAME) \
+ HYP_CONCAT(__hyp_section_, HYP_SECTION_NAME(NAME))
+
+/*
+ * Helper to generate linker script statements starting a hyp section.
+ *
+ * A symbol with a well-known name is defined at the first byte. This
+ * is used as a base for hyp relocations (see gen-hyprel.c). It must
+ * be defined inside the section so the linker of `vmlinux` cannot
+ * separate it from the section data.
+ */
+#define BEGIN_HYP_SECTION(NAME)\
+ HYP_SECTION_NAME(NAME) : { \
+ HYP_SECTION_SYMBOL_NAME(NAME) = .;
+
+/* Helper to generate linker script statements ending a hyp section. */
+#define END_HYP_SECTION\
+ }
+
/* Defines an ELF hyp section from input section @NAME and its subsections. */
-#define HYP_SECTION(NAME) \
- HYP_SECTION_NAME(NAME) : { *(NAME NAME##.*) }
+#define HYP_SECTION(NAME) \
+ BEGIN_HYP_SECTION(NAME) \
+ *(NAME NAME##.*)\
+ END_HYP_SECTION
/*
* Defines a linker script alias of a kernel-proper symbol referenced by
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index cfdc59b4329b..cd119d82d8e3 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -22,7 +22,7 @@ SECTIONS {
* alignment for when linking into vmlinux.
*/
. = ALIGN(PAGE_SIZE);
- HYP_SECTION_NAME(.data..percpu) : {
+ BEGIN_HYP_SECTION(.data..percpu)
PERCPU_INPUT(L1_CACHE_BYTES)
- }
+ END_HYP_SECTION
}
--
2.29.2.576.ga3fc446d84-goog
Re: [RFC PATCH 5/6] kvm: arm64: Fix constant-pool users in hyp
Hey, relized I never replied to this... On Tue, Nov 24, 2020 at 03:08:20PM +0100, Ard Biesheuvel wrote: > On Thu, 19 Nov 2020 at 17:26, David Brazdil wrote: > > > > Hyp code used to use absolute addressing via a constant pool to obtain > > the kernel VA of 3 symbols - panic, __hyp_panic_string and > > __kvm_handle_stub_hvc. This used to work because the kernel would > > relocate the addresses in the constant pool to kernel VA at boot and hyp > > would simply load them from there. > > > > Now that relocations are fixed up to point to hyp VAs, this does not > > work any longer. Rework the helpers to convert hyp VA to kernel VA / PA > > as needed. > > > > Ok, so the reason for the problem is that the literal exists inside > the HYP text, and all literals are fixed up using the HYP mapping, > even if they don't point to something that is mapped at HYP. Would it > make sense to simply disregard literals that point outside of the HYP > VA mapping? That would be possible - I'm about to post a v1 with build-time generation of these relocs and kernel symbols are easily recognizable as they're undefined in that ELF object. But I do worry that that would confuse people a lot. And if somebody referenced a kernel symbol in C (maybe not a use case, but...) they would get different results depending on which addressing mode the compiler picked. > > > Signed-off-by: David Brazdil > > --- > > arch/arm64/include/asm/kvm_mmu.h | 29 +++-- > > arch/arm64/kvm/hyp/nvhe/host.S | 29 +++-- > > 2 files changed, 34 insertions(+), 24 deletions(-) > > > > diff --git a/arch/arm64/include/asm/kvm_mmu.h > > b/arch/arm64/include/asm/kvm_mmu.h > > index 8cb8974ec9cc..0676ff2105bb 100644 > > --- a/arch/arm64/include/asm/kvm_mmu.h > > +++ b/arch/arm64/include/asm/kvm_mmu.h > > @@ -72,9 +72,14 @@ alternative_cb kvm_update_va_mask > > alternative_cb_end > > .endm > > > > +.macro hyp_pa reg, tmp > > + ldr_l \tmp, hyp_physvirt_offset > > + add \reg, \reg, \tmp > > +.endm > > + > > /* > > - * Convert a kernel image address to a PA > > - * reg: kernel address to be converted in place > > + * Convert a hypervisor VA to a kernel image address > > + * reg: hypervisor address to be converted in place > > * tmp: temporary register > > * > > * The actual code generation takes place in kvm_get_kimage_voffset, and > > @@ -82,18 +87,22 @@ alternative_cb_end > > * perform the register allocation (kvm_get_kimage_voffset uses the > > * specific registers encoded in the instructions). > > */ > > -.macro kimg_pa reg, tmp > > +.macro hyp_kimg reg, tmp > > + /* Convert hyp VA -> PA. */ > > + hyp_pa \reg, \tmp > > + > > + /* Load kimage_voffset. */ > > alternative_cb kvm_get_kimage_voffset > > - movz\tmp, #0 > > - movk\tmp, #0, lsl #16 > > - movk\tmp, #0, lsl #32 > > - movk\tmp, #0, lsl #48 > > + movz\tmp, #0 > > + movk\tmp, #0, lsl #16 > > + movk\tmp, #0, lsl #32 > > + movk\tmp, #0, lsl #48 > > alternative_cb_end > > > > - /* reg = __pa(reg) */ > > - sub \reg, \reg, \tmp > > + /* Convert PA -> kimg VA. */ > > + add \reg, \reg, \tmp > > .endm > > - > > + > > #else > > > > #include > > diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S > > index 596dd5ae8e77..bcb80d525d8c 100644 > > --- a/arch/arm64/kvm/hyp/nvhe/host.S > > +++ b/arch/arm64/kvm/hyp/nvhe/host.S > > @@ -74,27 +74,28 @@ SYM_FUNC_END(__host_enter) > > * void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, > > u64 par); > > */ > > SYM_FUNC_START(__hyp_do_panic) > > - /* Load the format arguments into x1-7 */ > > - mov x6, x3 > > - get_vcpu_ptr x7, x3 > > - > > - mrs x3, esr_el2 > > - mrs x4, far_el2 > > - mrs x5, hpfar_el2 > > - > > /* Prepare and exit to the host's panic funciton. */ > > mov lr, #(PSR_F_BIT | PSR_I_BIT | PSR_A_BIT | PSR_D_BIT |\ > > PSR_MODE_EL1h) > > msr spsr_el2, lr > > ldr lr, =panic > > + hyp_kimg lr, x6 > > msr elr_el2, lr > > > > - /* > > -* Set the panic format string and enter the host, conditionally > > -* restoring the host context.
Re: [PATCH v4 00/26] Opt-in always-on nVHE hypervisor
Hey Marc, On Thu, Dec 03, 2020 at 07:23:19PM +, Marc Zyngier wrote: > On Wed, 2 Dec 2020 18:40:56 +0000, David Brazdil wrote: > > As we progress towards being able to keep guest state private to the > > host running nVHE hypervisor, this series allows the hypervisor to > > install itself on newly booted CPUs before the host is allowed to run > > on them. > > > > All functionality described below is opt-in, guarded by an early param > > 'kvm-arm.mode=protected'. Future patches specific to the new protected > > mode should be hidden behind the same param. > > > > [...] > > Applied to kvm-arm64/psci-relay, thanks! > > Note that although I pushed it to -next, I still need people to > eyeball it and give it some Acks. The commit-IDs below will > thus change as I apply tags, if any. > I'm looking at -next and I think the merge with Mark Rutland's el2_setup refactor didn't go as planned. The `#ifdef CONFIG_ARM64_VHE` section needs to cover everything between init_el2 and init_el2_nvhe. Currently the code falls through into VHE init when CONFIG_ARM64_VHE is not set. Here's the snippet: SYM_INNER_LABEL(init_el2, SYM_L_LOCAL) #ifdef CONFIG_ARM64_VHE /* * Check for VHE being present. x2 being non-zero indicates that we * do have VHE, and that the kernel is intended to run at EL2. */ mrs x2, id_aa64mmfr1_el1 ubfxx2, x2, #ID_AA64MMFR1_VHE_SHIFT, #4 cbz x2, init_el2_nvhe #endif // <--- THIS <... initialize VHE ...> msr elr_el2, lr mov w0, #BOOT_CPU_MODE_EL2 eret // <--- NEEDS TO MOVE HERE SYM_INNER_LABEL(init_el2_nvhe, SYM_L_LOCAL) <... initialize nVHE ...> -David
[PATCH 6/6] kvm: arm64: Move skip_host_instruction to adjust_pc.h
Move function for skipping host instruction in the host trap handler to
a header file containing analogical helpers for guests.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/include/hyp/adjust_pc.h | 9 +
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 12 ++--
2 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h
b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h
index b1f60923a8fe..61716359035d 100644
--- a/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h
+++ b/arch/arm64/kvm/hyp/include/hyp/adjust_pc.h
@@ -59,4 +59,13 @@ static inline void __adjust_pc(struct kvm_vcpu *vcpu)
}
}
+/*
+ * Skip an instruction while host sysregs are live.
+ * Assumes host is always 64-bit.
+ */
+static inline void kvm_skip_host_instr(void)
+{
+ write_sysreg_el2(read_sysreg_el2(SYS_ELR) + 4, SYS_ELR);
+}
+
#endif
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index bde658d51404..a906f9e2ff34 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -157,11 +157,6 @@ static void default_host_smc_handler(struct
kvm_cpu_context *host_ctxt)
__kvm_hyp_host_forward_smc(host_ctxt);
}
-static void skip_host_instruction(void)
-{
- write_sysreg_el2(read_sysreg_el2(SYS_ELR) + 4, SYS_ELR);
-}
-
static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
{
bool handled;
@@ -170,11 +165,8 @@ static void handle_host_smc(struct kvm_cpu_context
*host_ctxt)
if (!handled)
default_host_smc_handler(host_ctxt);
- /*
-* Unlike HVC, the return address of an SMC is the instruction's PC.
-* Move the return address past the instruction.
-*/
- skip_host_instruction();
+ /* SMC was trapped, move ELR past the current PC. */
+ kvm_skip_host_instr();
}
void handle_trap(struct kvm_cpu_context *host_ctxt)
--
2.29.2.576.ga3fc446d84-goog
[PATCH 5/6] kvm: arm64: Remove unused includes in psci-relay.c
Minor cleanup removing unused includes. Signed-off-by: David Brazdil --- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c b/arch/arm64/kvm/hyp/nvhe/psci-relay.c index 0d6f4aa39621..1f7237e45148 100644 --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c @@ -7,11 +7,8 @@ #include #include #include -#include #include #include -#include -#include #include #include -- 2.29.2.576.ga3fc446d84-goog
[PATCH 3/6] kvm: arm64: Skip computing hyp VA layout for VHE
Computing the hyp VA layout is redundant when the kernel runs in EL2 and
hyp shares its VA mappings. Make calling kvm_compute_layout()
conditional on not just CONFIG_KVM but also !is_kernel_in_hyp_mode().
Signed-off-by: David Brazdil
---
arch/arm64/kernel/smp.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 18e9727d3f64..4e585cc892e8 100644
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -434,7 +434,7 @@ static void __init hyp_mode_check(void)
"CPU: CPUs started in inconsistent modes");
else
pr_info("CPU: All CPU(s) started at EL1\n");
- if (IS_ENABLED(CONFIG_KVM))
+ if (IS_ENABLED(CONFIG_KVM) && !is_kernel_in_hyp_mode())
kvm_compute_layout();
}
--
2.29.2.576.ga3fc446d84-goog
[PATCH 2/6] kvm: arm64: Use lm_alias in nVHE-only VA conversion
init_hyp_physvirt_offset() computes PA from a kernel VA. Conversion to kernel linear-map is required first but the code used kvm_ksym_ref() for this purpose. Under VHE that is a NOP and resulted in a runtime warning. Replace kvm_ksym_ref with lm_alias. Reported-by: Qian Cai Signed-off-by: David Brazdil --- arch/arm64/kvm/va_layout.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c index d8cc51bd60bf..914732b88c69 100644 --- a/arch/arm64/kvm/va_layout.c +++ b/arch/arm64/kvm/va_layout.c @@ -42,7 +42,7 @@ static void init_hyp_physvirt_offset(void) u64 kern_va, hyp_va; /* Compute the offset from the hyp VA and PA of a random symbol. */ - kern_va = (u64)kvm_ksym_ref(__hyp_text_start); + kern_va = (u64)lm_alias(__hyp_text_start); hyp_va = __early_kern_hyp_va(kern_va); CHOOSE_NVHE_SYM(hyp_physvirt_offset) = (s64)__pa(kern_va) - (s64)hyp_va; } -- 2.29.2.576.ga3fc446d84-goog
[PATCH 1/6] kvm: arm64: Prevent use of invalid PSCI v0.1 function IDs
PSCI driver exposes a struct containing the PSCI v0.1 function IDs
configured in the DT. However, the struct does not convey the
information whether these were set from DT or contain the default value
zero. This could be a problem for PSCI proxy in KVM protected mode.
Extend config passed to KVM with a bit mask with individual bits set
depending on whether the corresponding function pointer in psci_ops is
set, eg. set bit for PSCI_CPU_SUSPEND if psci_ops.cpu_suspend != NULL.
Previously config was split into multiple global variables. Put
everything into a single struct for convenience.
Reported-by: Mark Rutland
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_host.h| 20 +++
arch/arm64/kvm/arm.c | 14 +---
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 53 +---
3 files changed, 70 insertions(+), 17 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h
b/arch/arm64/include/asm/kvm_host.h
index 11beda85ee7e..828d50d40dc2 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -17,6 +17,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -240,6 +241,25 @@ struct kvm_host_data {
struct kvm_pmu_events pmu_events;
};
+#define KVM_HOST_PSCI_0_1_CPU_SUSPEND BIT(0)
+#define KVM_HOST_PSCI_0_1_CPU_ON BIT(1)
+#define KVM_HOST_PSCI_0_1_CPU_OFF BIT(2)
+#define KVM_HOST_PSCI_0_1_MIGRATE BIT(3)
+
+struct kvm_host_psci_config {
+ /* PSCI version used by host. */
+ u32 version;
+
+ /* Function IDs used by host if version is v0.1. */
+ struct psci_0_1_function_ids function_ids_0_1;
+
+ /* Bitmask of functions enabled for v0.1, bits KVM_HOST_PSCI_0_1_*. */
+ unsigned int enabled_functions_0_1;
+};
+
+extern struct kvm_host_psci_config kvm_nvhe_sym(kvm_host_psci_config);
+#define kvm_host_psci_config CHOOSE_NVHE_SYM(kvm_host_psci_config)
+
struct vcpu_reset_state {
unsigned long pc;
unsigned long r0;
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 6e637d2b4cfb..6a2f4e01b04f 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -66,8 +66,6 @@ static DEFINE_PER_CPU(unsigned char,
kvm_arm_hardware_enabled);
DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
extern u64 kvm_nvhe_sym(__cpu_logical_map)[NR_CPUS];
-extern u32 kvm_nvhe_sym(kvm_host_psci_version);
-extern struct psci_0_1_function_ids
kvm_nvhe_sym(kvm_host_psci_0_1_function_ids);
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
{
@@ -1618,8 +1616,16 @@ static bool init_psci_relay(void)
return false;
}
- kvm_nvhe_sym(kvm_host_psci_version) = psci_ops.get_version();
- kvm_nvhe_sym(kvm_host_psci_0_1_function_ids) =
get_psci_0_1_function_ids();
+ kvm_host_psci_config.version = psci_ops.get_version();
+
+ if (kvm_host_psci_config.version == PSCI_VERSION(0, 1)) {
+ kvm_host_psci_config.function_ids_0_1 =
get_psci_0_1_function_ids();
+ kvm_host_psci_config.enabled_functions_0_1 =
+ (psci_ops.cpu_suspend ? KVM_HOST_PSCI_0_1_CPU_SUSPEND :
0) |
+ (psci_ops.cpu_off ? KVM_HOST_PSCI_0_1_CPU_OFF : 0) |
+ (psci_ops.cpu_on ? KVM_HOST_PSCI_0_1_CPU_ON : 0) |
+ (psci_ops.migrate ? KVM_HOST_PSCI_0_1_MIGRATE : 0);
+ }
return true;
}
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 08dc9de69314..0d6f4aa39621 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -22,9 +22,8 @@ void kvm_hyp_cpu_resume(unsigned long r0);
void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
/* Config options set by the host. */
-__ro_after_init u32 kvm_host_psci_version;
-__ro_after_init struct psci_0_1_function_ids kvm_host_psci_0_1_function_ids;
-__ro_after_init s64 hyp_physvirt_offset;
+struct kvm_host_psci_config __ro_after_init kvm_host_psci_config;
+s64 __ro_after_init hyp_physvirt_offset;
#define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset)
@@ -54,12 +53,41 @@ static u64 get_psci_func_id(struct kvm_cpu_context
*host_ctxt)
return func_id;
}
+static inline bool is_psci_0_1_function_enabled(unsigned int fn_bit)
+{
+ return kvm_host_psci_config.enabled_functions_0_1 & fn_bit;
+}
+
+static inline bool is_psci_0_1_cpu_suspend(u64 func_id)
+{
+ return is_psci_0_1_function_enabled(KVM_HOST_PSCI_0_1_CPU_SUSPEND) &&
+ (func_id == kvm_host_psci_config.function_ids_0_1.cpu_suspend);
+}
+
+static inline bool is_psci_0_1_cpu_on(u64 func_id)
+{
+ return is_psci_0_1_function_enabled(KVM_HOST_PSCI_0_1_CPU_ON) &&
+ (func_id == kvm_host_psci_config.function_ids_0_1.cpu_on);
+}
+
+static inline bool is_psci_0_1_cpu_off(u64 func_id)
+{
+ return is_psci_0_1_function_enabled(KVM_H
[PATCH 4/6] kvm: arm64: Minor cleanup of hyp variables used in host
Small cleanup moving declarations of hyp-exported variables to
kvm_host.h and using macros to avoid having to refer to them with
kvm_nvhe_sym() in host.
No functional change intended.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_host.h | 6 ++
arch/arm64/kvm/arm.c | 4 +---
arch/arm64/kvm/hyp/nvhe/hyp-smp.c | 6 +++---
arch/arm64/kvm/va_layout.c| 5 ++---
4 files changed, 12 insertions(+), 9 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_host.h
b/arch/arm64/include/asm/kvm_host.h
index 828d50d40dc2..bce2452b305c 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -260,6 +260,12 @@ struct kvm_host_psci_config {
extern struct kvm_host_psci_config kvm_nvhe_sym(kvm_host_psci_config);
#define kvm_host_psci_config CHOOSE_NVHE_SYM(kvm_host_psci_config)
+extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
+#define hyp_physvirt_offset CHOOSE_NVHE_SYM(hyp_physvirt_offset)
+
+extern u64 kvm_nvhe_sym(hyp_cpu_logical_map)[NR_CPUS];
+#define hyp_cpu_logical_map CHOOSE_NVHE_SYM(hyp_cpu_logical_map)
+
struct vcpu_reset_state {
unsigned long pc;
unsigned long r0;
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 6a2f4e01b04f..836ca763b91d 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -65,8 +65,6 @@ static bool vgic_present;
static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
-extern u64 kvm_nvhe_sym(__cpu_logical_map)[NR_CPUS];
-
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
{
return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
@@ -1602,7 +1600,7 @@ static void init_cpu_logical_map(void)
* allow any other CPUs from the `possible` set to boot.
*/
for_each_online_cpu(cpu)
- kvm_nvhe_sym(__cpu_logical_map)[cpu] = cpu_logical_map(cpu);
+ hyp_cpu_logical_map[cpu] = cpu_logical_map(cpu);
}
static bool init_psci_relay(void)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
index cbab0c6246e2..2997aa156d8e 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
@@ -14,14 +14,14 @@
* Other CPUs should not be allowed to boot because their features were
* not checked against the finalized system capabilities.
*/
-u64 __ro_after_init __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] =
INVALID_HWID };
+u64 __ro_after_init hyp_cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] =
INVALID_HWID };
u64 cpu_logical_map(unsigned int cpu)
{
- if (cpu >= ARRAY_SIZE(__cpu_logical_map))
+ if (cpu >= ARRAY_SIZE(hyp_cpu_logical_map))
hyp_panic();
- return __cpu_logical_map[cpu];
+ return hyp_cpu_logical_map[cpu];
}
unsigned long __hyp_per_cpu_offset(unsigned int cpu)
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index 914732b88c69..70fcd6a12fe1 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -34,17 +34,16 @@ static u64 __early_kern_hyp_va(u64 addr)
}
/*
- * Store a hyp VA <-> PA offset into a hyp-owned variable.
+ * Store a hyp VA <-> PA offset into a EL2-owned variable.
*/
static void init_hyp_physvirt_offset(void)
{
- extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
u64 kern_va, hyp_va;
/* Compute the offset from the hyp VA and PA of a random symbol. */
kern_va = (u64)lm_alias(__hyp_text_start);
hyp_va = __early_kern_hyp_va(kern_va);
- CHOOSE_NVHE_SYM(hyp_physvirt_offset) = (s64)__pa(kern_va) - (s64)hyp_va;
+ hyp_physvirt_offset = (s64)__pa(kern_va) - (s64)hyp_va;
}
/*
--
2.29.2.576.ga3fc446d84-goog
[PATCH 0/6] Fixes and cleanups of PSCI relay for kvmarm/next
Small batch of improvements for the 'Opt-in always-on nVHE hypervisor' series, now merged in kvmarm/next. Patch #1 fixes potential use of invalid v0.1 functions IDs reported by Mark Rutland, patch #2 fixes a warning reported by Qian Cai. Patch #3 avoids a code path not used in VHE, can be dropped if any concerns arise. The remaining patches are minor cleanups from review. -David David Brazdil (6): kvm: arm64: Prevent use of invalid PSCI v0.1 function IDs kvm: arm64: Use lm_alias in nVHE-only VA conversion kvm: arm64: Skip computing hyp VA layout for VHE kvm: arm64: Minor cleanup of hyp variables used in host kvm: arm64: Remove unused includes in psci-relay.c kvm: arm64: Move skip_host_instruction to adjust_pc.h arch/arm64/include/asm/kvm_host.h | 26 ++ arch/arm64/kernel/smp.c| 2 +- arch/arm64/kvm/arm.c | 18 --- arch/arm64/kvm/hyp/include/hyp/adjust_pc.h | 9 arch/arm64/kvm/hyp/nvhe/hyp-main.c | 12 + arch/arm64/kvm/hyp/nvhe/hyp-smp.c | 6 +-- arch/arm64/kvm/hyp/nvhe/psci-relay.c | 56 +++--- arch/arm64/kvm/va_layout.c | 7 ++- 8 files changed, 95 insertions(+), 41 deletions(-) -- 2.29.2.576.ga3fc446d84-goog
Re: [PATCH v4 17/26] kvm: arm64: Add offset for hyp VA <-> PA conversion
Hi Qian,
> > +/*
> > + * Store a hyp VA <-> PA offset into a hyp-owned variable.
> > + */
> > +static void init_hyp_physvirt_offset(void)
> > +{
> > + extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
> > + u64 kern_va, hyp_va;
> > +
> > + /* Compute the offset from the hyp VA and PA of a random symbol. */
> > + kern_va = (u64)kvm_ksym_ref(__hyp_text_start);
> > + hyp_va = __early_kern_hyp_va(kern_va);
> > + CHOOSE_NVHE_SYM(hyp_physvirt_offset) = (s64)__pa(kern_va) - (s64)hyp_va;
>
> The code here introduced a warning on TX2 from today's linux-next.
>
> .config:
> https://cailca.coding.net/public/linux/mm/git/files/master/arm64.config
>
> [ 29.356963] CPU255: Booted secondary processor 0x011f03 [0x431f0af1]
> [ 29.358301] smp: Brought up 2 nodes, 256 CPUs
> [ 29.364962] SMP: Total of 256 processors activated.
> [ 29.364985] CPU features: detected: Privileged Access Never
> [ 29.365003] CPU features: detected: LSE atomic instructions
> [ 29.365023] CPU features: detected: CRC32 instructions
> [ 29.431660] CPU: All CPU(s) started at EL2
> [ 29.431685] [ cut here ]
> [ 29.431713] virt_to_phys used for non-linear address: (ptrval)
> (__hyp_idmap_text_end+0x0/0x534)
> [ 29.431744] WARNING: CPU: 0 PID: 1 at arch/arm64/mm/physaddr.c:15
> __virt_to_phys+0x80/0xc0
> [ 29.431759] Modules linked in:
> [ 29.431787] CPU: 0 PID: 1 Comm: swapper/0 Not tainted
> 5.10.0-rc6-next-20201207+ #2
> [ 29.431804] pstate: 1049 (nzcV daif +PAN -UAO -TCO BTYPE=--)
> [ 29.431819] pc : __virt_to_phys+0x80/0xc0
> [ 29.431834] lr : __virt_to_phys+0x80/0xc0
> [ 29.431848] sp : 05fefc90
> [ 29.431862] x29: 05fefc90 x28: 8000191c9010
> [ 29.431891] x27: 05f21228 x26: b14e19fe279ae3eb
> [ 29.431920] x25: 8000191c9010 x24: 8000191c9000
> [ 29.431948] x23: 8000191c9000 x22: 000f800011235acc
> [ 29.431975] x21: 0001 x20: 000f8000
> [ 29.432003] x19: 800011235acc x18: 6001cedcc336
> [ 29.432031] x17: 1308 x16: 0002
> [ 29.432058] x15: x14: 7261656e696c2d6e
> [ 29.432086] x13: 60bfdee7 x12: 1fffe0bfdee6
> [ 29.432113] x11: 1fffe0bfdee6 x10: 60bfdee6
> [ 29.432141] x9 : 80001020a928 x8 : 05fef737
> [ 29.432169] x7 : 0001 x6 : 60bfdee7
> [ 29.432196] x5 : 60bfdee7 x4 : 1fffe0bfdedc
> [ 29.432223] x3 : 1fffe0be4009 x2 : 60bfdf5c
> [ 29.432251] x1 : 8fd448c3d76ca800 x0 :
> [ 29.432279] Call trace:
> [ 29.432294] __virt_to_phys+0x80/0xc0
> [ 29.432312] kvm_compute_layout+0x21c/0x264
> init_hyp_physvirt_offset at arch/arm64/kvm/va_layout.c:47
> (inlined by) kvm_compute_layout at arch/arm64/kvm/va_layout.c:82
> [ 29.432327] smp_cpus_done+0x164/0x17c
> [ 29.432342] smp_init+0xc4/0xd8
> [ 29.432358] kernel_init_freeable+0x4ec/0x734
> [ 29.432375] kernel_init+0x18/0x12c
> [ 29.432391] ret_from_fork+0x10/0x1c
> [ 29.432405] irq event stamp: 490612
> [ 29.432424] hardirqs last enabled at (490611): []
> console_unlock+0x8e0/0xca0
> [ 29.432440] hardirqs last disabled at (490612): []
> el1_dbg+0x24/0x50
> [ 29.432455] softirqs last enabled at (487946): []
> _stext+0xa98/0x113c
> [ 29.432473] softirqs last disabled at (487939): []
> irq_exit+0x500/0x5e0
> [ 29.432492] ---[ end trace 96247b4cbbdf9333 ]---
>
Thanks for the report, I've been able to reproduce with CONFIG_DEBUG_VIRTUAL=y
and the kernel running in EL2. Doesn't pose any functional issues as the results
of this are never used. I'll post a fix later today.
David
Re: [PATCH v4 06/26] psci: Add accessor for psci_0_1_function_ids
On Thu, Dec 03, 2020 at 10:47:12AM +, Mark Rutland wrote: > On Wed, Dec 02, 2020 at 06:41:02PM +0000, David Brazdil wrote: > > Make it possible to retrieve a copy of the psci_0_1_function_ids struct. > > This is useful for KVM if it is configured to intercept host's PSCI SMCs. > > > > Signed-off-by: David Brazdil > > Acked-by: Mark Rutland > > ... just to check, does KVM snapshot which function IDs are valid, or do > we want to add that state here too? That can be a follow-up if > necessary. Ah, that's a good point. It doesn't, but can infer that from psci_ops.
[PATCH v4 07/26] arm64: Make cpu_logical_map() take unsigned int
CPU index should never be negative. Change the signature of
(set_)cpu_logical_map to take an unsigned int.
This still works even if the users treat the CPU index as an int,
and will allow the hypervisor's implementation to check that the index
is valid with a single upper-bound check.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/smp.h | 4 ++--
arch/arm64/kernel/setup.c| 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index 2e7f529ec5a6..bcb01ca15325 100644
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -46,9 +46,9 @@ DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
* Logical CPU mapping.
*/
extern u64 __cpu_logical_map[NR_CPUS];
-extern u64 cpu_logical_map(int cpu);
+extern u64 cpu_logical_map(unsigned int cpu);
-static inline void set_cpu_logical_map(int cpu, u64 hwid)
+static inline void set_cpu_logical_map(unsigned int cpu, u64 hwid)
{
__cpu_logical_map[cpu] = hwid;
}
diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 133257ffd859..2f2973bc67c7 100644
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -276,7 +276,7 @@ arch_initcall(reserve_memblock_reserved_regions);
u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
-u64 cpu_logical_map(int cpu)
+u64 cpu_logical_map(unsigned int cpu)
{
return __cpu_logical_map[cpu];
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 08/26] arm64: Extract parts of el2_setup into a macro
When a CPU is booted in EL2, the kernel checks for VHE support and initializes the CPU core accordingly. For nVHE it also installs the stub vectors and drops down to EL1. Once KVM gains the ability to boot cores without going through the kernel entry point, it will need to initialize the CPU the same way. Extract the relevant bits of el2_setup into an init_el2_state macro with an argument specifying whether to initialize for VHE or nVHE. The following ifdefs are removed: * CONFIG_ARM_GIC_V3 - always selected on arm64 * CONFIG_COMPAT - hstr_el2 can be set even without 32-bit support No functional change intended. Size of el2_setup increased by 148 bytes due to duplication. Signed-off-by: David Brazdil --- arch/arm64/include/asm/el2_setup.h | 182 + arch/arm64/kernel/head.S | 144 +++ 2 files changed, 198 insertions(+), 128 deletions(-) create mode 100644 arch/arm64/include/asm/el2_setup.h diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h new file mode 100644 index ..b6cfd8aa2a81 --- /dev/null +++ b/arch/arm64/include/asm/el2_setup.h @@ -0,0 +1,182 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2012,2013 - ARM Ltd + * Author: Marc Zyngier + */ + +#ifndef __ARM_KVM_INIT_H__ +#define __ARM_KVM_INIT_H__ + +#ifndef __ASSEMBLY__ +#error Assembly-only header +#endif + +#include +#include +#include +#include + +.macro __init_el2_sctlr + mov_q x0, (SCTLR_EL2_RES1 | ENDIAN_SET_EL2) + msr sctlr_el2, x0 + isb +.endm + +/* + * Allow Non-secure EL1 and EL0 to access physical timer and counter. + * This is not necessary for VHE, since the host kernel runs in EL2, + * and EL0 accesses are configured in the later stage of boot process. + * Note that when HCR_EL2.E2H == 1, CNTHCTL_EL2 has the same bit layout + * as CNTKCTL_EL1, and CNTKCTL_EL1 accessing instructions are redefined + * to access CNTHCTL_EL2. This allows the kernel designed to run at EL1 + * to transparently mess with the EL0 bits via CNTKCTL_EL1 access in + * EL2. + */ +.macro __init_el2_timers mode +.ifeqs "\mode", "nvhe" + mrs x0, cnthctl_el2 + orr x0, x0, #3 // Enable EL1 physical timers + msr cnthctl_el2, x0 +.endif + msr cntvoff_el2, xzr// Clear virtual offset +.endm + +.macro __init_el2_debug mode + mrs x1, id_aa64dfr0_el1 + sbfxx0, x1, #ID_AA64DFR0_PMUVER_SHIFT, #4 + cmp x0, #1 + b.lt1f // Skip if no PMU present + mrs x0, pmcr_el0// Disable debug access traps + ubfxx0, x0, #11, #5 // to EL2 and allow access to +1: + cselx2, xzr, x0, lt // all PMU counters from EL1 + + /* Statistical profiling */ + ubfxx0, x1, #ID_AA64DFR0_PMSVER_SHIFT, #4 + cbz x0, 3f // Skip if SPE not present + +.ifeqs "\mode", "nvhe" + mrs_s x0, SYS_PMBIDR_EL1 // If SPE available at EL2, + and x0, x0, #(1 << SYS_PMBIDR_EL1_P_SHIFT) + cbnzx0, 2f // then permit sampling of physical + mov x0, #(1 << SYS_PMSCR_EL2_PCT_SHIFT | \ + 1 << SYS_PMSCR_EL2_PA_SHIFT) + msr_s SYS_PMSCR_EL2, x0 // addresses and physical counter +2: + mov x0, #(MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT) + orr x2, x2, x0 // If we don't have VHE, then + // use EL1&0 translation. +.else + orr x2, x2, #MDCR_EL2_TPMS // For VHE, use EL2 translation + // and disable access from EL1 +.endif + +3: + msr mdcr_el2, x2// Configure debug traps +.endm + +/* LORegions */ +.macro __init_el2_lor + mrs x1, id_aa64mmfr1_el1 + ubfxx0, x1, #ID_AA64MMFR1_LOR_SHIFT, 4 + cbz x0, 1f + msr_s SYS_LORC_EL1, xzr +1: +.endm + +/* Stage-2 translation */ +.macro __init_el2_stage2 + msr vttbr_el2, xzr +.endm + +/* GICv3 system register access */ +.macro __init_el2_gicv3 + mrs x0, id_aa64pfr0_el1 + ubfxx0, x0, #ID_AA64PFR0_GIC_SHIFT, #4 + cbz x0, 1f + + mrs_s x0, SYS_ICC_SRE_EL2 + orr x0, x0, #ICC_SRE_EL2_SRE// Set ICC_SRE_EL2.SRE==1 + orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1 + msr_s SYS_ICC_SRE_EL2, x0 + isb // Make sure SRE is now set + mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back, + tbz x0, #0, 1f // and check that it sticks + msr_s SYS_ICH_HCR_EL2, xzr// Reset ICC_HCR_EL2 to default
[PATCH v4 25/26] kvm: arm64: Trap host SMCs in protected mode
While protected KVM is installed, start trapping all host SMCs. For now these are simply forwarded to EL3, except PSCI CPU_ON/CPU_SUSPEND/SYSTEM_SUSPEND which are intercepted and the hypervisor installed on newly booted cores. Create new constant HCR_HOST_NVHE_PROTECTED_FLAGS with the new set of HCR flags to use while the nVHE vector is installed when the kernel was booted with the protected flag enabled. Switch back to the default HCR flags when switching back to the stub vector. Signed-off-by: David Brazdil --- arch/arm64/include/asm/kvm_arm.h | 1 + arch/arm64/kvm/hyp/nvhe/hyp-init.S | 10 ++ arch/arm64/kvm/hyp/nvhe/switch.c | 5 - 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 64ce29378467..4e90c2debf70 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -80,6 +80,7 @@ HCR_FMO | HCR_IMO | HCR_PTW ) #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF) #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA) +#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) /* TCR_EL2 Registers bits */ diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index a2e251547625..31b060a44045 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -88,6 +88,11 @@ SYM_CODE_END(__kvm_hyp_init) * x0: struct kvm_nvhe_init_params PA */ SYM_CODE_START_LOCAL(___kvm_hyp_init) +alternative_if ARM64_KVM_PROTECTED_MODE + mov_q x1, HCR_HOST_NVHE_PROTECTED_FLAGS + msr hcr_el2, x1 +alternative_else_nop_endif + ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] msr tpidr_el2, x1 @@ -230,6 +235,11 @@ reset: msr sctlr_el2, x5 isb +alternative_if ARM64_KVM_PROTECTED_MODE + mov_q x5, HCR_HOST_NVHE_FLAGS + msr hcr_el2, x5 +alternative_else_nop_endif + /* Install stub vectors */ adr_l x5, __hyp_stub_vectors msr vbar_el2, x5 diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 3e50ff35aa4f..f3d0e9eca56c 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -97,7 +97,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT; write_sysreg(mdcr_el2, mdcr_el2); - write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2); + if (is_protected_kvm_enabled()) + write_sysreg(HCR_HOST_NVHE_PROTECTED_FLAGS, hcr_el2); + else + write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2); write_sysreg(CPTR_EL2_DEFAULT, cptr_el2); write_sysreg(__kvm_hyp_host_vector, vbar_el2); } -- 2.29.2.454.gaff20da3a2-goog
[PATCH v4 24/26] kvm: arm64: Keep nVHE EL2 vector installed
KVM by default keeps the stub vector installed and installs the nVHE
vector only briefly for init and later on demand. Change this policy
to install the vector at init and then never uninstall it if the kernel
was given the protected KVM command line parameter.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/arm.c | 12
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index fda7bfa440d5..5f8776eefe46 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1531,7 +1531,8 @@ static void _kvm_arch_hardware_disable(void *discard)
void kvm_arch_hardware_disable(void)
{
- _kvm_arch_hardware_disable(NULL);
+ if (!is_protected_kvm_enabled())
+ _kvm_arch_hardware_disable(NULL);
}
#ifdef CONFIG_CPU_PM
@@ -1574,11 +1575,13 @@ static struct notifier_block hyp_init_cpu_pm_nb = {
static void __init hyp_cpu_pm_init(void)
{
- cpu_pm_register_notifier(_init_cpu_pm_nb);
+ if (!is_protected_kvm_enabled())
+ cpu_pm_register_notifier(_init_cpu_pm_nb);
}
static void __init hyp_cpu_pm_exit(void)
{
- cpu_pm_unregister_notifier(_init_cpu_pm_nb);
+ if (!is_protected_kvm_enabled())
+ cpu_pm_unregister_notifier(_init_cpu_pm_nb);
}
#else
static inline void hyp_cpu_pm_init(void)
@@ -1666,7 +1669,8 @@ static int init_subsystems(void)
kvm_sys_reg_table_init();
out:
- on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
+ if (err || !is_protected_kvm_enabled())
+ on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
return err;
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 26/26] kvm: arm64: Fix EL2 mode availability checks
With protected nVHE hyp code interception host's PSCI SMCs, the host
starts seeing new CPUs boot in EL1 instead of EL2. The kernel logic
that keeps track of the boot mode needs to be adjusted.
Add a static key enabled if KVM protected mode initialization is
successful.
When the key is enabled, is_hyp_mode_available continues to report
`true` because its users either treat it as a check whether KVM will be
/ was initialized, or whether stub HVCs can be made (eg. hibernate).
is_hyp_mode_mismatched is changed to report `false` when the key is
enabled. That's because all cores' modes matched at the point of KVM
init and KVM will not allow cores not present at init to boot. That
said, the function is never used after KVM is initialized.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/virt.h | 18 ++
arch/arm64/kvm/arm.c | 9 ++---
2 files changed, 24 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h
index eb81dcc220b6..ee6a48df89d9 100644
--- a/arch/arm64/include/asm/virt.h
+++ b/arch/arm64/include/asm/virt.h
@@ -65,9 +65,19 @@ extern u32 __boot_cpu_mode[2];
void __hyp_set_vectors(phys_addr_t phys_vector_base);
void __hyp_reset_vectors(void);
+DECLARE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
+
/* Reports the availability of HYP mode */
static inline bool is_hyp_mode_available(void)
{
+ /*
+* If KVM protected mode is initialized, all CPUs must have been booted
+* in EL2. Avoid checking __boot_cpu_mode as CPUs now come up in EL1.
+*/
+ if (IS_ENABLED(CONFIG_KVM) &&
+ static_branch_likely(_protected_mode_initialized))
+ return true;
+
return (__boot_cpu_mode[0] == BOOT_CPU_MODE_EL2 &&
__boot_cpu_mode[1] == BOOT_CPU_MODE_EL2);
}
@@ -75,6 +85,14 @@ static inline bool is_hyp_mode_available(void)
/* Check if the bootloader has booted CPUs in different modes */
static inline bool is_hyp_mode_mismatched(void)
{
+ /*
+* If KVM protected mode is initialized, all CPUs must have been booted
+* in EL2. Avoid checking __boot_cpu_mode as CPUs now come up in EL1.
+*/
+ if (IS_ENABLED(CONFIG_KVM) &&
+ static_branch_likely(_protected_mode_initialized))
+ return false;
+
return __boot_cpu_mode[0] != __boot_cpu_mode[1];
}
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 5f8776eefe46..6e637d2b4cfb 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -47,6 +47,7 @@ __asm__(".arch_extension virt");
#endif
static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
+DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
@@ -1926,12 +1927,14 @@ int kvm_arch_init(void *opaque)
if (err)
goto out_hyp;
- if (is_protected_kvm_enabled())
+ if (is_protected_kvm_enabled()) {
+ static_branch_enable(_protected_mode_initialized);
kvm_info("Protected nVHE mode initialized successfully\n");
- else if (in_hyp_mode)
+ } else if (in_hyp_mode) {
kvm_info("VHE mode initialized successfully\n");
- else
+ } else {
kvm_info("Hyp mode initialized successfully\n");
+ }
return 0;
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 22/26] kvm: arm64: Intercept host's CPU_SUSPEND PSCI SMCs
Add a handler of CPU_SUSPEND host PSCI SMCs. The SMC can either enter
a sleep state indistinguishable from a WFI or a deeper sleep state that
behaves like a CPU_OFF+CPU_ON except that the core is still considered
online while asleep.
The handler saves r0,pc of the host and makes the same call to EL3 with
the hyp CPU entry point. It either returns back to the handler and then
back to the host, or wakes up into the entry point and initializes EL2
state before dropping back to EL1. No EL2 state needs to be
saved/restored for this purpose.
CPU_ON and CPU_SUSPEND are both implemented using struct psci_boot_args
to store the state upon powerup, with each CPU having separate structs
for CPU_ON and CPU_SUSPEND so that CPU_SUSPEND can operate locklessly
and so that a CPU_ON call targeting a CPU cannot interfere with
a concurrent CPU_SUSPEND call on that CPU.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 10 +++
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 44 ++--
2 files changed, 52 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index d07e75f8242e..0853f62b052b 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -150,6 +150,16 @@ SYM_CODE_START(kvm_hyp_cpu_entry)
b __kvm_hyp_init_cpu
SYM_CODE_END(kvm_hyp_cpu_entry)
+/*
+ * PSCI CPU_SUSPEND entry point
+ *
+ * x0: struct kvm_nvhe_init_params PA
+ */
+SYM_CODE_START(kvm_hyp_cpu_resume)
+ mov x1, #0 // is_cpu_on = false
+ b __kvm_hyp_init_cpu
+SYM_CODE_END(kvm_hyp_cpu_resume)
+
/*
* Common code for CPU entry points. Initializes EL2 state and
* installs the hypervisor before handing over to a C handler.
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 637e22ed71fc..688cf7f40d42 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -17,6 +17,7 @@
#include
void kvm_hyp_cpu_entry(unsigned long r0);
+void kvm_hyp_cpu_resume(unsigned long r0);
void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
@@ -44,6 +45,7 @@ struct psci_boot_args {
})
static DEFINE_PER_CPU(struct psci_boot_args, cpu_on_args) =
PSCI_BOOT_ARGS_INIT;
+static DEFINE_PER_CPU(struct psci_boot_args, suspend_args) =
PSCI_BOOT_ARGS_INIT;
static u64 get_psci_func_id(struct kvm_cpu_context *host_ctxt)
{
@@ -171,17 +173,51 @@ static int psci_cpu_on(u64 func_id, struct
kvm_cpu_context *host_ctxt)
return ret;
}
+static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
+{
+ DECLARE_REG(u64, power_state, host_ctxt, 1);
+ DECLARE_REG(unsigned long, pc, host_ctxt, 2);
+ DECLARE_REG(unsigned long, r0, host_ctxt, 3);
+
+ struct psci_boot_args *boot_args;
+ struct kvm_nvhe_init_params *init_params;
+
+ boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));
+ init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params));
+
+ /*
+* No need to acquire a lock before writing to boot_args because a core
+* can only suspend itself. Racy CPU_ON calls use a separate struct.
+*/
+ boot_args->pc = pc;
+ boot_args->r0 = r0;
+
+ /*
+* Will either return if shallow sleep state, or wake up into the entry
+* point if it is a deep sleep state.
+*/
+ return psci_call(func_id, power_state,
+__hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)),
+__hyp_pa(init_params));
+}
+
asmlinkage void __noreturn kvm_host_psci_cpu_entry(bool is_cpu_on)
{
struct psci_boot_args *boot_args;
struct kvm_cpu_context *host_ctxt;
host_ctxt = _cpu_ptr(hyp_symbol_addr(kvm_host_data))->host_ctxt;
- boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_on_args));
+
+ if (is_cpu_on)
+ boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_on_args));
+ else
+ boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));
cpu_reg(host_ctxt, 0) = boot_args->r0;
write_sysreg_el2(boot_args->pc, SYS_ELR);
- release_boot_args(boot_args);
+
+ if (is_cpu_on)
+ release_boot_args(boot_args);
__host_enter(host_ctxt);
}
@@ -193,6 +229,8 @@ static unsigned long psci_0_1_handler(u64 func_id, struct
kvm_cpu_context *host_
return psci_forward(host_ctxt);
else if (func_id == kvm_host_psci_0_1_function_ids.cpu_on)
return psci_cpu_on(func_id, host_ctxt);
+ else if (func_id == kvm_host_psci_0_1_function_ids.cpu_suspend)
+ return psci_cpu_suspend(func_id, host_ctxt);
else
return PSCI_RET_NOT_SUPPORTED;
}
@@ -211,6 +249,8 @@ static unsigned long psci_0_2_handler(u64 func_id, struct
kvm_cpu_context *host_
case PSCI_0_2_
[PATCH v4 23/26] kvm: arm64: Intercept host's SYSTEM_SUSPEND PSCI SMCs
Add a handler of SYSTEM_SUSPEND host PSCI SMCs. The semantics are
equivalent to CPU_SUSPEND, typically called on the last online CPU.
Reuse the same entry point and boot args struct as CPU_SUSPEND.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 2 +-
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 26 ++
2 files changed, 27 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index 0853f62b052b..a2e251547625 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -151,7 +151,7 @@ SYM_CODE_START(kvm_hyp_cpu_entry)
SYM_CODE_END(kvm_hyp_cpu_entry)
/*
- * PSCI CPU_SUSPEND entry point
+ * PSCI CPU_SUSPEND / SYSTEM_SUSPEND entry point
*
* x0: struct kvm_nvhe_init_params PA
*/
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 688cf7f40d42..08dc9de69314 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -201,6 +201,30 @@ static int psci_cpu_suspend(u64 func_id, struct
kvm_cpu_context *host_ctxt)
__hyp_pa(init_params));
}
+static int psci_system_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
+{
+ DECLARE_REG(unsigned long, pc, host_ctxt, 1);
+ DECLARE_REG(unsigned long, r0, host_ctxt, 2);
+
+ struct psci_boot_args *boot_args;
+ struct kvm_nvhe_init_params *init_params;
+
+ boot_args = this_cpu_ptr(hyp_symbol_addr(suspend_args));
+ init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params));
+
+ /*
+* No need to acquire a lock before writing to boot_args because a core
+* can only suspend itself. Racy CPU_ON calls use a separate struct.
+*/
+ boot_args->pc = pc;
+ boot_args->r0 = r0;
+
+ /* Will only return on error. */
+ return psci_call(func_id,
+__hyp_pa(hyp_symbol_addr(kvm_hyp_cpu_resume)),
+__hyp_pa(init_params), 0);
+}
+
asmlinkage void __noreturn kvm_host_psci_cpu_entry(bool is_cpu_on)
{
struct psci_boot_args *boot_args;
@@ -265,6 +289,8 @@ static unsigned long psci_1_0_handler(u64 func_id, struct
kvm_cpu_context *host_
case PSCI_1_0_FN_SET_SUSPEND_MODE:
case PSCI_1_1_FN64_SYSTEM_RESET2:
return psci_forward(host_ctxt);
+ case PSCI_1_0_FN64_SYSTEM_SUSPEND:
+ return psci_system_suspend(func_id, host_ctxt);
default:
return psci_0_2_handler(func_id, host_ctxt);
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 13/26] kvm: arm64: Support per_cpu_ptr in nVHE hyp code
When compiling with __KVM_NVHE_HYPERVISOR__, redefine per_cpu_offset()
to __hyp_per_cpu_offset() which looks up the base of the nVHE per-CPU
region of the given cpu and computes its offset from the
.hyp.data..percpu section.
This enables use of per_cpu_ptr() helpers in nVHE hyp code. Until now
only this_cpu_ptr() was supported by setting TPIDR_EL2.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/percpu.h | 6 ++
arch/arm64/kernel/image-vars.h| 3 +++
arch/arm64/kvm/hyp/nvhe/Makefile | 3 ++-
arch/arm64/kvm/hyp/nvhe/hyp-smp.c | 24
4 files changed, 35 insertions(+), 1 deletion(-)
create mode 100644 arch/arm64/kvm/hyp/nvhe/hyp-smp.c
diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h
index 1599e17379d8..8f1661603b78 100644
--- a/arch/arm64/include/asm/percpu.h
+++ b/arch/arm64/include/asm/percpu.h
@@ -239,6 +239,12 @@ PERCPU_RET_OP(add, add, ldadd)
#define this_cpu_cmpxchg_8(pcp, o, n) \
_pcp_protect_return(cmpxchg_relaxed, pcp, o, n)
+#ifdef __KVM_NVHE_HYPERVISOR__
+extern unsigned long __hyp_per_cpu_offset(unsigned int cpu);
+#define __per_cpu_offset
+#define per_cpu_offset(cpu)__hyp_per_cpu_offset((cpu))
+#endif
+
#include
/* Redefine macros for nVHE hyp under DEBUG_PREEMPT to avoid its dependencies.
*/
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 08e69faedf6c..39289d75118d 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -99,6 +99,9 @@ KVM_NVHE_ALIAS(gic_nonsecure_priorities);
KVM_NVHE_ALIAS(__start___kvm_ex_table);
KVM_NVHE_ALIAS(__stop___kvm_ex_table);
+/* Array containing bases of nVHE per-CPU memory regions. */
+KVM_NVHE_ALIAS(kvm_arm_hyp_percpu_base);
+
#endif /* CONFIG_KVM */
#endif /* __ARM64_KERNEL_IMAGE_VARS_H */
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index 77b8c4e06f2f..cf11f8182756 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -6,7 +6,8 @@
asflags-y := -D__KVM_NVHE_HYPERVISOR__
ccflags-y := -D__KVM_NVHE_HYPERVISOR__
-obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o
hyp-main.o
+obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
+hyp-main.o hyp-smp.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o ../exception.o
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
new file mode 100644
index ..7b0363b4857f
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
@@ -0,0 +1,24 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2020 - Google LLC
+ * Author: David Brazdil
+ */
+
+#include
+#include
+#include
+
+unsigned long __hyp_per_cpu_offset(unsigned int cpu)
+{
+ unsigned long *cpu_base_array;
+ unsigned long this_cpu_base;
+ unsigned long elf_base;
+
+ if (cpu >= ARRAY_SIZE(kvm_arm_hyp_percpu_base))
+ hyp_panic();
+
+ cpu_base_array = (unsigned long
*)hyp_symbol_addr(kvm_arm_hyp_percpu_base);
+ this_cpu_base = kern_hyp_va(cpu_base_array[cpu]);
+ elf_base = (unsigned long)hyp_symbol_addr(__per_cpu_start);
+ return this_cpu_base - elf_base;
+}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 15/26] kvm: arm64: Add SMC handler in nVHE EL2
Add handler of host SMCs in KVM nVHE trap handler. Forward all SMCs to
EL3 and propagate the result back to EL1. This is done in preparation
for validating host SMCs in KVM protected mode.
The implementation assumes that firmware uses SMCCC v1.2 or older. That
means x0-x17 can be used both for arguments and results, other GPRs are
preserved.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/host.S | 38 ++
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 35 ---
2 files changed, 70 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index fe2740b224cf..2b56f0bdf874 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -180,3 +180,41 @@ SYM_CODE_START(__kvm_hyp_host_vector)
invalid_host_el1_vect // FIQ 32-bit EL1
invalid_host_el1_vect // Error 32-bit EL1
SYM_CODE_END(__kvm_hyp_host_vector)
+
+/*
+ * Forward SMC with arguments in struct kvm_cpu_context, and
+ * store the result into the same struct. Assumes SMCCC 1.2 or older.
+ *
+ * x0: struct kvm_cpu_context*
+ */
+SYM_CODE_START(__kvm_hyp_host_forward_smc)
+ /*
+* Use x18 to keep the pointer to the host context because
+* x18 is callee-saved in SMCCC but not in AAPCS64.
+*/
+ mov x18, x0
+
+ ldp x0, x1, [x18, #CPU_XREG_OFFSET(0)]
+ ldp x2, x3, [x18, #CPU_XREG_OFFSET(2)]
+ ldp x4, x5, [x18, #CPU_XREG_OFFSET(4)]
+ ldp x6, x7, [x18, #CPU_XREG_OFFSET(6)]
+ ldp x8, x9, [x18, #CPU_XREG_OFFSET(8)]
+ ldp x10, x11, [x18, #CPU_XREG_OFFSET(10)]
+ ldp x12, x13, [x18, #CPU_XREG_OFFSET(12)]
+ ldp x14, x15, [x18, #CPU_XREG_OFFSET(14)]
+ ldp x16, x17, [x18, #CPU_XREG_OFFSET(16)]
+
+ smc #0
+
+ stp x0, x1, [x18, #CPU_XREG_OFFSET(0)]
+ stp x2, x3, [x18, #CPU_XREG_OFFSET(2)]
+ stp x4, x5, [x18, #CPU_XREG_OFFSET(4)]
+ stp x6, x7, [x18, #CPU_XREG_OFFSET(6)]
+ stp x8, x9, [x18, #CPU_XREG_OFFSET(8)]
+ stp x10, x11, [x18, #CPU_XREG_OFFSET(10)]
+ stp x12, x13, [x18, #CPU_XREG_OFFSET(12)]
+ stp x14, x15, [x18, #CPU_XREG_OFFSET(14)]
+ stp x16, x17, [x18, #CPU_XREG_OFFSET(16)]
+
+ ret
+SYM_CODE_END(__kvm_hyp_host_forward_smc)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index a4f1cac714d7..f25680ede080 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -18,6 +18,8 @@
DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
+void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);
+
static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);
@@ -152,12 +154,39 @@ static void handle_host_hcall(struct kvm_cpu_context
*host_ctxt)
cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED;
}
+static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt)
+{
+ __kvm_hyp_host_forward_smc(host_ctxt);
+}
+
+static void skip_host_instruction(void)
+{
+ write_sysreg_el2(read_sysreg_el2(SYS_ELR) + 4, SYS_ELR);
+}
+
+static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
+{
+ default_host_smc_handler(host_ctxt);
+
+ /*
+* Unlike HVC, the return address of an SMC is the instruction's PC.
+* Move the return address past the instruction.
+*/
+ skip_host_instruction();
+}
+
void handle_trap(struct kvm_cpu_context *host_ctxt)
{
u64 esr = read_sysreg_el2(SYS_ESR);
- if (unlikely(ESR_ELx_EC(esr) != ESR_ELx_EC_HVC64))
+ switch (ESR_ELx_EC(esr)) {
+ case ESR_ELx_EC_HVC64:
+ handle_host_hcall(host_ctxt);
+ break;
+ case ESR_ELx_EC_SMC64:
+ handle_host_smc(host_ctxt);
+ break;
+ default:
hyp_panic();
-
- handle_host_hcall(host_ctxt);
+ }
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 11/26] kvm: arm64: Init MAIR/TCR_EL2 from params struct
MAIR_EL2 and TCR_EL2 are currently initialized from their _EL1 values.
This will not work once KVM starts intercepting PSCI ON/SUSPEND SMCs
and initializing EL2 state before EL1 state.
Obtain the EL1 values during KVM init and store them in the init params
struct. The struct will stay in memory and can be used when booting new
cores.
Take the opportunity to move copying the T0SZ value from idmap_t0sz in
KVM init rather than in .hyp.idmap.text. This avoids the need for the
idmap_t0sz symbol alias.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_asm.h | 2 ++
arch/arm64/kernel/asm-offsets.c| 2 ++
arch/arm64/kernel/image-vars.h | 3 ---
arch/arm64/kvm/arm.c | 22 +
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 38 +++---
5 files changed, 34 insertions(+), 33 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 1a7b91534a16..7ccf770c53d9 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -149,6 +149,8 @@ extern void *__vhe_undefined_symbol;
#endif
struct kvm_nvhe_init_params {
+ unsigned long mair_el2;
+ unsigned long tcr_el2;
unsigned long tpidr_el2;
unsigned long stack_hyp_va;
phys_addr_t pgd_pa;
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 8d6272a01a00..ba01185ef281 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -110,6 +110,8 @@ int main(void)
DEFINE(CPU_APGAKEYLO_EL1,offsetof(struct kvm_cpu_context,
sys_regs[APGAKEYLO_EL1]));
DEFINE(HOST_CONTEXT_VCPU,offsetof(struct kvm_cpu_context,
__hyp_running_vcpu));
DEFINE(HOST_DATA_CONTEXT,offsetof(struct kvm_host_data, host_ctxt));
+ DEFINE(NVHE_INIT_MAIR_EL2, offsetof(struct kvm_nvhe_init_params,
mair_el2));
+ DEFINE(NVHE_INIT_TCR_EL2,offsetof(struct kvm_nvhe_init_params, tcr_el2));
DEFINE(NVHE_INIT_TPIDR_EL2, offsetof(struct kvm_nvhe_init_params,
tpidr_el2));
DEFINE(NVHE_INIT_STACK_HYP_VA, offsetof(struct kvm_nvhe_init_params,
stack_hyp_va));
DEFINE(NVHE_INIT_PGD_PA, offsetof(struct kvm_nvhe_init_params, pgd_pa));
diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 4b32588918d9..08e69faedf6c 100644
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -77,9 +77,6 @@ KVM_NVHE_ALIAS(panic);
/* Vectors installed by hyp-init on reset HVC. */
KVM_NVHE_ALIAS(__hyp_stub_vectors);
-/* IDMAP TCR_EL1.T0SZ as computed by the EL1 init code */
-KVM_NVHE_ALIAS(idmap_t0sz);
-
/* Kernel symbol used by icache_is_vpipt(). */
KVM_NVHE_ALIAS(__icache_flags);
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 93a408c00249..6c8594378865 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1387,6 +1387,7 @@ static void cpu_init_hyp_mode(void)
{
struct kvm_nvhe_init_params *params =
this_cpu_ptr_nvhe_sym(kvm_init_params);
struct arm_smccc_res res;
+ unsigned long tcr;
/* Switch from the HYP stub to our own HYP init vector */
__hyp_set_vectors(kvm_get_idmap_vector());
@@ -1399,6 +1400,27 @@ static void cpu_init_hyp_mode(void)
params->tpidr_el2 = (unsigned
long)this_cpu_ptr_nvhe_sym(__per_cpu_start) -
(unsigned
long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
+ params->mair_el2 = read_sysreg(mair_el1);
+
+ /*
+* The ID map may be configured to use an extended virtual address
+* range. This is only the case if system RAM is out of range for the
+* currently configured page size and VA_BITS, in which case we will
+* also need the extended virtual range for the HYP ID map, or we won't
+* be able to enable the EL2 MMU.
+*
+* However, at EL2, there is only one TTBR register, and we can't switch
+* between translation tables *and* update TCR_EL2.T0SZ at the same
+* time. Bottom line: we need to use the extended range with *both* our
+* translation tables.
+*
+* So use the same T0SZ value we use for the ID map.
+*/
+ tcr = (read_sysreg(tcr_el1) & TCR_EL2_MASK) | TCR_EL2_RES1;
+ tcr &= ~TCR_T0SZ_MASK;
+ tcr |= (idmap_t0sz & GENMASK(TCR_TxSZ_WIDTH - 1, 0)) << TCR_T0SZ_OFFSET;
+ params->tcr_el2 = tcr;
+
params->stack_hyp_va =
kern_hyp_va(__this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE);
params->pgd_pa = kvm_mmu_get_httbr();
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index e712e317337c..712f57289357 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -71,48 +71,26 @@ __do_hyp_init:
1: ldr x0, [x1, #NVHE_INIT_TPIDR_EL2]
msr tpidr_el2, x0
+ ldr x0, [x1, #NVHE_INIT_MAIR_EL2]
+ msr mair_
[PATCH v4 21/26] kvm: arm64: Intercept host's CPU_ON SMCs
Add a handler of the CPU_ON PSCI call from host. When invoked, it looks
up the logical CPU ID corresponding to the provided MPIDR and populates
the state struct of the target CPU with the provided x0, pc. It then
calls CPU_ON itself, with an entry point in hyp that initializes EL2
state before returning ERET to the provided PC in EL1.
There is a simple atomic lock around the boot args struct. If it is
already locked, CPU_ON will return PENDING_ON error code.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 48 +++
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 115 +++
2 files changed, 163 insertions(+)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index b0856b006bc0..d07e75f8242e 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -9,6 +9,7 @@
#include
#include
+#include
#include
#include
#include
@@ -139,6 +140,53 @@ alternative_else_nop_endif
ret
SYM_CODE_END(___kvm_hyp_init)
+/*
+ * PSCI CPU_ON entry point
+ *
+ * x0: struct kvm_nvhe_init_params PA
+ */
+SYM_CODE_START(kvm_hyp_cpu_entry)
+ mov x1, #1 // is_cpu_on = true
+ b __kvm_hyp_init_cpu
+SYM_CODE_END(kvm_hyp_cpu_entry)
+
+/*
+ * Common code for CPU entry points. Initializes EL2 state and
+ * installs the hypervisor before handing over to a C handler.
+ *
+ * x0: struct kvm_nvhe_init_params PA
+ * x1: bool is_cpu_on
+ */
+SYM_CODE_START_LOCAL(__kvm_hyp_init_cpu)
+ mov x28, x0 // Stash arguments
+ mov x29, x1
+
+ /* Check that the core was booted in EL2. */
+ mrs x0, CurrentEL
+ cmp x0, #CurrentEL_EL2
+ b.eq2f
+
+ /* The core booted in EL1. KVM cannot be initialized on it. */
+1: wfe
+ wfi
+ b 1b
+
+2: msr SPsel, #1 // We want to use SP_EL{1,2}
+
+ /* Initialize EL2 CPU state to sane values. */
+ init_el2_state nvhe // Clobbers x0..x2
+
+ /* Enable MMU, set vectors and stack. */
+ mov x0, x28
+ bl ___kvm_hyp_init // Clobbers x0..x3
+
+ /* Leave idmap. */
+ mov x0, x29
+ ldr x1, =kvm_host_psci_cpu_entry
+ kimg_hyp_va x1, x2
+ br x1
+SYM_CODE_END(__kvm_hyp_init_cpu)
+
SYM_CODE_START(__kvm_handle_stub_hvc)
cmp x0, #HVC_SOFT_RESTART
b.ne1f
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 5ad56a875ffa..637e22ed71fc 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -9,12 +9,17 @@
#include
#include
#include
+#include
#include
#include
#include
#include
+void kvm_hyp_cpu_entry(unsigned long r0);
+
+void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
+
/* Config options set by the host. */
__ro_after_init u32 kvm_host_psci_version;
__ro_after_init struct psci_0_1_function_ids kvm_host_psci_0_1_function_ids;
@@ -22,6 +27,24 @@ __ro_after_init s64 hyp_physvirt_offset;
#define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset)
+#define INVALID_CPU_ID UINT_MAX
+
+struct psci_boot_args {
+ atomic_t lock;
+ unsigned long pc;
+ unsigned long r0;
+};
+
+#define PSCI_BOOT_ARGS_UNLOCKED0
+#define PSCI_BOOT_ARGS_LOCKED 1
+
+#define PSCI_BOOT_ARGS_INIT\
+ ((struct psci_boot_args){ \
+ .lock = ATOMIC_INIT(PSCI_BOOT_ARGS_UNLOCKED), \
+ })
+
+static DEFINE_PER_CPU(struct psci_boot_args, cpu_on_args) =
PSCI_BOOT_ARGS_INIT;
+
static u64 get_psci_func_id(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(u64, func_id, host_ctxt, 0);
@@ -75,11 +98,101 @@ static __noreturn unsigned long
psci_forward_noreturn(struct kvm_cpu_context *ho
hyp_panic(); /* unreachable */
}
+static unsigned int find_cpu_id(u64 mpidr)
+{
+ unsigned int i;
+
+ /* Reject invalid MPIDRs */
+ if (mpidr & ~MPIDR_HWID_BITMASK)
+ return INVALID_CPU_ID;
+
+ for (i = 0; i < NR_CPUS; i++) {
+ if (cpu_logical_map(i) == mpidr)
+ return i;
+ }
+
+ return INVALID_CPU_ID;
+}
+
+static __always_inline bool try_acquire_boot_args(struct psci_boot_args *args)
+{
+ return atomic_cmpxchg_acquire(>lock,
+ PSCI_BOOT_ARGS_UNLOCKED,
+ PSCI_BOOT_ARGS_LOCKED) ==
+ PSCI_BOOT_ARGS_UNLOCKED;
+}
+
+static __always_inline void release_boot_args(struct psci_boot_args *args)
+{
+ atomic_set_release(>lock, PSCI_BOOT_ARGS_UNLOCKED);
+}
+
+static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt)
+{
+ DECLARE_REG(u64, mpidr, host_ctxt, 1);
+ DECLARE_REG(
[PATCH v4 19/26] kvm: arm64: Extract __do_hyp_init into a helper function
In preparation for adding a CPU entry point in nVHE hyp code, extract most of __do_hyp_init hypervisor initialization code into a common helper function. This will be invoked by the entry point to install KVM on the newly booted CPU. Signed-off-by: David Brazdil --- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 47 -- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 712f57289357..b0856b006bc0 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -68,17 +68,36 @@ __do_hyp_init: mov x0, #SMCCC_RET_NOT_SUPPORTED eret -1: ldr x0, [x1, #NVHE_INIT_TPIDR_EL2] - msr tpidr_el2, x0 +1: mov x0, x1 + mov x4, lr + bl ___kvm_hyp_init + mov lr, x4 - ldr x0, [x1, #NVHE_INIT_MAIR_EL2] - msr mair_el2, x0 + /* Hello, World! */ + mov x0, #SMCCC_RET_SUCCESS + eret +SYM_CODE_END(__kvm_hyp_init) + +/* + * Initialize the hypervisor in EL2. + * + * Only uses x0..x3 so as to not clobber callee-saved SMCCC registers + * and leave x4 for the caller. + * + * x0: struct kvm_nvhe_init_params PA + */ +SYM_CODE_START_LOCAL(___kvm_hyp_init) + ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] + msr tpidr_el2, x1 + + ldr x1, [x0, #NVHE_INIT_STACK_HYP_VA] + mov sp, x1 - ldr x0, [x1, #NVHE_INIT_STACK_HYP_VA] - mov sp, x0 + ldr x1, [x0, #NVHE_INIT_MAIR_EL2] + msr mair_el2, x1 - ldr x0, [x1, #NVHE_INIT_PGD_PA] - phys_to_ttbr x2, x0 + ldr x1, [x0, #NVHE_INIT_PGD_PA] + phys_to_ttbr x2, x1 alternative_if ARM64_HAS_CNP orr x2, x2, #TTBR_CNP_BIT alternative_else_nop_endif @@ -87,9 +106,9 @@ alternative_else_nop_endif /* * Set the PS bits in TCR_EL2. */ - ldr x0, [x1, #NVHE_INIT_TCR_EL2] - tcr_compute_pa_size x0, #TCR_EL2_PS_SHIFT, x1, x2 - msr tcr_el2, x0 + ldr x1, [x0, #NVHE_INIT_TCR_EL2] + tcr_compute_pa_size x1, #TCR_EL2_PS_SHIFT, x2, x3 + msr tcr_el2, x1 isb @@ -117,10 +136,8 @@ alternative_else_nop_endif kimg_hyp_va x0, x1 msr vbar_el2, x0 - /* Hello, World! */ - mov x0, #SMCCC_RET_SUCCESS - eret -SYM_CODE_END(__kvm_hyp_init) + ret +SYM_CODE_END(___kvm_hyp_init) SYM_CODE_START(__kvm_handle_stub_hvc) cmp x0, #HVC_SOFT_RESTART -- 2.29.2.454.gaff20da3a2-goog
[PATCH v4 20/26] kvm: arm64: Add function to enter host from KVM nVHE hyp code
All nVHE hyp code is currently executed as handlers of host's HVCs. This will change as nVHE starts intercepting host's PSCI CPU_ON SMCs. The newly booted CPU will need to initialize EL2 state and then enter the host. Add __host_enter function that branches into the existing host state-restoring code after the trap handler would have returned. Signed-off-by: David Brazdil --- arch/arm64/kvm/hyp/nvhe/host.S | 9 + 1 file changed, 9 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 2b56f0bdf874..a820dfdc9c25 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -39,6 +39,7 @@ SYM_FUNC_START(__host_exit) bl handle_trap /* Restore host regs x0-x17 */ +__host_enter_restore_full: ldp x0, x1, [x29, #CPU_XREG_OFFSET(0)] ldp x2, x3, [x29, #CPU_XREG_OFFSET(2)] ldp x4, x5, [x29, #CPU_XREG_OFFSET(4)] @@ -61,6 +62,14 @@ __host_enter_without_restoring: sb SYM_FUNC_END(__host_exit) +/* + * void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); + */ +SYM_FUNC_START(__host_enter) + mov x29, x0 + b __host_enter_restore_full +SYM_FUNC_END(__host_enter) + /* * void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par); */ -- 2.29.2.454.gaff20da3a2-goog
[PATCH v4 18/26] kvm: arm64: Forward safe PSCI SMCs coming from host
Forward the following PSCI SMCs issued by host to EL3 as they do not
require the hypervisor's intervention. This assumes that EL3 correctly
implements the PSCI specification.
Only function IDs implemented in Linux are included.
Where both 32-bit and 64-bit variants exist, it is assumed that the host
will always use the 64-bit variant.
* SMCs that only return information about the system
* PSCI_VERSION- PSCI version implemented by EL3
* PSCI_FEATURES - optional features supported by EL3
* AFFINITY_INFO - power state of core/cluster
* MIGRATE_INFO_TYPE - whether Trusted OS can be migrated
* MIGRATE_INFO_UP_CPU - resident core of Trusted OS
* operations which do not affect the hypervisor
* MIGRATE - migrate Trusted OS to a different core
* SET_SUSPEND_MODE- toggle OS-initiated mode
* system shutdown/reset
* SYSTEM_OFF
* SYSTEM_RESET
* SYSTEM_RESET2
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 42 +++-
1 file changed, 41 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 70b42f433449..5ad56a875ffa 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -54,14 +54,50 @@ static bool is_psci_call(u64 func_id)
}
}
+static unsigned long psci_call(unsigned long fn, unsigned long arg0,
+ unsigned long arg1, unsigned long arg2)
+{
+ struct arm_smccc_res res;
+
+ arm_smccc_1_1_smc(fn, arg0, arg1, arg2, );
+ return res.a0;
+}
+
+static unsigned long psci_forward(struct kvm_cpu_context *host_ctxt)
+{
+ return psci_call(cpu_reg(host_ctxt, 0), cpu_reg(host_ctxt, 1),
+cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3));
+}
+
+static __noreturn unsigned long psci_forward_noreturn(struct kvm_cpu_context
*host_ctxt)
+{
+ psci_forward(host_ctxt);
+ hyp_panic(); /* unreachable */
+}
+
static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context
*host_ctxt)
{
- return PSCI_RET_NOT_SUPPORTED;
+ if ((func_id == kvm_host_psci_0_1_function_ids.cpu_off) ||
+ (func_id == kvm_host_psci_0_1_function_ids.migrate))
+ return psci_forward(host_ctxt);
+ else
+ return PSCI_RET_NOT_SUPPORTED;
}
static unsigned long psci_0_2_handler(u64 func_id, struct kvm_cpu_context
*host_ctxt)
{
switch (func_id) {
+ case PSCI_0_2_FN_PSCI_VERSION:
+ case PSCI_0_2_FN_CPU_OFF:
+ case PSCI_0_2_FN64_AFFINITY_INFO:
+ case PSCI_0_2_FN64_MIGRATE:
+ case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
+ case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU:
+ return psci_forward(host_ctxt);
+ case PSCI_0_2_FN_SYSTEM_OFF:
+ case PSCI_0_2_FN_SYSTEM_RESET:
+ psci_forward_noreturn(host_ctxt);
+ unreachable();
default:
return PSCI_RET_NOT_SUPPORTED;
}
@@ -70,6 +106,10 @@ static unsigned long psci_0_2_handler(u64 func_id, struct
kvm_cpu_context *host_
static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context
*host_ctxt)
{
switch (func_id) {
+ case PSCI_1_0_FN_PSCI_FEATURES:
+ case PSCI_1_0_FN_SET_SUSPEND_MODE:
+ case PSCI_1_1_FN64_SYSTEM_RESET2:
+ return psci_forward(host_ctxt);
default:
return psci_0_2_handler(func_id, host_ctxt);
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 17/26] kvm: arm64: Add offset for hyp VA <-> PA conversion
Add a host-initialized constant to KVM nVHE hyp code for converting
between EL2 linear map virtual addresses and physical addresses.
Also add `__hyp_pa` macro that performs the conversion.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 3 +++
arch/arm64/kvm/va_layout.c | 30 +---
2 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 61375d4571c2..70b42f433449 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -18,6 +18,9 @@
/* Config options set by the host. */
__ro_after_init u32 kvm_host_psci_version;
__ro_after_init struct psci_0_1_function_ids kvm_host_psci_0_1_function_ids;
+__ro_after_init s64 hyp_physvirt_offset;
+
+#define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset)
static u64 get_psci_func_id(struct kvm_cpu_context *host_ctxt)
{
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index 4130b72e6891..d8cc51bd60bf 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -23,6 +23,30 @@ static u8 tag_lsb;
static u64 tag_val;
static u64 va_mask;
+/*
+ * Compute HYP VA by using the same computation as kern_hyp_va().
+ */
+static u64 __early_kern_hyp_va(u64 addr)
+{
+ addr &= va_mask;
+ addr |= tag_val << tag_lsb;
+ return addr;
+}
+
+/*
+ * Store a hyp VA <-> PA offset into a hyp-owned variable.
+ */
+static void init_hyp_physvirt_offset(void)
+{
+ extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
+ u64 kern_va, hyp_va;
+
+ /* Compute the offset from the hyp VA and PA of a random symbol. */
+ kern_va = (u64)kvm_ksym_ref(__hyp_text_start);
+ hyp_va = __early_kern_hyp_va(kern_va);
+ CHOOSE_NVHE_SYM(hyp_physvirt_offset) = (s64)__pa(kern_va) - (s64)hyp_va;
+}
+
/*
* We want to generate a hyp VA with the following format (with V ==
* vabits_actual):
@@ -54,6 +78,8 @@ __init void kvm_compute_layout(void)
tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2,
tag_lsb);
}
tag_val >>= tag_lsb;
+
+ init_hyp_physvirt_offset();
}
static u32 compute_instruction(int n, u32 rd, u32 rn)
@@ -146,9 +172,7 @@ void kvm_patch_vector_branch(struct alt_instr *alt,
/*
* Compute HYP VA by using the same computation as kern_hyp_va()
*/
- addr = (uintptr_t)kvm_ksym_ref(__kvm_hyp_vector);
- addr &= va_mask;
- addr |= tag_val << tag_lsb;
+ addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
/* Use PC[10:7] to branch to the same vector in KVM */
addr |= ((u64)origptr & GENMASK_ULL(10, 7));
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 16/26] kvm: arm64: Bootstrap PSCI SMC handler in nVHE EL2
Add a handler of PSCI SMCs in nVHE hyp code. The handler is initialized
with the version used by the host's PSCI driver and the function IDs it
was configured with. If the SMC function ID matches one of the
configured PSCI calls (for v0.1) or falls into the PSCI function ID
range (for v0.2+), the SMC is handled by the PSCI handler. For now, all
SMCs return PSCI_RET_NOT_SUPPORTED.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_hyp.h | 2 +
arch/arm64/kvm/arm.c | 25 -
.../arm64/kvm/hyp/include/nvhe/trap_handler.h | 18
arch/arm64/kvm/hyp/nvhe/Makefile | 2 +-
arch/arm64/kvm/hyp/nvhe/hyp-main.c| 10 +-
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 100 ++
6 files changed, 151 insertions(+), 6 deletions(-)
create mode 100644 arch/arm64/kvm/hyp/include/nvhe/trap_handler.h
create mode 100644 arch/arm64/kvm/hyp/nvhe/psci-relay.c
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index cb25c15e3d8d..c0450828378b 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -93,6 +93,8 @@ void deactivate_traps_vhe_put(void);
u64 __guest_enter(struct kvm_vcpu *vcpu);
+bool kvm_host_psci_handler(struct kvm_cpu_context *host_ctxt);
+
void __noreturn hyp_panic(void);
#ifdef __KVM_NVHE_HYPERVISOR__
void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par);
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 09bb4098502b..fda7bfa440d5 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -19,6 +19,7 @@
#include
#include
#include
+#include
#include
#define CREATE_TRACE_POINTS
@@ -64,6 +65,8 @@ static DEFINE_PER_CPU(unsigned char,
kvm_arm_hardware_enabled);
DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
extern u64 kvm_nvhe_sym(__cpu_logical_map)[NR_CPUS];
+extern u32 kvm_nvhe_sym(kvm_host_psci_version);
+extern struct psci_0_1_function_ids
kvm_nvhe_sym(kvm_host_psci_0_1_function_ids);
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
{
@@ -1600,6 +1603,22 @@ static void init_cpu_logical_map(void)
kvm_nvhe_sym(__cpu_logical_map)[cpu] = cpu_logical_map(cpu);
}
+static bool init_psci_relay(void)
+{
+ /*
+* If PSCI has not been initialized, protected KVM cannot install
+* itself on newly booted CPUs.
+*/
+ if (!psci_ops.get_version) {
+ kvm_err("Cannot initialize protected mode without PSCI\n");
+ return false;
+ }
+
+ kvm_nvhe_sym(kvm_host_psci_version) = psci_ops.get_version();
+ kvm_nvhe_sym(kvm_host_psci_0_1_function_ids) =
get_psci_0_1_function_ids();
+ return true;
+}
+
static int init_common_resources(void)
{
return kvm_set_ipa_limit();
@@ -1772,9 +1791,13 @@ static int init_hyp_mode(void)
}
}
- if (is_protected_kvm_enabled())
+ if (is_protected_kvm_enabled()) {
init_cpu_logical_map();
+ if (!init_psci_relay())
+ goto out_err;
+ }
+
return 0;
out_err:
diff --git a/arch/arm64/kvm/hyp/include/nvhe/trap_handler.h
b/arch/arm64/kvm/hyp/include/nvhe/trap_handler.h
new file mode 100644
index ..1e6d995968a1
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/nvhe/trap_handler.h
@@ -0,0 +1,18 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Trap handler helpers.
+ *
+ * Copyright (C) 2020 - Google LLC
+ * Author: Marc Zyngier
+ */
+
+#ifndef __ARM64_KVM_NVHE_TRAP_HANDLER_H__
+#define __ARM64_KVM_NVHE_TRAP_HANDLER_H__
+
+#include
+
+#define cpu_reg(ctxt, r) (ctxt)->regs.regs[r]
+#define DECLARE_REG(type, name, ctxt, reg) \
+ type name = (type)cpu_reg(ctxt, (reg))
+
+#endif /* __ARM64_KVM_NVHE_TRAP_HANDLER_H__ */
diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index cf11f8182756..1f1e351c5fe2 100644
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -7,7 +7,7 @@ asflags-y := -D__KVM_NVHE_HYPERVISOR__
ccflags-y := -D__KVM_NVHE_HYPERVISOR__
obj-y := timer-sr.o sysreg-sr.o debug-sr.o switch.o tlb.o hyp-init.o host.o \
-hyp-main.o hyp-smp.o
+hyp-main.o hyp-smp.o psci-relay.o
obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
../fpsimd.o ../hyp-entry.o ../exception.o
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index f25680ede080..bde658d51404 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -12,9 +12,7 @@
#include
#include
-#define cpu_reg(ctxt, r) (ctxt)->regs.regs[r]
-#define DECLARE_REG(type, name, ctxt, reg) \
- type name = (type)cpu_reg(ctxt, (reg))
+#include
DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
@@ -166,7 +164,1
[PATCH v4 09/26] kvm: arm64: Remove vector_ptr param of hyp-init
KVM precomputes the hyp VA of __kvm_hyp_host_vector, essentially a
constant (minus ASLR), before passing it to __kvm_hyp_init.
Now that we have alternatives for converting kimg VA to hyp VA, replace
this with computing the constant inside __kvm_hyp_init, thus removing
the need for an argument.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_asm.h | 2 --
arch/arm64/include/asm/kvm_mmu.h | 24
arch/arm64/kvm/arm.c | 4 +---
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 9 ++---
4 files changed, 31 insertions(+), 8 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 4a6a77d8d13e..531f9d04eefd 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -163,10 +163,8 @@ struct kvm_vcpu;
struct kvm_s2_mmu;
DECLARE_KVM_NVHE_SYM(__kvm_hyp_init);
-DECLARE_KVM_NVHE_SYM(__kvm_hyp_host_vector);
DECLARE_KVM_HYP_SYM(__kvm_hyp_vector);
#define __kvm_hyp_init CHOOSE_NVHE_SYM(__kvm_hyp_init)
-#define __kvm_hyp_host_vector CHOOSE_NVHE_SYM(__kvm_hyp_host_vector)
#define __kvm_hyp_vector CHOOSE_HYP_SYM(__kvm_hyp_vector)
extern unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h
index e298191a854d..e52d82aeadca 100644
--- a/arch/arm64/include/asm/kvm_mmu.h
+++ b/arch/arm64/include/asm/kvm_mmu.h
@@ -94,6 +94,30 @@ alternative_cb_end
sub \reg, \reg, \tmp
.endm
+/*
+ * Convert a kernel image address to a hyp VA
+ * reg: kernel address to be converted in place
+ * tmp: temporary register
+ *
+ * The actual code generation takes place in kvm_get_kimage_voffset, and
+ * the instructions below are only there to reserve the space and
+ * perform the register allocation (kvm_update_kimg_phys_offset uses the
+ * specific registers encoded in the instructions).
+ */
+.macro kimg_hyp_va reg, tmp
+alternative_cb kvm_update_kimg_phys_offset
+ movz\tmp, #0
+ movk\tmp, #0, lsl #16
+ movk\tmp, #0, lsl #32
+ movk\tmp, #0, lsl #48
+alternative_cb_end
+
+ sub \reg, \reg, \tmp
+ mov_q \tmp, PAGE_OFFSET
+ orr \reg, \reg, \tmp
+ kern_hyp_va \reg
+.endm
+
#else
#include
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index d9e8d7dc7e11..9e5d0ea84822 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1386,7 +1386,6 @@ static void cpu_init_hyp_mode(void)
{
phys_addr_t pgd_ptr;
unsigned long hyp_stack_ptr;
- unsigned long vector_ptr;
unsigned long tpidr_el2;
struct arm_smccc_res res;
@@ -1404,7 +1403,6 @@ static void cpu_init_hyp_mode(void)
pgd_ptr = kvm_mmu_get_httbr();
hyp_stack_ptr = __this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE;
hyp_stack_ptr = kern_hyp_va(hyp_stack_ptr);
- vector_ptr = (unsigned
long)kern_hyp_va(kvm_ksym_ref(__kvm_hyp_host_vector));
/*
* Call initialization code, and switch to the full blown HYP code.
@@ -1414,7 +1412,7 @@ static void cpu_init_hyp_mode(void)
*/
BUG_ON(!system_capabilities_finalized());
arm_smccc_1_1_hvc(KVM_HOST_SMCCC_FUNC(__kvm_hyp_init),
- pgd_ptr, tpidr_el2, hyp_stack_ptr, vector_ptr, );
+ pgd_ptr, tpidr_el2, hyp_stack_ptr, );
WARN_ON(res.a0 != SMCCC_RET_SUCCESS);
/*
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index b11a9d7db677..931a8c38f085 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -50,7 +50,6 @@ __invalid:
* x1: HYP pgd
* x2: per-CPU offset
* x3: HYP stack
-* x4: HYP vectors
*/
__do_hyp_init:
/* Check for a stub HVC call */
@@ -134,9 +133,13 @@ alternative_else_nop_endif
msr sctlr_el2, x0
isb
- /* Set the stack and new vectors */
+ /* Set the stack */
mov sp, x3
- msr vbar_el2, x4
+
+ /* Set the host vector */
+ ldr x0, =__kvm_hyp_host_vector
+ kimg_hyp_va x0, x1
+ msr vbar_el2, x0
/* Hello, World! */
mov x0, #SMCCC_RET_SUCCESS
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 10/26] kvm: arm64: Move hyp-init params to a per-CPU struct
Once we start initializing KVM on newly booted cores before the rest of
the kernel, parameters to __do_hyp_init will need to be provided by EL2
rather than EL1. At that point it will not be possible to pass its three
arguments directly because PSCI_CPU_ON only supports one context
argument.
Refactor __do_hyp_init to accept its parameters in a struct. This
prepares the code for KVM booting cores as well as removes any limits on
the number of __do_hyp_init arguments.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/kvm_asm.h | 6 ++
arch/arm64/include/asm/kvm_hyp.h | 2 +-
arch/arm64/kernel/asm-offsets.c| 3 +++
arch/arm64/kvm/arm.c | 23 +--
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 16 +++-
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 2 ++
6 files changed, 32 insertions(+), 20 deletions(-)
diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 531f9d04eefd..1a7b91534a16 100644
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -148,6 +148,12 @@ extern void *__vhe_undefined_symbol;
#endif
+struct kvm_nvhe_init_params {
+ unsigned long tpidr_el2;
+ unsigned long stack_hyp_va;
+ phys_addr_t pgd_pa;
+};
+
/* Translate a kernel address @ptr into its equivalent linear mapping */
#define kvm_ksym_ref(ptr) \
({ \
diff --git a/arch/arm64/include/asm/kvm_hyp.h b/arch/arm64/include/asm/kvm_hyp.h
index 6b664de5ec1f..cb25c15e3d8d 100644
--- a/arch/arm64/include/asm/kvm_hyp.h
+++ b/arch/arm64/include/asm/kvm_hyp.h
@@ -14,6 +14,7 @@
DECLARE_PER_CPU(struct kvm_cpu_context, kvm_hyp_ctxt);
DECLARE_PER_CPU(unsigned long, kvm_hyp_vector);
+DECLARE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
#define read_sysreg_elx(r,nvh,vh) \
({ \
@@ -98,4 +99,3 @@ void __noreturn __hyp_do_panic(bool restore_host, u64 spsr,
u64 elr, u64 par);
#endif
#endif /* __ARM64_KVM_HYP_H__ */
-
diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 7d32fc959b1a..8d6272a01a00 100644
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -110,6 +110,9 @@ int main(void)
DEFINE(CPU_APGAKEYLO_EL1,offsetof(struct kvm_cpu_context,
sys_regs[APGAKEYLO_EL1]));
DEFINE(HOST_CONTEXT_VCPU,offsetof(struct kvm_cpu_context,
__hyp_running_vcpu));
DEFINE(HOST_DATA_CONTEXT,offsetof(struct kvm_host_data, host_ctxt));
+ DEFINE(NVHE_INIT_TPIDR_EL2, offsetof(struct kvm_nvhe_init_params,
tpidr_el2));
+ DEFINE(NVHE_INIT_STACK_HYP_VA, offsetof(struct kvm_nvhe_init_params,
stack_hyp_va));
+ DEFINE(NVHE_INIT_PGD_PA, offsetof(struct kvm_nvhe_init_params, pgd_pa));
#endif
#ifdef CONFIG_CPU_PM
DEFINE(CPU_CTX_SP, offsetof(struct cpu_suspend_ctx, sp));
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 9e5d0ea84822..93a408c00249 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -51,6 +51,7 @@ DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
unsigned long kvm_arm_hyp_percpu_base[NR_CPUS];
+DECLARE_KVM_NVHE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
/* The VMID used in the VTTBR */
static atomic64_t kvm_vmid_gen = ATOMIC64_INIT(1);
@@ -1384,9 +1385,7 @@ static int kvm_init_vector_slots(void)
static void cpu_init_hyp_mode(void)
{
- phys_addr_t pgd_ptr;
- unsigned long hyp_stack_ptr;
- unsigned long tpidr_el2;
+ struct kvm_nvhe_init_params *params =
this_cpu_ptr_nvhe_sym(kvm_init_params);
struct arm_smccc_res res;
/* Switch from the HYP stub to our own HYP init vector */
@@ -1397,12 +1396,17 @@ static void cpu_init_hyp_mode(void)
* kernel's mapping to the linear mapping, and store it in tpidr_el2
* so that we can use adr_l to access per-cpu variables in EL2.
*/
- tpidr_el2 = (unsigned long)this_cpu_ptr_nvhe_sym(__per_cpu_start) -
- (unsigned
long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
+ params->tpidr_el2 = (unsigned
long)this_cpu_ptr_nvhe_sym(__per_cpu_start) -
+ (unsigned
long)kvm_ksym_ref(CHOOSE_NVHE_SYM(__per_cpu_start));
- pgd_ptr = kvm_mmu_get_httbr();
- hyp_stack_ptr = __this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE;
- hyp_stack_ptr = kern_hyp_va(hyp_stack_ptr);
+ params->stack_hyp_va =
kern_hyp_va(__this_cpu_read(kvm_arm_hyp_stack_page) + PAGE_SIZE);
+ params->pgd_pa = kvm_mmu_get_httbr();
+
+ /*
+* Flush the init params from the data cache because the struct will
+* be read while the MMU is off.
+*/
+ kvm_flush_dcache_to_poc(params, sizeo
[PATCH v4 12/26] kvm: arm64: Add .hyp.data..ro_after_init ELF section
Add rules for renaming the .data..ro_after_init ELF section in KVM nVHE
object files to .hyp.data..ro_after_init, linking it into the kernel
and mapping it in hyp at runtime.
The section is RW to the host, then mapped RO in hyp. The expectation is
that the host populates the variables in the section and they are never
changed by hyp afterwards.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/sections.h | 1 +
arch/arm64/kernel/vmlinux.lds.S | 10 ++
arch/arm64/kvm/arm.c | 8
arch/arm64/kvm/hyp/nvhe/hyp.lds.S | 1 +
4 files changed, 20 insertions(+)
diff --git a/arch/arm64/include/asm/sections.h
b/arch/arm64/include/asm/sections.h
index 3994169985ef..8ff579361731 100644
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -11,6 +11,7 @@ extern char __alt_instructions[], __alt_instructions_end[];
extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[];
extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
extern char __hyp_text_start[], __hyp_text_end[];
+extern char __hyp_data_ro_after_init_start[], __hyp_data_ro_after_init_end[];
extern char __idmap_text_start[], __idmap_text_end[];
extern char __initdata_begin[], __initdata_end[];
extern char __inittext_begin[], __inittext_end[];
diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 1bda604f4c70..4382b5d0645d 100644
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -30,6 +30,13 @@ jiffies = jiffies_64;
*(__kvm_ex_table) \
__stop___kvm_ex_table = .;
+#define HYPERVISOR_DATA_SECTIONS \
+ HYP_SECTION_NAME(.data..ro_after_init) : { \
+ __hyp_data_ro_after_init_start = .; \
+ *(HYP_SECTION_NAME(.data..ro_after_init)) \
+ __hyp_data_ro_after_init_end = .; \
+ }
+
#define HYPERVISOR_PERCPU_SECTION \
. = ALIGN(PAGE_SIZE); \
HYP_SECTION_NAME(.data..percpu) : { \
@@ -37,6 +44,7 @@ jiffies = jiffies_64;
}
#else /* CONFIG_KVM */
#define HYPERVISOR_EXTABLE
+#define HYPERVISOR_DATA_SECTIONS
#define HYPERVISOR_PERCPU_SECTION
#endif
@@ -234,6 +242,8 @@ SECTIONS
_sdata = .;
RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
+ HYPERVISOR_DATA_SECTIONS
+
/*
* Data written with the MMU off but read with the MMU on requires
* cache lines to be invalidated, discarding up to a Cache Writeback
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 6c8594378865..40857cbed3d1 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1705,6 +1705,14 @@ static int init_hyp_mode(void)
goto out_err;
}
+ err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_ro_after_init_start),
+ kvm_ksym_ref(__hyp_data_ro_after_init_end),
+ PAGE_HYP_RO);
+ if (err) {
+ kvm_err("Cannot map .hyp.data..ro_after_init section\n");
+ goto out_err;
+ }
+
err = create_hyp_mappings(kvm_ksym_ref(__start_rodata),
kvm_ksym_ref(__end_rodata), PAGE_HYP_RO);
if (err) {
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index bb2d986ff696..5d76ff2ba63e 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -16,4 +16,5 @@ SECTIONS {
HYP_SECTION_NAME(.data..percpu) : {
PERCPU_INPUT(L1_CACHE_BYTES)
}
+ HYP_SECTION(.data..ro_after_init)
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 14/26] kvm: arm64: Create nVHE copy of cpu_logical_map
When KVM starts validating host's PSCI requests, it will need to map
MPIDR back to the CPU ID. To this end, copy cpu_logical_map into nVHE
hyp memory when KVM is initialized.
Only copy the information for CPUs that are online at the point of KVM
initialization so that KVM rejects CPUs whose features were not checked
against the finalized capabilities.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/arm.c | 19 +++
arch/arm64/kvm/hyp/nvhe/hyp-smp.c | 16
2 files changed, 35 insertions(+)
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 40857cbed3d1..09bb4098502b 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -63,6 +63,8 @@ static bool vgic_present;
static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
+extern u64 kvm_nvhe_sym(__cpu_logical_map)[NR_CPUS];
+
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
{
return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
@@ -1584,6 +1586,20 @@ static inline void hyp_cpu_pm_exit(void)
}
#endif
+static void init_cpu_logical_map(void)
+{
+ unsigned int cpu;
+
+ /*
+* Copy the MPIDR <-> logical CPU ID mapping to hyp.
+* Only copy the set of online CPUs whose features have been chacked
+* against the finalized system capabilities. The hypervisor will not
+* allow any other CPUs from the `possible` set to boot.
+*/
+ for_each_online_cpu(cpu)
+ kvm_nvhe_sym(__cpu_logical_map)[cpu] = cpu_logical_map(cpu);
+}
+
static int init_common_resources(void)
{
return kvm_set_ipa_limit();
@@ -1756,6 +1772,9 @@ static int init_hyp_mode(void)
}
}
+ if (is_protected_kvm_enabled())
+ init_cpu_logical_map();
+
return 0;
out_err:
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
index 7b0363b4857f..cbab0c6246e2 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
@@ -8,6 +8,22 @@
#include
#include
+/*
+ * nVHE copy of data structures tracking available CPU cores.
+ * Only entries for CPUs that were online at KVM init are populated.
+ * Other CPUs should not be allowed to boot because their features were
+ * not checked against the finalized system capabilities.
+ */
+u64 __ro_after_init __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] =
INVALID_HWID };
+
+u64 cpu_logical_map(unsigned int cpu)
+{
+ if (cpu >= ARRAY_SIZE(__cpu_logical_map))
+ hyp_panic();
+
+ return __cpu_logical_map[cpu];
+}
+
unsigned long __hyp_per_cpu_offset(unsigned int cpu)
{
unsigned long *cpu_base_array;
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 06/26] psci: Add accessor for psci_0_1_function_ids
Make it possible to retrieve a copy of the psci_0_1_function_ids struct.
This is useful for KVM if it is configured to intercept host's PSCI SMCs.
Signed-off-by: David Brazdil
---
drivers/firmware/psci/psci.c | 12 +---
include/linux/psci.h | 9 +
2 files changed, 14 insertions(+), 7 deletions(-)
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index 593fdd0e09a2..f5fc429cae3f 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -58,15 +58,13 @@ typedef unsigned long (psci_fn)(unsigned long, unsigned
long,
unsigned long, unsigned long);
static psci_fn *invoke_psci_fn;
-struct psci_0_1_function_ids {
- u32 cpu_suspend;
- u32 cpu_on;
- u32 cpu_off;
- u32 migrate;
-};
-
static struct psci_0_1_function_ids psci_0_1_function_ids;
+struct psci_0_1_function_ids get_psci_0_1_function_ids(void)
+{
+ return psci_0_1_function_ids;
+}
+
#define PSCI_0_2_POWER_STATE_MASK \
(PSCI_0_2_POWER_STATE_ID_MASK | \
PSCI_0_2_POWER_STATE_TYPE_MASK | \
diff --git a/include/linux/psci.h b/include/linux/psci.h
index 2a1bfb890e58..4ca0060a3fc4 100644
--- a/include/linux/psci.h
+++ b/include/linux/psci.h
@@ -34,6 +34,15 @@ struct psci_operations {
extern struct psci_operations psci_ops;
+struct psci_0_1_function_ids {
+ u32 cpu_suspend;
+ u32 cpu_on;
+ u32 cpu_off;
+ u32 migrate;
+};
+
+struct psci_0_1_function_ids get_psci_0_1_function_ids(void);
+
#if defined(CONFIG_ARM_PSCI_FW)
int __init psci_dt_init(void);
#else
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 05/26] psci: Replace psci_function_id array with a struct
Small refactor that replaces array of v0.1 function IDs indexed by an
enum of function-name constants with a struct of function IDs "indexed"
by field names. This is done in preparation for exposing the IDs to
other parts of the kernel. Exposing a struct avoids the need for
bounds checking.
Signed-off-by: David Brazdil
---
drivers/firmware/psci/psci.c | 29 ++---
1 file changed, 14 insertions(+), 15 deletions(-)
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index 13b9ed71b446..593fdd0e09a2 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -58,15 +58,14 @@ typedef unsigned long (psci_fn)(unsigned long, unsigned
long,
unsigned long, unsigned long);
static psci_fn *invoke_psci_fn;
-enum psci_function {
- PSCI_FN_CPU_SUSPEND,
- PSCI_FN_CPU_ON,
- PSCI_FN_CPU_OFF,
- PSCI_FN_MIGRATE,
- PSCI_FN_MAX,
+struct psci_0_1_function_ids {
+ u32 cpu_suspend;
+ u32 cpu_on;
+ u32 cpu_off;
+ u32 migrate;
};
-static u32 psci_function_id[PSCI_FN_MAX];
+static struct psci_0_1_function_ids psci_0_1_function_ids;
#define PSCI_0_2_POWER_STATE_MASK \
(PSCI_0_2_POWER_STATE_ID_MASK | \
@@ -178,7 +177,7 @@ static int __psci_cpu_suspend(u32 fn, u32 state, unsigned
long entry_point)
static int psci_0_1_cpu_suspend(u32 state, unsigned long entry_point)
{
- return __psci_cpu_suspend(psci_function_id[PSCI_FN_CPU_SUSPEND],
+ return __psci_cpu_suspend(psci_0_1_function_ids.cpu_suspend,
state, entry_point);
}
@@ -198,7 +197,7 @@ static int __psci_cpu_off(u32 fn, u32 state)
static int psci_0_1_cpu_off(u32 state)
{
- return __psci_cpu_off(psci_function_id[PSCI_FN_CPU_OFF], state);
+ return __psci_cpu_off(psci_0_1_function_ids.cpu_off, state);
}
static int psci_0_2_cpu_off(u32 state)
@@ -216,7 +215,7 @@ static int __psci_cpu_on(u32 fn, unsigned long cpuid,
unsigned long entry_point)
static int psci_0_1_cpu_on(unsigned long cpuid, unsigned long entry_point)
{
- return __psci_cpu_on(psci_function_id[PSCI_FN_CPU_ON], cpuid,
entry_point);
+ return __psci_cpu_on(psci_0_1_function_ids.cpu_on, cpuid, entry_point);
}
static int psci_0_2_cpu_on(unsigned long cpuid, unsigned long entry_point)
@@ -234,7 +233,7 @@ static int __psci_migrate(u32 fn, unsigned long cpuid)
static int psci_0_1_migrate(unsigned long cpuid)
{
- return __psci_migrate(psci_function_id[PSCI_FN_MIGRATE], cpuid);
+ return __psci_migrate(psci_0_1_function_ids.migrate, cpuid);
}
static int psci_0_2_migrate(unsigned long cpuid)
@@ -548,22 +547,22 @@ static int __init psci_0_1_init(struct device_node *np)
psci_ops.get_version = psci_0_1_get_version;
if (!of_property_read_u32(np, "cpu_suspend", )) {
- psci_function_id[PSCI_FN_CPU_SUSPEND] = id;
+ psci_0_1_function_ids.cpu_suspend = id;
psci_ops.cpu_suspend = psci_0_1_cpu_suspend;
}
if (!of_property_read_u32(np, "cpu_off", )) {
- psci_function_id[PSCI_FN_CPU_OFF] = id;
+ psci_0_1_function_ids.cpu_off = id;
psci_ops.cpu_off = psci_0_1_cpu_off;
}
if (!of_property_read_u32(np, "cpu_on", )) {
- psci_function_id[PSCI_FN_CPU_ON] = id;
+ psci_0_1_function_ids.cpu_on = id;
psci_ops.cpu_on = psci_0_1_cpu_on;
}
if (!of_property_read_u32(np, "migrate", )) {
- psci_function_id[PSCI_FN_MIGRATE] = id;
+ psci_0_1_function_ids.migrate = id;
psci_ops.migrate = psci_0_1_migrate;
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 04/26] psci: Split functions to v0.1 and v0.2+ variants
Refactor implementation of v0.1+ functions (CPU_SUSPEND, CPU_OFF,
CPU_ON, MIGRATE) to have two functions psci_0_1_foo / psci_0_2_foo that
select the function ID and call a common helper __psci_foo.
This is a small cleanup so that the function ID array is only used for
v0.1 configurations.
Signed-off-by: David Brazdil
---
drivers/firmware/psci/psci.c | 94 +++-
1 file changed, 60 insertions(+), 34 deletions(-)
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index ace5b9ac676c..13b9ed71b446 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -168,46 +168,80 @@ int psci_set_osi_mode(bool enable)
return psci_to_linux_errno(err);
}
-static int psci_cpu_suspend(u32 state, unsigned long entry_point)
+static int __psci_cpu_suspend(u32 fn, u32 state, unsigned long entry_point)
{
int err;
- u32 fn;
- fn = psci_function_id[PSCI_FN_CPU_SUSPEND];
err = invoke_psci_fn(fn, state, entry_point, 0);
return psci_to_linux_errno(err);
}
-static int psci_cpu_off(u32 state)
+static int psci_0_1_cpu_suspend(u32 state, unsigned long entry_point)
+{
+ return __psci_cpu_suspend(psci_function_id[PSCI_FN_CPU_SUSPEND],
+ state, entry_point);
+}
+
+static int psci_0_2_cpu_suspend(u32 state, unsigned long entry_point)
+{
+ return __psci_cpu_suspend(PSCI_FN_NATIVE(0_2, CPU_SUSPEND),
+ state, entry_point);
+}
+
+static int __psci_cpu_off(u32 fn, u32 state)
{
int err;
- u32 fn;
- fn = psci_function_id[PSCI_FN_CPU_OFF];
err = invoke_psci_fn(fn, state, 0, 0);
return psci_to_linux_errno(err);
}
-static int psci_cpu_on(unsigned long cpuid, unsigned long entry_point)
+static int psci_0_1_cpu_off(u32 state)
+{
+ return __psci_cpu_off(psci_function_id[PSCI_FN_CPU_OFF], state);
+}
+
+static int psci_0_2_cpu_off(u32 state)
+{
+ return __psci_cpu_off(PSCI_0_2_FN_CPU_OFF, state);
+}
+
+static int __psci_cpu_on(u32 fn, unsigned long cpuid, unsigned long
entry_point)
{
int err;
- u32 fn;
- fn = psci_function_id[PSCI_FN_CPU_ON];
err = invoke_psci_fn(fn, cpuid, entry_point, 0);
return psci_to_linux_errno(err);
}
-static int psci_migrate(unsigned long cpuid)
+static int psci_0_1_cpu_on(unsigned long cpuid, unsigned long entry_point)
+{
+ return __psci_cpu_on(psci_function_id[PSCI_FN_CPU_ON], cpuid,
entry_point);
+}
+
+static int psci_0_2_cpu_on(unsigned long cpuid, unsigned long entry_point)
+{
+ return __psci_cpu_on(PSCI_FN_NATIVE(0_2, CPU_ON), cpuid, entry_point);
+}
+
+static int __psci_migrate(u32 fn, unsigned long cpuid)
{
int err;
- u32 fn;
- fn = psci_function_id[PSCI_FN_MIGRATE];
err = invoke_psci_fn(fn, cpuid, 0, 0);
return psci_to_linux_errno(err);
}
+static int psci_0_1_migrate(unsigned long cpuid)
+{
+ return __psci_migrate(psci_function_id[PSCI_FN_MIGRATE], cpuid);
+}
+
+static int psci_0_2_migrate(unsigned long cpuid)
+{
+ return __psci_migrate(PSCI_FN_NATIVE(0_2, MIGRATE), cpuid);
+}
+
static int psci_affinity_info(unsigned long target_affinity,
unsigned long lowest_affinity_level)
{
@@ -352,7 +386,7 @@ static void __init psci_init_system_suspend(void)
static void __init psci_init_cpu_suspend(void)
{
- int feature = psci_features(psci_function_id[PSCI_FN_CPU_SUSPEND]);
+ int feature = psci_features(PSCI_FN_NATIVE(0_2, CPU_SUSPEND));
if (feature != PSCI_RET_NOT_SUPPORTED)
psci_cpu_suspend_feature = feature;
@@ -426,24 +460,16 @@ static void __init psci_init_smccc(void)
static void __init psci_0_2_set_functions(void)
{
pr_info("Using standard PSCI v0.2 function IDs\n");
- psci_ops.get_version = psci_0_2_get_version;
-
- psci_function_id[PSCI_FN_CPU_SUSPEND] =
- PSCI_FN_NATIVE(0_2, CPU_SUSPEND);
- psci_ops.cpu_suspend = psci_cpu_suspend;
-
- psci_function_id[PSCI_FN_CPU_OFF] = PSCI_0_2_FN_CPU_OFF;
- psci_ops.cpu_off = psci_cpu_off;
-
- psci_function_id[PSCI_FN_CPU_ON] = PSCI_FN_NATIVE(0_2, CPU_ON);
- psci_ops.cpu_on = psci_cpu_on;
- psci_function_id[PSCI_FN_MIGRATE] = PSCI_FN_NATIVE(0_2, MIGRATE);
- psci_ops.migrate = psci_migrate;
-
- psci_ops.affinity_info = psci_affinity_info;
-
- psci_ops.migrate_info_type = psci_migrate_info_type;
+ psci_ops = (struct psci_operations){
+ .get_version = psci_0_2_get_version,
+ .cpu_suspend = psci_0_2_cpu_suspend,
+ .cpu_off = psci_0_2_cpu_off,
+ .cpu_on = psci_0_2_cpu_on,
+ .migrate = psci_0_2_migrate,
+ .affinity_info = psci_affinity_info,
+ .migrate_info_type = psci_migrate_info_type,
+ };
arm_pm_restart = psci_sys_reset;
[PATCH v4 03/26] psci: Support psci_ops.get_version for v0.1
KVM's host PSCI SMC filter needs to be aware of the PSCI version of the
system but currently it is impossible to distinguish between v0.1 and
PSCI disabled because both have get_version == NULL.
Populate get_version for v0.1 with a function that returns a constant.
psci_opt.get_version is currently unused so this has no effect on
existing functionality.
Acked-by: Mark Rutland
Signed-off-by: David Brazdil
---
drivers/firmware/psci/psci.c | 13 ++---
1 file changed, 10 insertions(+), 3 deletions(-)
diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
index 00af99b6f97c..ace5b9ac676c 100644
--- a/drivers/firmware/psci/psci.c
+++ b/drivers/firmware/psci/psci.c
@@ -146,7 +146,12 @@ static int psci_to_linux_errno(int errno)
return -EINVAL;
}
-static u32 psci_get_version(void)
+static u32 psci_0_1_get_version(void)
+{
+ return PSCI_VERSION(0, 1);
+}
+
+static u32 psci_0_2_get_version(void)
{
return invoke_psci_fn(PSCI_0_2_FN_PSCI_VERSION, 0, 0, 0);
}
@@ -421,7 +426,7 @@ static void __init psci_init_smccc(void)
static void __init psci_0_2_set_functions(void)
{
pr_info("Using standard PSCI v0.2 function IDs\n");
- psci_ops.get_version = psci_get_version;
+ psci_ops.get_version = psci_0_2_get_version;
psci_function_id[PSCI_FN_CPU_SUSPEND] =
PSCI_FN_NATIVE(0_2, CPU_SUSPEND);
@@ -450,7 +455,7 @@ static void __init psci_0_2_set_functions(void)
*/
static int __init psci_probe(void)
{
- u32 ver = psci_get_version();
+ u32 ver = psci_0_2_get_version();
pr_info("PSCIv%d.%d detected in firmware.\n",
PSCI_VERSION_MAJOR(ver),
@@ -514,6 +519,8 @@ static int __init psci_0_1_init(struct device_node *np)
pr_info("Using PSCI v0.1 Function IDs from DT\n");
+ psci_ops.get_version = psci_0_1_get_version;
+
if (!of_property_read_u32(np, "cpu_suspend", )) {
psci_function_id[PSCI_FN_CPU_SUSPEND] = id;
psci_ops.cpu_suspend = psci_cpu_suspend;
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 02/26] kvm: arm64: Add ARM64_KVM_PROTECTED_MODE CPU capability
Expose the boolean value whether the system is running with KVM in
protected mode (nVHE + kernel param). CPU capability was selected over
a global variable to allow use in alternatives.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/cpucaps.h | 3 ++-
arch/arm64/include/asm/kvm_host.h | 1 +
arch/arm64/include/asm/virt.h | 8
arch/arm64/kernel/cpufeature.c| 22 ++
arch/arm64/kvm/arm.c | 9 -
5 files changed, 41 insertions(+), 2 deletions(-)
diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 162539d4c8cd..42f850718d4b 100644
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -66,7 +66,8 @@
#define ARM64_HAS_TLB_RANGE56
#define ARM64_MTE 57
#define ARM64_WORKAROUND_1508412 58
+#define ARM64_KVM_PROTECTED_MODE 59
-#define ARM64_NCAPS59
+#define ARM64_NCAPS60
#endif /* __ASM_CPUCAPS_H */
diff --git a/arch/arm64/include/asm/kvm_host.h
b/arch/arm64/include/asm/kvm_host.h
index bca38ccade58..11beda85ee7e 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -58,6 +58,7 @@ enum kvm_mode {
KVM_MODE_DEFAULT,
KVM_MODE_PROTECTED,
};
+enum kvm_mode kvm_get_mode(void);
DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h
index 6069be50baf9..eb81dcc220b6 100644
--- a/arch/arm64/include/asm/virt.h
+++ b/arch/arm64/include/asm/virt.h
@@ -97,6 +97,14 @@ static __always_inline bool has_vhe(void)
return cpus_have_final_cap(ARM64_HAS_VIRT_HOST_EXTN);
}
+static __always_inline bool is_protected_kvm_enabled(void)
+{
+ if (is_vhe_hyp_code())
+ return false;
+ else
+ return cpus_have_final_cap(ARM64_KVM_PROTECTED_MODE);
+}
+
#endif /* __ASSEMBLY__ */
#endif /* ! __ASM__VIRT_H */
diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 280b10762f6b..5f9e557fa1f7 100644
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -74,6 +74,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -1709,6 +1710,21 @@ static void cpu_enable_mte(struct arm64_cpu_capabilities
const *cap)
}
#endif /* CONFIG_ARM64_MTE */
+#ifdef CONFIG_KVM
+static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry,
int __unused)
+{
+ if (kvm_get_mode() != KVM_MODE_PROTECTED)
+ return false;
+
+ if (is_kernel_in_hyp_mode()) {
+ pr_warn("Protected KVM not available with VHE\n");
+ return false;
+ }
+
+ return true;
+}
+#endif /* CONFIG_KVM */
+
/* Internal helper functions to match cpu capability type */
static bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
@@ -1822,6 +1838,12 @@ static const struct arm64_cpu_capabilities
arm64_features[] = {
.field_pos = ID_AA64PFR0_EL1_SHIFT,
.min_field_value = ID_AA64PFR0_EL1_32BIT_64BIT,
},
+ {
+ .desc = "Protected KVM",
+ .capability = ARM64_KVM_PROTECTED_MODE,
+ .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+ .matches = is_kvm_protected_mode,
+ },
#endif
{
.desc = "Kernel page table isolation (KPTI)",
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 01cf6ac5b2cb..d9e8d7dc7e11 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1849,7 +1849,9 @@ int kvm_arch_init(void *opaque)
if (err)
goto out_hyp;
- if (in_hyp_mode)
+ if (is_protected_kvm_enabled())
+ kvm_info("Protected nVHE mode initialized successfully\n");
+ else if (in_hyp_mode)
kvm_info("VHE mode initialized successfully\n");
else
kvm_info("Hyp mode initialized successfully\n");
@@ -1884,6 +1886,11 @@ static int __init early_kvm_mode_cfg(char *arg)
}
early_param("kvm-arm.mode", early_kvm_mode_cfg);
+enum kvm_mode kvm_get_mode(void)
+{
+ return kvm_mode;
+}
+
static int arm_init(void)
{
int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 01/26] kvm: arm64: Add kvm-arm.mode early kernel parameter
Add an early parameter that allows users to select the mode of operation
for KVM/arm64.
For now, the only supported value is "protected". By passing this flag
users opt into the hypervisor placing additional restrictions on the
host kernel. These allow the hypervisor to spawn guests whose state is
kept private from the host. Restrictions will include stage-2 address
translation to prevent host from accessing guest memory, filtering its
SMC calls, etc.
Without this parameter, the default behaviour remains selecting VHE/nVHE
based on hardware support and CONFIG_ARM64_VHE.
Signed-off-by: David Brazdil
---
Documentation/admin-guide/kernel-parameters.txt | 10 ++
arch/arm64/include/asm/kvm_host.h | 9 +
arch/arm64/kvm/arm.c| 16
3 files changed, 35 insertions(+)
diff --git a/Documentation/admin-guide/kernel-parameters.txt
b/Documentation/admin-guide/kernel-parameters.txt
index 526d65d8573a..ee9f13776388 100644
--- a/Documentation/admin-guide/kernel-parameters.txt
+++ b/Documentation/admin-guide/kernel-parameters.txt
@@ -2259,6 +2259,16 @@
for all guests.
Default is 1 (enabled) if in 64-bit or 32-bit PAE mode.
+ kvm-arm.mode=
+ [KVM,ARM] Select one of KVM/arm64's modes of operation.
+
+ protected: nVHE-based mode with support for guests whose
+ state is kept private from the host.
+ Not valid if the kernel is running in EL2.
+
+ Defaults to VHE/nVHE based on hardware support and
+ the value of CONFIG_ARM64_VHE.
+
kvm-arm.vgic_v3_group0_trap=
[KVM,ARM] Trap guest accesses to GICv3 group-0
system registers
diff --git a/arch/arm64/include/asm/kvm_host.h
b/arch/arm64/include/asm/kvm_host.h
index 21ce5c420247..bca38ccade58 100644
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -50,6 +50,15 @@
#define KVM_DIRTY_LOG_MANUAL_CAPS (KVM_DIRTY_LOG_MANUAL_PROTECT_ENABLE | \
KVM_DIRTY_LOG_INITIALLY_SET)
+/*
+ * Mode of operation configurable with kvm-arm.mode early param.
+ * See Documentation/admin-guide/kernel-parameters.txt for more information.
+ */
+enum kvm_mode {
+ KVM_MODE_DEFAULT,
+ KVM_MODE_PROTECTED,
+};
+
DECLARE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
extern unsigned int kvm_sve_max_vl;
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 7e86207fa2fc..01cf6ac5b2cb 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -45,6 +45,8 @@
__asm__(".arch_extension virt");
#endif
+static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT;
+
DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
@@ -1868,6 +1870,20 @@ void kvm_arch_exit(void)
kvm_perf_teardown();
}
+static int __init early_kvm_mode_cfg(char *arg)
+{
+ if (!arg)
+ return -EINVAL;
+
+ if (strcmp(arg, "protected") == 0) {
+ kvm_mode = KVM_MODE_PROTECTED;
+ return 0;
+ }
+
+ return -EINVAL;
+}
+early_param("kvm-arm.mode", early_kvm_mode_cfg);
+
static int arm_init(void)
{
int rc = kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE);
--
2.29.2.454.gaff20da3a2-goog
[PATCH v4 00/26] Opt-in always-on nVHE hypervisor
As we progress towards being able to keep guest state private to the host running nVHE hypervisor, this series allows the hypervisor to install itself on newly booted CPUs before the host is allowed to run on them. All functionality described below is opt-in, guarded by an early param 'kvm-arm.mode=protected'. Future patches specific to the new protected mode should be hidden behind the same param. The hypervisor starts trapping host SMCs and intercepting host's PSCI calls which boot CPUs. It replaces the host's entry point with its own, initializes the EL2 state of the new CPU and installs the nVHE hyp vector before ERETing to the host's entry point. The kernel checks new cores' features against the finalized system capabilities. To avoid the need to move this code/data to EL2, the implementation only allows to boot cores that were online at the time of KVM initialization and therefore had been checked already. Other PSCI SMCs are forwarded to EL3, though only the known set of SMCs implemented in the kernel is allowed. Non-PSCI SMCs are also forwarded to EL3. Future changes will need to ensure the safety of all SMCs wrt. protected guests. The host is still allowed to reset EL2 back to the stub vector, eg. for hibernation or kexec, but will not disable nVHE when there are no VMs. Tested on Rock Pi 4B, based on kvmarm/queue, itself on top of 5.10-rc4. Patches also available at: https://android-kvm.googlesource.com/linux topic/psci-on-master_v4 changes since v3: * generic 'kvm-arm.mode' kernel param instead of 'kvm-arm.protected' * implement SYSTEM_SUSPEND * refactor PSCI driver to expose fn IDs more cleanly * init MAIR_EL2, TCR_EL2 from nVHE params struct changes since v2: * avoid non-spec error in CPU_SUSPEND * refuse to init without PSCI * compute hyp VA args of hyp-init in hyp instead of using params struct * use hyp_symbol_addr in per-cpu calls * simplify memory.h/sysreg.h includes * rebase on kvmarm/queue, use trap handler args macros changes since v1: * early param sets a capability instead of a static key * assume SMCCC v1.2 for host SMC forwarding * fix reserved SMC ID range for PSCI * split init_el2_state into smaller macros, move to el2_setup.h * many small cleanups changes since RFC: * add early param to make features opt-in * simplify CPU_ON/SUSPEND implementation * replace spinlocks with CAS atomic * make cpu_logical_map ro_after_init David Brazdil (26): kvm: arm64: Add kvm-arm.mode early kernel parameter kvm: arm64: Add ARM64_KVM_PROTECTED_MODE CPU capability psci: Support psci_ops.get_version for v0.1 psci: Split functions to v0.1 and v0.2+ variants psci: Replace psci_function_id array with a struct psci: Add accessor for psci_0_1_function_ids arm64: Make cpu_logical_map() take unsigned int arm64: Extract parts of el2_setup into a macro kvm: arm64: Remove vector_ptr param of hyp-init kvm: arm64: Move hyp-init params to a per-CPU struct kvm: arm64: Init MAIR/TCR_EL2 from params struct kvm: arm64: Add .hyp.data..ro_after_init ELF section kvm: arm64: Support per_cpu_ptr in nVHE hyp code kvm: arm64: Create nVHE copy of cpu_logical_map kvm: arm64: Add SMC handler in nVHE EL2 kvm: arm64: Bootstrap PSCI SMC handler in nVHE EL2 kvm: arm64: Add offset for hyp VA <-> PA conversion kvm: arm64: Forward safe PSCI SMCs coming from host kvm: arm64: Extract __do_hyp_init into a helper function kvm: arm64: Add function to enter host from KVM nVHE hyp code kvm: arm64: Intercept host's CPU_ON SMCs kvm: arm64: Intercept host's CPU_SUSPEND PSCI SMCs kvm: arm64: Intercept host's SYSTEM_SUSPEND PSCI SMCs kvm: arm64: Keep nVHE EL2 vector installed kvm: arm64: Trap host SMCs in protected mode kvm: arm64: Fix EL2 mode availability checks .../admin-guide/kernel-parameters.txt | 10 + arch/arm64/include/asm/cpucaps.h | 3 +- arch/arm64/include/asm/el2_setup.h| 182 ++ arch/arm64/include/asm/kvm_arm.h | 1 + arch/arm64/include/asm/kvm_asm.h | 10 +- arch/arm64/include/asm/kvm_host.h | 10 + arch/arm64/include/asm/kvm_hyp.h | 4 +- arch/arm64/include/asm/kvm_mmu.h | 24 ++ arch/arm64/include/asm/percpu.h | 6 + arch/arm64/include/asm/sections.h | 1 + arch/arm64/include/asm/smp.h | 4 +- arch/arm64/include/asm/virt.h | 26 ++ arch/arm64/kernel/asm-offsets.c | 5 + arch/arm64/kernel/cpufeature.c| 22 ++ arch/arm64/kernel/head.S | 144 +--- arch/arm64/kernel/image-vars.h| 6 +- arch/arm64/kernel/setup.c | 2 +- arch/arm64/kernel/vmlinux.lds.S | 10 + arch/arm64/kvm/arm.c | 139 +++- .../arm64/kvm/hyp/include/nvhe/trap_handler.h | 18 + arch/arm64/kvm/hyp/nvhe/Makefile | 3 +-
Re: [PATCH v3 16/23] kvm: arm64: Forward safe PSCI SMCs coming from host
On Fri, Nov 27, 2020 at 10:14:33AM +, Lorenzo Pieralisi wrote: > On Thu, Nov 26, 2020 at 03:54:14PM +0000, David Brazdil wrote: > > Forward the following PSCI SMCs issued by host to EL3 as they do not > > require the hypervisor's intervention. This assumes that EL3 correctly > > implements the PSCI specification. > > > > Only function IDs implemented in Linux are included. > > > > Where both 32-bit and 64-bit variants exist, it is assumed that the host > > will always use the 64-bit variant. > > > > * SMCs that only return information about the system > >* PSCI_VERSION- PSCI version implemented by EL3 > >* PSCI_FEATURES - optional features supported by EL3 > >* AFFINITY_INFO - power state of core/cluster > >* MIGRATE_INFO_TYPE - whether Trusted OS can be migrated > >* MIGRATE_INFO_UP_CPU - resident core of Trusted OS > > * operations which do not affect the hypervisor > >* MIGRATE - migrate Trusted OS to a different core > >* SET_SUSPEND_MODE- toggle OS-initiated mode > > * system shutdown/reset > >* SYSTEM_OFF > >* SYSTEM_RESET > >* SYSTEM_RESET2 > > What about SYSTEM_SUSPEND ? Oops, forgot that one. Will add a handler similar to CPU_SUSPEND.
Re: [PATCH v3 03/23] arm64: Make cpu_logical_map() take unsigned int
On Thu, Nov 26, 2020 at 05:28:38PM +, Mark Rutland wrote: > On Thu, Nov 26, 2020 at 03:54:01PM +0000, David Brazdil wrote: > > CPU index should never be negative. Change the signature of > > (set_)cpu_logical_map to take an unsigned int. > > > > Signed-off-by: David Brazdil > > Is there a function problem here, or is this just cleanup from > inspection? > > Core code including the cpuhp_*() callbacks uses an int, so if there's a > strong justification to change this, it suggests there's some treewide > cleanup that should be done. > > I don't have strong feelings on the matter, but I'd like to understand > the rationale. Yeah, it's a mess. Marc and I felt that using a uint was less error-prone wrt bounds checks. If this gets an int, it still works and only checking the upper bound is required. Does that make sense? David
Re: [PATCH v3 06/23] kvm: arm64: Add kvm-arm.protected early kernel parameter
> > > be just me, but if you agree please update so that it doesn't give remote > > > idea that it is not valid on VHE enabled hardware. > > > > > > I was trying to run this on the hardware and was trying to understand the > > > details on how to do that. > > > > I see what you're saying, but !CONFIG_ARM64_VHE isn't accurate either. The > > option makes sense if: > > 1) all cores booted in EL2 > > == is_hyp_mode_available() > > 2) ID_AA64MMFR1_EL1.VH=0 or !CONFIG_ARM64_VHE > > == !is_kernel_in_hyp_mode() > > > > The former feels implied for KVM, the latter could be 'Valid if the kernel > > is running in EL1'? WDYT? > > I reckon we can avoid the restriction if we instead add an early stub > like with have for KASLR. That way we could parse the command line > early, and if necessary re-initialize EL2 and drop to EL1 before the > main kernel has to make any decisions about how to initialize things. > That would allow us to have a more general kvm-arm.mode option where a > single kernel Image could support: > > * "protected" mode on nVHE or VHE HW > * "nvhe" mode on nVHE or VHE HW > * "vhe" mode on VHE HW > > ... defaulting to VHE/nVHE modes depending on HW support. > > That would also be somewhat future-proof if we have to add other > variants of protected mode in future, as we could extend the mode option > with parameters for each mode. Agreed that 'mode' is a more future-proof flag and I would very much love to have an option to force nVHE on VHE HW. I however expect that the early stub would not be a trivial addition and would not want to get into that in this series. Could we agree on 'protected' as the only supported value for the time being? David
Re: [PATCH v3 04/23] arm64: Move MAIR_EL1_SET to asm/memory.h
Hey Mark, > > diff --git a/arch/arm64/include/asm/memory.h > > b/arch/arm64/include/asm/memory.h > > index cd61239bae8c..54a22cb5b17b 100644 > > --- a/arch/arm64/include/asm/memory.h > > +++ b/arch/arm64/include/asm/memory.h > > @@ -152,6 +152,19 @@ > > #define MT_S2_FWB_NORMAL 6 > > #define MT_S2_FWB_DEVICE_nGnRE 1 > > > > +/* > > + * Default MAIR_ELx. MT_NORMAL_TAGGED is initially mapped as Normal memory > > and > > + * changed during __cpu_setup to Normal Tagged if the system supports MTE. > > + */ > > +#define MAIR_ELx_SET > > \ > > + (MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRnE, MT_DEVICE_nGnRnE) | \ > > +MAIR_ATTRIDX(MAIR_ATTR_DEVICE_nGnRE, MT_DEVICE_nGnRE) |\ > > +MAIR_ATTRIDX(MAIR_ATTR_DEVICE_GRE, MT_DEVICE_GRE) |\ > > +MAIR_ATTRIDX(MAIR_ATTR_NORMAL_NC, MT_NORMAL_NC) | \ > > +MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL) |\ > > +MAIR_ATTRIDX(MAIR_ATTR_NORMAL_WT, MT_NORMAL_WT) | \ > > +MAIR_ATTRIDX(MAIR_ATTR_NORMAL, MT_NORMAL_TAGGED)) > > Patch 7 initializes MAIR_EL2 with this directly rather than copying it > from MAIR_EL1, which means that MT_NORMAL_TAGGED will never be tagged > within the nVHE hyp code. > > Is that expected? I suspect it's worth a comment here (introduced in > patch 7), just to make that clear. Ouch, that didn't use to be there. In that case let's not build more implicit assumptions into the code. I'll pass MAIR_EL1 in kvm_nvhe_init_params. David
Re: [PATCH v3 05/23] arm64: Extract parts of el2_setup into a macro
Hey Mark, > In the head.S code, this was under an ifdef CONFIG_ARM_GIC_V3, but that > ifdef wasn't carried into the macro here, or into its use below. I'm not > sure of the impact, but that does seem to be a functional change. > > > + > > +.macro __init_el2_hstr > > + msr hstr_el2, xzr // Disable CP15 traps to EL2 > > +.endm > > Likewise, this used to be be guarded by CONFIG_COMPAT, but that's not > carried into the macro or its use. > > If the intent was to remove the conditionality, then that should be > mentioned in the commit message, since it is a potential functional > change. Apologies, and well spotted. Marc suggested removing the ifdefs as redundant during his review of v2. I'll update the commit message to reflect that. David
Re: [PATCH v3 19/23] kvm: arm64: Intercept host's CPU_ON SMCs
Hey Sudeep,
> > +static unsigned int find_cpu_id(u64 mpidr)
> > +{
> > + unsigned int i;
> > +
> > + /* Reject invalid MPIDRs */
> > + if (mpidr & ~MPIDR_HWID_BITMASK)
> > + return INVALID_CPU_ID;
> > +
> > + for (i = 0; i < NR_CPUS; i++) {
>
> I may not have understood the flow correctly, so just asking:
> This is just called for secondaries on boot right ?
No, secondaries are booted before KVM is initialized. kvm_arch_init() installs
the hypervisor on each core that is online at that point. That flow does not
touch this code.
But the kernel can later power down some of those cares and then this handler
is called if it tries to power them on again. You can exercise this with:
# echo 0 > /sys/devices/system/cpu/cpu5/online
# echo 1 > /sys/devices/system/cpu/cpu5/online
> And the cpumasks are setup by then ?
Cpumasks are initialized before KVM init, so yes, we could copy that
information up to EL2 and use it here. That comes down to copying `nr_cpu_ids`
because the possible set is logical IDs 0..nr_cpu_ids-1 (see smp_init_cpus()).
> Just trying to see if we can use cpu_possible_mask instead of running through
> all 256/1k/4k cpus(ofcourse based on NR_CPUS config)
I decided to keep things simple because a valid MPIDR should not need to
scan the entire array, at most the first `nr_cpu_ids` entries. An invalid MPIDR
will scan all NR_CPUS entries but that does not seem worth optimizing for.
David
Re: [PATCH v3 16/23] kvm: arm64: Forward safe PSCI SMCs coming from host
On Fri, Nov 27, 2020 at 04:51:59PM +, Sudeep Holla wrote:
> On Thu, Nov 26, 2020 at 03:54:14PM +0000, David Brazdil wrote:
> > Forward the following PSCI SMCs issued by host to EL3 as they do not
> > require the hypervisor's intervention. This assumes that EL3 correctly
> > implements the PSCI specification.
> >
> > Only function IDs implemented in Linux are included.
> >
> > Where both 32-bit and 64-bit variants exist, it is assumed that the host
> > will always use the 64-bit variant.
> >
> > * SMCs that only return information about the system
> >* PSCI_VERSION- PSCI version implemented by EL3
> >* PSCI_FEATURES - optional features supported by EL3
> >* AFFINITY_INFO - power state of core/cluster
> >* MIGRATE_INFO_TYPE - whether Trusted OS can be migrated
> >* MIGRATE_INFO_UP_CPU - resident core of Trusted OS
> > * operations which do not affect the hypervisor
> >* MIGRATE - migrate Trusted OS to a different core
> >* SET_SUSPEND_MODE- toggle OS-initiated mode
> > * system shutdown/reset
> >* SYSTEM_OFF
> >* SYSTEM_RESET
> >* SYSTEM_RESET2
> >
> > Signed-off-by: David Brazdil
> > ---
> > arch/arm64/kvm/hyp/nvhe/psci-relay.c | 43 +++-
> > 1 file changed, 42 insertions(+), 1 deletion(-)
> >
> > diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
> > b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
> > index e7091d89f0fc..7aa87ab7f5ce 100644
> > --- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
> > +++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
> > @@ -57,14 +57,51 @@ static bool is_psci_call(u64 func_id)
> > }
> > }
> >
> > +static unsigned long psci_call(unsigned long fn, unsigned long arg0,
> > + unsigned long arg1, unsigned long arg2)
> > +{
> > + struct arm_smccc_res res;
> > +
> > + arm_smccc_1_1_smc(fn, arg0, arg1, arg2, );
> > + return res.a0;
> > +}
> > +
> > +static unsigned long psci_forward(struct kvm_cpu_context *host_ctxt)
> > +{
> > + return psci_call(cpu_reg(host_ctxt, 0), cpu_reg(host_ctxt, 1),
> > +cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3));
> > +}
> > +
> > +static __noreturn unsigned long psci_forward_noreturn(struct
> > kvm_cpu_context *host_ctxt)
> > +{
> > + psci_forward(host_ctxt);
> > + hyp_panic(); /* unreachable */
> > +}
> > +
> > static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context
> > *host_ctxt)
> > {
> > - return PSCI_RET_NOT_SUPPORTED;
> > + if (func_id == kvm_host_psci_function_id[PSCI_FN_CPU_OFF])
> > + return psci_forward(host_ctxt);
> > + else if (func_id == kvm_host_psci_function_id[PSCI_FN_MIGRATE])
> > + return psci_forward(host_ctxt);
>
> Looks weird or I am not seeing something right ? Same action for both
> right ? Can't they be combined ?
Sure, happy to combine them. I thought visually it made sense to have one
action per ID.
Re: [PATCH v3 06/23] kvm: arm64: Add kvm-arm.protected early kernel parameter
Hey Sudeep, > > diff --git a/Documentation/admin-guide/kernel-parameters.txt > > b/Documentation/admin-guide/kernel-parameters.txt > > index 526d65d8573a..06c89975c29c 100644 > > --- a/Documentation/admin-guide/kernel-parameters.txt > > +++ b/Documentation/admin-guide/kernel-parameters.txt > > @@ -2259,6 +2259,11 @@ > > for all guests. > > Default is 1 (enabled) if in 64-bit or 32-bit PAE mode. > > > > + kvm-arm.protected= > > + [KVM,ARM] Allow spawning protected guests whose state > > + is kept private from the host. Only valid for non-VHE. > > + Default is 0 (disabled). > > + > > Sorry for being pedantic. Can we reword this to say valid for > !CONFIG_ARM64_VHE ? I read this as valid only for non-VHE hardware, it may > be just me, but if you agree please update so that it doesn't give remote > idea that it is not valid on VHE enabled hardware. > > I was trying to run this on the hardware and was trying to understand the > details on how to do that. I see what you're saying, but !CONFIG_ARM64_VHE isn't accurate either. The option makes sense if: 1) all cores booted in EL2 == is_hyp_mode_available() 2) ID_AA64MMFR1_EL1.VH=0 or !CONFIG_ARM64_VHE == !is_kernel_in_hyp_mode() The former feels implied for KVM, the latter could be 'Valid if the kernel is running in EL1'? WDYT? -David
Re: [PATCH v3 02/23] psci: Accessor for configured PSCI function IDs
Hey Mark,
On Thu, Nov 26, 2020 at 05:24:50PM +, Mark Rutland wrote:
> On Thu, Nov 26, 2020 at 03:54:00PM +0000, David Brazdil wrote:
> > Function IDs used by PSCI are configurable for v0.1 via DT/APCI. If the
> > host is using PSCI v0.1, KVM's host PSCI proxy needs to use the same IDs.
> > Expose the array holding the information with a read-only accessor.
> >
> > Signed-off-by: David Brazdil
> > ---
> > drivers/firmware/psci/psci.c | 16
> > include/linux/psci.h | 10 ++
> > 2 files changed, 18 insertions(+), 8 deletions(-)
> >
> > diff --git a/drivers/firmware/psci/psci.c b/drivers/firmware/psci/psci.c
> > index 213c68418a65..40609564595e 100644
> > --- a/drivers/firmware/psci/psci.c
> > +++ b/drivers/firmware/psci/psci.c
> > @@ -58,16 +58,16 @@ typedef unsigned long (psci_fn)(unsigned long, unsigned
> > long,
> > unsigned long, unsigned long);
> > static psci_fn *invoke_psci_fn;
> >
> > -enum psci_function {
> > - PSCI_FN_CPU_SUSPEND,
> > - PSCI_FN_CPU_ON,
> > - PSCI_FN_CPU_OFF,
> > - PSCI_FN_MIGRATE,
> > - PSCI_FN_MAX,
> > -};
> > -
> > static u32 psci_function_id[PSCI_FN_MAX];
> >
> > +u32 psci_get_function_id(enum psci_function fn)
> > +{
> > + if (WARN_ON_ONCE(fn < 0 || fn >= PSCI_FN_MAX))
> > + return 0;
> > +
> > + return psci_function_id[fn];
> > +}
>
> I'd really like if we could namespace this with a psci_0_1_* prefix
> before we expose it outside of the PSCI code. I appreciate that's a
> larger change, but I reckon we only need a couple of new patches:
>
> 1) Split the ops which consume the FN ids into separate psci_0_1_*() and
>psci_0_2_*() variants, with a common __psci_*() helper that takes the
>function ID as an argument. The 0_1 variants would read the function
>ID from a variable, and the 0_2 variants would hard-code the id.
>
> 2) Replace the psci_function_id array with:
>
>struct psci_0_1_function_ids {
> u32 suspend;
> u32 cpu_on;
> u32 cpu_off;
> u32 migrate;
>};
>
>... and remove enum psci_function entirely.
>
> 3) Add a helper which returns the entire psci_0_1_function_ids struct in
>one go. No warnings necessary.
>
> Does that sound OK to you?
Sure, sounds easy enough and 2) is in line with how I structured the handlers
in KVM.
Thanks,
David
[PATCH v3 12/23] kvm: arm64: Create nVHE copy of cpu_logical_map
When KVM starts validating host's PSCI requests, it will need to map
MPIDR back to the CPU ID. To this end, copy cpu_logical_map into nVHE
hyp memory when KVM is initialized.
Only copy the information for CPUs that are online at the point of KVM
initialization so that KVM rejects CPUs whose features were not checked
against the finalized capabilities.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/arm.c | 19 +++
arch/arm64/kvm/hyp/nvhe/hyp-smp.c | 16
2 files changed, 35 insertions(+)
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index bb07f0401c68..bb3c541d3ddb 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -61,6 +61,8 @@ static bool vgic_present;
static DEFINE_PER_CPU(unsigned char, kvm_arm_hardware_enabled);
DEFINE_STATIC_KEY_FALSE(userspace_irqchip_in_use);
+extern u64 kvm_nvhe_sym(__cpu_logical_map)[NR_CPUS];
+
int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
{
return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE;
@@ -1531,6 +1533,20 @@ static inline void hyp_cpu_pm_exit(void)
}
#endif
+static void init_cpu_logical_map(void)
+{
+ unsigned int cpu;
+
+ /*
+* Copy the MPIDR <-> logical CPU ID mapping to hyp.
+* Only copy the set of online CPUs whose features have been chacked
+* against the finalized system capabilities. The hypervisor will not
+* allow any other CPUs from the `possible` set to boot.
+*/
+ for_each_online_cpu(cpu)
+ kvm_nvhe_sym(__cpu_logical_map)[cpu] = cpu_logical_map(cpu);
+}
+
static int init_common_resources(void)
{
return kvm_set_ipa_limit();
@@ -1703,6 +1719,9 @@ static int init_hyp_mode(void)
}
}
+ if (is_protected_kvm_enabled())
+ init_cpu_logical_map();
+
return 0;
out_err:
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
index c168d86f885a..ceb427aabb91 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-smp.c
@@ -8,6 +8,22 @@
#include
#include
+/*
+ * nVHE copy of data structures tracking available CPU cores.
+ * Only entries for CPUs that were online at KVM init are populated.
+ * Other CPUs should not be allowed to boot because their features were
+ * not checked against the finalized system capabilities.
+ */
+u64 __ro_after_init __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] =
INVALID_HWID };
+
+u64 cpu_logical_map(unsigned int cpu)
+{
+ if (cpu >= ARRAY_SIZE(__cpu_logical_map))
+ hyp_panic();
+
+ return __cpu_logical_map[cpu];
+}
+
unsigned long __hyp_per_cpu_offset(unsigned int cpu)
{
unsigned long *cpu_base_array;
--
2.29.2.454.gaff20da3a2-goog
[PATCH v3 23/23] kvm: arm64: Fix EL2 mode availability checks
With protected nVHE hyp code interception host's PSCI CPU_ON/SUSPEND
SMCs, the host starts seeing new CPUs boot in EL1 instead of EL2. The
kernel logic that keeps track of the boot mode needs to be adjusted.
Add a static key enabled if KVM protected nVHE initialization is
successful.
When the key is enabled, is_hyp_mode_available continues to report
`true` because its users either treat it as a check whether KVM will be
/ was initialized, or whether stub HVCs can be made (eg. hibernate).
is_hyp_mode_mismatched is changed to report `false` when the key is
enabled. That's because all cores' modes matched at the point of KVM
init and KVM will not allow cores not present at init to boot. That
said, the function is never used after KVM is initialized.
Signed-off-by: David Brazdil
---
arch/arm64/include/asm/virt.h | 18 ++
arch/arm64/kvm/arm.c | 10 +++---
2 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h
index 2fde1186b962..f7cf3f0e5297 100644
--- a/arch/arm64/include/asm/virt.h
+++ b/arch/arm64/include/asm/virt.h
@@ -65,9 +65,19 @@ extern u32 __boot_cpu_mode[2];
void __hyp_set_vectors(phys_addr_t phys_vector_base);
void __hyp_reset_vectors(void);
+DECLARE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
+
/* Reports the availability of HYP mode */
static inline bool is_hyp_mode_available(void)
{
+ /*
+* If KVM protected mode is initialized, all CPUs must have been booted
+* in EL2. Avoid checking __boot_cpu_mode as CPUs now come up in EL1.
+*/
+ if (IS_ENABLED(CONFIG_KVM) &&
+ static_branch_likely(_protected_mode_initialized))
+ return true;
+
return (__boot_cpu_mode[0] == BOOT_CPU_MODE_EL2 &&
__boot_cpu_mode[1] == BOOT_CPU_MODE_EL2);
}
@@ -75,6 +85,14 @@ static inline bool is_hyp_mode_available(void)
/* Check if the bootloader has booted CPUs in different modes */
static inline bool is_hyp_mode_mismatched(void)
{
+ /*
+* If KVM protected mode is initialized, all CPUs must have been booted
+* in EL2. Avoid checking __boot_cpu_mode as CPUs now come up in EL1.
+*/
+ if (IS_ENABLED(CONFIG_KVM) &&
+ static_branch_likely(_protected_mode_initialized))
+ return false;
+
return __boot_cpu_mode[0] != __boot_cpu_mode[1];
}
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 6ec8ddf74643..b153c08e50fa 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -46,6 +46,8 @@
__asm__(".arch_extension virt");
#endif
+DEFINE_STATIC_KEY_FALSE(kvm_protected_mode_initialized);
+
DECLARE_KVM_HYP_PER_CPU(unsigned long, kvm_hyp_vector);
static DEFINE_PER_CPU(unsigned long, kvm_arm_hyp_stack_page);
@@ -1877,12 +1879,14 @@ int kvm_arch_init(void *opaque)
if (err)
goto out_hyp;
- if (is_protected_kvm_enabled())
+ if (is_protected_kvm_enabled()) {
+ static_branch_enable(_protected_mode_initialized);
kvm_info("Protected nVHE mode initialized successfully\n");
- else if (in_hyp_mode)
+ } else if (in_hyp_mode) {
kvm_info("VHE mode initialized successfully\n");
- else
+ } else {
kvm_info("Hyp mode initialized successfully\n");
+ }
return 0;
--
2.29.2.454.gaff20da3a2-goog
[PATCH v3 21/23] kvm: arm64: Keep nVHE EL2 vector installed
KVM by default keeps the stub vector installed and installs the nVHE
vector only briefly for init and later on demand. Change this policy
to install the vector at init and then never uninstall it if the kernel
was given the protected KVM command line parameter.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/arm.c | 12
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 7a17b5048454..6ec8ddf74643 100644
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -1478,7 +1478,8 @@ static void _kvm_arch_hardware_disable(void *discard)
void kvm_arch_hardware_disable(void)
{
- _kvm_arch_hardware_disable(NULL);
+ if (!is_protected_kvm_enabled())
+ _kvm_arch_hardware_disable(NULL);
}
#ifdef CONFIG_CPU_PM
@@ -1521,11 +1522,13 @@ static struct notifier_block hyp_init_cpu_pm_nb = {
static void __init hyp_cpu_pm_init(void)
{
- cpu_pm_register_notifier(_init_cpu_pm_nb);
+ if (!is_protected_kvm_enabled())
+ cpu_pm_register_notifier(_init_cpu_pm_nb);
}
static void __init hyp_cpu_pm_exit(void)
{
- cpu_pm_unregister_notifier(_init_cpu_pm_nb);
+ if (!is_protected_kvm_enabled())
+ cpu_pm_unregister_notifier(_init_cpu_pm_nb);
}
#else
static inline void hyp_cpu_pm_init(void)
@@ -1617,7 +1620,8 @@ static int init_subsystems(void)
kvm_sys_reg_table_init();
out:
- on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
+ if (err || !is_protected_kvm_enabled())
+ on_each_cpu(_kvm_arch_hardware_disable, NULL, 1);
return err;
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v3 20/23] kvm: arm64: Intercept host's CPU_SUSPEND PSCI SMCs
Add a handler of CPU_SUSPEND host PSCI SMCs. The SMC can either enter
a sleep state indistinguishable from a WFI or a deeper sleep state that
behaves like a CPU_OFF+CPU_ON except that the core is still considered
online when asleep.
The handler saves r0,pc of the host and makes the same call to EL3 with
the hyp CPU entry point. It either returns back to the handler and then
back to the host, or wakes up into the entry point and initializes EL2
state before dropping back to EL1.
A core can only suspend itself but other cores can concurrently invoke
CPU_ON with this core as target. To avoid racing them for the same
boot args struct, CPU_SUSPEND uses a different struct instance and entry
point. Each entry point selects the corresponding struct to restore host
boot args from. This avoids the need for locking in CPU_SUSPEND.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 9 +
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 50 +---
2 files changed, 54 insertions(+), 5 deletions(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index ea71f653af55..fbb195851fb9 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -162,7 +162,15 @@ alternative_else_nop_endif
ret
SYM_CODE_END(___kvm_hyp_init)
+SYM_CODE_START(__kvm_hyp_cpu_suspend_entry)
+ mov x28, #0 // is_cpu_on = false
+ b __kvm_hyp_cpu_common_entry
+SYM_CODE_END(__kvm_hyp_cpu_suspend_entry)
+
SYM_CODE_START(__kvm_hyp_cpu_on_entry)
+ mov x28, #1 // is_cpu_on = true
+
+SYM_INNER_LABEL(__kvm_hyp_cpu_common_entry, SYM_L_LOCAL)
msr SPsel, #1 // We want to use SP_EL{1,2}
/* Check that the core was booted in EL2. */
@@ -188,6 +196,7 @@ SYM_CODE_START(__kvm_hyp_cpu_on_entry)
kimg_hyp_va x1, x2
/* Leave idmap. */
+ mov x0, x28
br x1
SYM_CODE_END(__kvm_hyp_cpu_on_entry)
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 39e507672e6e..592c11e9851c 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -17,6 +17,7 @@
#include
extern char __kvm_hyp_cpu_on_entry[];
+extern char __kvm_hyp_cpu_suspend_entry[];
void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
@@ -39,6 +40,7 @@ struct cpu_boot_args {
static DEFINE_PER_CPU(atomic_t, cpu_on_lock) = ATOMIC_INIT(0);
static DEFINE_PER_CPU(struct cpu_boot_args, cpu_on_args);
+static DEFINE_PER_CPU(struct cpu_boot_args, cpu_suspend_args);
static u64 get_psci_func_id(struct kvm_cpu_context *host_ctxt)
{
@@ -112,6 +114,34 @@ static unsigned int find_cpu_id(u64 mpidr)
return INVALID_CPU_ID;
}
+static int psci_cpu_suspend(u64 func_id, struct kvm_cpu_context *host_ctxt)
+{
+ DECLARE_REG(u64, power_state, host_ctxt, 1);
+ DECLARE_REG(unsigned long, pc, host_ctxt, 2);
+ DECLARE_REG(unsigned long, r0, host_ctxt, 3);
+
+ struct cpu_boot_args *boot_args;
+ struct kvm_nvhe_init_params *init_params;
+
+ boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_suspend_args));
+ init_params = this_cpu_ptr(hyp_symbol_addr(kvm_init_params));
+
+ /*
+* No need to acquire a lock before writing to boot_args because a core
+* can only suspend itself and the racy CPU_ON uses a separate struct.
+*/
+
+ *boot_args = (struct cpu_boot_args){ .pc = pc, .r0 = r0 };
+
+ /*
+* Will either return if shallow sleep state, or wake up into the entry
+* point if it is a deep sleep state.
+*/
+ return psci_call(func_id, power_state,
+__hyp_pa(hyp_symbol_addr(__kvm_hyp_cpu_suspend_entry)),
+__hyp_pa(init_params));
+}
+
static __always_inline bool try_acquire_cpu_on_lock(atomic_t *l) {
return atomic_cmpxchg_acquire(l, CPU_UNLOCKED, CPU_LOCKED) ==
CPU_UNLOCKED;
}
@@ -165,27 +195,35 @@ static int psci_cpu_on(u64 func_id, struct
kvm_cpu_context *host_ctxt)
return ret;
}
-asmlinkage void __noreturn __kvm_hyp_psci_cpu_entry(void)
+asmlinkage void __noreturn __kvm_hyp_psci_cpu_entry(bool is_cpu_on)
{
atomic_t *lock;
struct cpu_boot_args *boot_args;
struct kvm_cpu_context *host_ctxt;
- lock = this_cpu_ptr(hyp_symbol_addr(cpu_on_lock));
- boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_on_args));
+ if (is_cpu_on)
+ boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_on_args));
+ else
+ boot_args = this_cpu_ptr(hyp_symbol_addr(cpu_suspend_args));
+
host_ctxt = _cpu_ptr(hyp_symbol_addr(kvm_host_data))->host_ctxt;
cpu_reg(host_ctxt, 0) = boot_args->r0;
write_sysreg_el2(boot_args->pc, SYS_ELR);
- release_cpu_on_lock(lock);
+ if (is_cpu_on) {
+ lock = thi
[PATCH v3 22/23] kvm: arm64: Trap host SMCs in protected mode
While protected nVHE KVM is installed, start trapping all host SMCs. By default, these are simply forwarded to EL3, but PSCI SMCs are validated first. Create new constant HCR_HOST_NVHE_PROTECTED_FLAGS with the new set of HCR flags to use while the nVHE vector is installed when the kernel was booted with the protected flag enabled. Switch back to the default HCR flags when switching back to the stub vector. Signed-off-by: David Brazdil --- arch/arm64/include/asm/kvm_arm.h | 1 + arch/arm64/kvm/hyp/nvhe/hyp-init.S | 10 ++ arch/arm64/kvm/hyp/nvhe/switch.c | 5 - 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 64ce29378467..4e90c2debf70 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -80,6 +80,7 @@ HCR_FMO | HCR_IMO | HCR_PTW ) #define HCR_VIRT_EXCP_MASK (HCR_VSE | HCR_VI | HCR_VF) #define HCR_HOST_NVHE_FLAGS (HCR_RW | HCR_API | HCR_APK | HCR_ATA) +#define HCR_HOST_NVHE_PROTECTED_FLAGS (HCR_HOST_NVHE_FLAGS | HCR_TSC) #define HCR_HOST_VHE_FLAGS (HCR_RW | HCR_TGE | HCR_E2H) /* TCR_EL2 Registers bits */ diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index fbb195851fb9..7af18fa1983d 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -88,6 +88,11 @@ SYM_CODE_END(__kvm_hyp_init) * x0: struct kvm_nvhe_init_params PA */ SYM_CODE_START(___kvm_hyp_init) +alternative_if ARM64_PROTECTED_KVM + mov_q x1, HCR_HOST_NVHE_PROTECTED_FLAGS + msr hcr_el2, x1 +alternative_else_nop_endif + ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] msr tpidr_el2, x1 @@ -233,6 +238,11 @@ reset: msr sctlr_el2, x5 isb +alternative_if ARM64_PROTECTED_KVM + mov_q x5, HCR_HOST_NVHE_FLAGS + msr hcr_el2, x5 +alternative_else_nop_endif + /* Install stub vectors */ adr_l x5, __hyp_stub_vectors msr vbar_el2, x5 diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c index 3e50ff35aa4f..f3d0e9eca56c 100644 --- a/arch/arm64/kvm/hyp/nvhe/switch.c +++ b/arch/arm64/kvm/hyp/nvhe/switch.c @@ -97,7 +97,10 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu) mdcr_el2 |= MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT; write_sysreg(mdcr_el2, mdcr_el2); - write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2); + if (is_protected_kvm_enabled()) + write_sysreg(HCR_HOST_NVHE_PROTECTED_FLAGS, hcr_el2); + else + write_sysreg(HCR_HOST_NVHE_FLAGS, hcr_el2); write_sysreg(CPTR_EL2_DEFAULT, cptr_el2); write_sysreg(__kvm_hyp_host_vector, vbar_el2); } -- 2.29.2.454.gaff20da3a2-goog
[PATCH v3 17/23] kvm: arm64: Extract __do_hyp_init into a helper function
In preparation for adding a CPU entry point in nVHE hyp code, extract most of __do_hyp_init hypervisor initialization code into a common helper function. This will be invoked by the entry point to install KVM on the newly booted CPU. Signed-off-by: David Brazdil --- arch/arm64/kvm/hyp/nvhe/hyp-init.S | 35 ++ 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S index 77c983aa90fa..98ce40e17b42 100644 --- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S +++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S @@ -68,13 +68,32 @@ __do_hyp_init: mov x0, #SMCCC_RET_NOT_SUPPORTED eret -1: ldr x0, [x1, #NVHE_INIT_TPIDR_EL2] - msr tpidr_el2, x0 +1: mov x0, x1 + mov x4, lr + bl ___kvm_hyp_init + mov lr, x4 - ldr x0, [x1, #NVHE_INIT_STACK_HYP_VA] - mov sp, x0 + /* Hello, World! */ + mov x0, #SMCCC_RET_SUCCESS + eret +SYM_CODE_END(__kvm_hyp_init) + +/* + * Initialize the hypervisor in EL2. + * + * Only uses x0..x3 so as to not clobber callee-saved SMCCC registers + * and leave x4 for the caller. + * + * x0: struct kvm_nvhe_init_params PA + */ +SYM_CODE_START(___kvm_hyp_init) + ldr x1, [x0, #NVHE_INIT_TPIDR_EL2] + msr tpidr_el2, x1 + + ldr x1, [x0, #NVHE_INIT_STACK_HYP_VA] + mov sp, x1 - ldr x1, [x1, #NVHE_INIT_PGD_PA] + ldr x1, [x0, #NVHE_INIT_PGD_PA] phys_to_ttbr x0, x1 alternative_if ARM64_HAS_CNP orr x0, x0, #TTBR_CNP_BIT @@ -139,10 +158,8 @@ alternative_else_nop_endif kimg_hyp_va x0, x1 msr vbar_el2, x0 - /* Hello, World! */ - mov x0, #SMCCC_RET_SUCCESS - eret -SYM_CODE_END(__kvm_hyp_init) + ret +SYM_CODE_END(___kvm_hyp_init) SYM_CODE_START(__kvm_handle_stub_hvc) cmp x0, #HVC_SOFT_RESTART -- 2.29.2.454.gaff20da3a2-goog
[PATCH v3 18/23] kvm: arm64: Add function to enter host from KVM nVHE hyp code
All nVHE hyp code is currently executed as handlers of host's HVCs. This will change as nVHE starts intercepting host's PSCI CPU_ON SMCs. The newly booted CPU will need to initialize EL2 state and then enter the host. Add __host_enter function that branches into the existing host state-restoring code after the trap handler would have returned. Signed-off-by: David Brazdil --- arch/arm64/kvm/hyp/nvhe/host.S | 9 + 1 file changed, 9 insertions(+) diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S index 2b56f0bdf874..a820dfdc9c25 100644 --- a/arch/arm64/kvm/hyp/nvhe/host.S +++ b/arch/arm64/kvm/hyp/nvhe/host.S @@ -39,6 +39,7 @@ SYM_FUNC_START(__host_exit) bl handle_trap /* Restore host regs x0-x17 */ +__host_enter_restore_full: ldp x0, x1, [x29, #CPU_XREG_OFFSET(0)] ldp x2, x3, [x29, #CPU_XREG_OFFSET(2)] ldp x4, x5, [x29, #CPU_XREG_OFFSET(4)] @@ -61,6 +62,14 @@ __host_enter_without_restoring: sb SYM_FUNC_END(__host_exit) +/* + * void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt); + */ +SYM_FUNC_START(__host_enter) + mov x29, x0 + b __host_enter_restore_full +SYM_FUNC_END(__host_enter) + /* * void __noreturn __hyp_do_panic(bool restore_host, u64 spsr, u64 elr, u64 par); */ -- 2.29.2.454.gaff20da3a2-goog
[PATCH v3 15/23] kvm: arm64: Add offset for hyp VA <-> PA conversion
Add a host-initialized constant to KVM nVHE hyp code for converting
between EL2 linear map virtual addresses and physical addresses.
Also add `__hyp_pa` macro that performs the conversion.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 3 +++
arch/arm64/kvm/va_layout.c | 30 +---
2 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 44a9fb462d24..e7091d89f0fc 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -18,6 +18,9 @@
/* Config options set by the host. */
u32 __ro_after_init kvm_host_psci_version;
u32 __ro_after_init kvm_host_psci_function_id[PSCI_FN_MAX];
+s64 __ro_after_init hyp_physvirt_offset;
+
+#define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset)
static u64 get_psci_func_id(struct kvm_cpu_context *host_ctxt)
{
diff --git a/arch/arm64/kvm/va_layout.c b/arch/arm64/kvm/va_layout.c
index 4130b72e6891..d8cc51bd60bf 100644
--- a/arch/arm64/kvm/va_layout.c
+++ b/arch/arm64/kvm/va_layout.c
@@ -23,6 +23,30 @@ static u8 tag_lsb;
static u64 tag_val;
static u64 va_mask;
+/*
+ * Compute HYP VA by using the same computation as kern_hyp_va().
+ */
+static u64 __early_kern_hyp_va(u64 addr)
+{
+ addr &= va_mask;
+ addr |= tag_val << tag_lsb;
+ return addr;
+}
+
+/*
+ * Store a hyp VA <-> PA offset into a hyp-owned variable.
+ */
+static void init_hyp_physvirt_offset(void)
+{
+ extern s64 kvm_nvhe_sym(hyp_physvirt_offset);
+ u64 kern_va, hyp_va;
+
+ /* Compute the offset from the hyp VA and PA of a random symbol. */
+ kern_va = (u64)kvm_ksym_ref(__hyp_text_start);
+ hyp_va = __early_kern_hyp_va(kern_va);
+ CHOOSE_NVHE_SYM(hyp_physvirt_offset) = (s64)__pa(kern_va) - (s64)hyp_va;
+}
+
/*
* We want to generate a hyp VA with the following format (with V ==
* vabits_actual):
@@ -54,6 +78,8 @@ __init void kvm_compute_layout(void)
tag_val |= get_random_long() & GENMASK_ULL(vabits_actual - 2,
tag_lsb);
}
tag_val >>= tag_lsb;
+
+ init_hyp_physvirt_offset();
}
static u32 compute_instruction(int n, u32 rd, u32 rn)
@@ -146,9 +172,7 @@ void kvm_patch_vector_branch(struct alt_instr *alt,
/*
* Compute HYP VA by using the same computation as kern_hyp_va()
*/
- addr = (uintptr_t)kvm_ksym_ref(__kvm_hyp_vector);
- addr &= va_mask;
- addr |= tag_val << tag_lsb;
+ addr = __early_kern_hyp_va((u64)kvm_ksym_ref(__kvm_hyp_vector));
/* Use PC[10:7] to branch to the same vector in KVM */
addr |= ((u64)origptr & GENMASK_ULL(10, 7));
--
2.29.2.454.gaff20da3a2-goog
[PATCH v3 19/23] kvm: arm64: Intercept host's CPU_ON SMCs
Add a handler of the CPU_ON PSCI call from host. When invoked, it looks
up the logical CPU ID corresponding to the provided MPIDR and populates
the state struct of the target CPU with the provided x0, pc. It then
calls CPU_ON itself, with an entry point in hyp that initializes EL2
state before returning ERET to the provided PC in EL1.
There is a simple atomic lock around the boot args struct. If it is
already locked, CPU_ON will return PENDING_ON error code.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/hyp-init.S | 30
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 109 +++
2 files changed, 139 insertions(+)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index 98ce40e17b42..ea71f653af55 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -9,6 +9,7 @@
#include
#include
+#include
#include
#include
#include
@@ -161,6 +162,35 @@ alternative_else_nop_endif
ret
SYM_CODE_END(___kvm_hyp_init)
+SYM_CODE_START(__kvm_hyp_cpu_on_entry)
+ msr SPsel, #1 // We want to use SP_EL{1,2}
+
+ /* Check that the core was booted in EL2. */
+ mrs x1, CurrentEL
+ cmp x1, #CurrentEL_EL2
+ b.eq2f
+
+ /* The core booted in EL1. KVM cannot be initialized on it. */
+1: wfe
+ wfi
+ b 1b
+
+ /* Initialize EL2 CPU state to sane values. */
+2: mov x29, x0
+ init_el2_state nvhe
+ mov x0, x29
+
+ /* Enable MMU, set vectors and stack. */
+ bl ___kvm_hyp_init
+
+ /* Load address of the C handler. */
+ ldr x1, =__kvm_hyp_psci_cpu_entry
+ kimg_hyp_va x1, x2
+
+ /* Leave idmap. */
+ br x1
+SYM_CODE_END(__kvm_hyp_cpu_on_entry)
+
SYM_CODE_START(__kvm_handle_stub_hvc)
cmp x0, #HVC_SOFT_RESTART
b.ne1f
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index 7aa87ab7f5ce..39e507672e6e 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -9,12 +9,17 @@
#include
#include
#include
+#include
#include
#include
#include
#include
+extern char __kvm_hyp_cpu_on_entry[];
+
+void __noreturn __host_enter(struct kvm_cpu_context *host_ctxt);
+
/* Config options set by the host. */
u32 __ro_after_init kvm_host_psci_version;
u32 __ro_after_init kvm_host_psci_function_id[PSCI_FN_MAX];
@@ -22,6 +27,19 @@ s64 __ro_after_init hyp_physvirt_offset;
#define __hyp_pa(x) ((phys_addr_t)((x)) + hyp_physvirt_offset)
+#define INVALID_CPU_ID UINT_MAX
+
+#define CPU_UNLOCKED 0
+#define CPU_LOCKED 1
+
+struct cpu_boot_args {
+ unsigned long pc;
+ unsigned long r0;
+};
+
+static DEFINE_PER_CPU(atomic_t, cpu_on_lock) = ATOMIC_INIT(0);
+static DEFINE_PER_CPU(struct cpu_boot_args, cpu_on_args);
+
static u64 get_psci_func_id(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(u64, func_id, host_ctxt, 0);
@@ -78,10 +96,99 @@ static __noreturn unsigned long
psci_forward_noreturn(struct kvm_cpu_context *ho
hyp_panic(); /* unreachable */
}
+static unsigned int find_cpu_id(u64 mpidr)
+{
+ unsigned int i;
+
+ /* Reject invalid MPIDRs */
+ if (mpidr & ~MPIDR_HWID_BITMASK)
+ return INVALID_CPU_ID;
+
+ for (i = 0; i < NR_CPUS; i++) {
+ if (cpu_logical_map(i) == mpidr)
+ return i;
+ }
+
+ return INVALID_CPU_ID;
+}
+
+static __always_inline bool try_acquire_cpu_on_lock(atomic_t *l) {
+ return atomic_cmpxchg_acquire(l, CPU_UNLOCKED, CPU_LOCKED) ==
CPU_UNLOCKED;
+}
+
+static __always_inline void release_cpu_on_lock(atomic_t *l) {
+ atomic_set_release(l, CPU_UNLOCKED);
+}
+
+static int psci_cpu_on(u64 func_id, struct kvm_cpu_context *host_ctxt)
+{
+ DECLARE_REG(u64, mpidr, host_ctxt, 1);
+ DECLARE_REG(unsigned long, pc, host_ctxt, 2);
+ DECLARE_REG(unsigned long, r0, host_ctxt, 3);
+
+ unsigned int cpu_id;
+ atomic_t *lock;
+ struct cpu_boot_args *boot_args;
+ struct kvm_nvhe_init_params *init_params;
+ int ret;
+
+ /*
+* Find the logical CPU ID for the given MPIDR. The search set is
+* the set of CPUs that were online at the point of KVM initialization.
+* Booting other CPUs is rejected because their cpufeatures were not
+* checked against the finalized capabilities. This could be relaxed
+* by doing the feature checks in hyp.
+*/
+ cpu_id = find_cpu_id(mpidr);
+ if (cpu_id == INVALID_CPU_ID)
+ return PSCI_RET_INVALID_PARAMS;
+
+ lock = per_cpu_ptr(hyp_symbol_addr(cpu_on_lock), cpu_id);
+ boot_args = per_cpu_ptr(hyp_symbol_addr(cpu_on_args), cpu_id);
+ init_params = per_cpu_ptr(hyp_symbol_addr(kvm_init_params), cpu_id);
+
+ /* Check if the target CPU is
[PATCH v3 16/23] kvm: arm64: Forward safe PSCI SMCs coming from host
Forward the following PSCI SMCs issued by host to EL3 as they do not
require the hypervisor's intervention. This assumes that EL3 correctly
implements the PSCI specification.
Only function IDs implemented in Linux are included.
Where both 32-bit and 64-bit variants exist, it is assumed that the host
will always use the 64-bit variant.
* SMCs that only return information about the system
* PSCI_VERSION- PSCI version implemented by EL3
* PSCI_FEATURES - optional features supported by EL3
* AFFINITY_INFO - power state of core/cluster
* MIGRATE_INFO_TYPE - whether Trusted OS can be migrated
* MIGRATE_INFO_UP_CPU - resident core of Trusted OS
* operations which do not affect the hypervisor
* MIGRATE - migrate Trusted OS to a different core
* SET_SUSPEND_MODE- toggle OS-initiated mode
* system shutdown/reset
* SYSTEM_OFF
* SYSTEM_RESET
* SYSTEM_RESET2
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/psci-relay.c | 43 +++-
1 file changed, 42 insertions(+), 1 deletion(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
index e7091d89f0fc..7aa87ab7f5ce 100644
--- a/arch/arm64/kvm/hyp/nvhe/psci-relay.c
+++ b/arch/arm64/kvm/hyp/nvhe/psci-relay.c
@@ -57,14 +57,51 @@ static bool is_psci_call(u64 func_id)
}
}
+static unsigned long psci_call(unsigned long fn, unsigned long arg0,
+ unsigned long arg1, unsigned long arg2)
+{
+ struct arm_smccc_res res;
+
+ arm_smccc_1_1_smc(fn, arg0, arg1, arg2, );
+ return res.a0;
+}
+
+static unsigned long psci_forward(struct kvm_cpu_context *host_ctxt)
+{
+ return psci_call(cpu_reg(host_ctxt, 0), cpu_reg(host_ctxt, 1),
+cpu_reg(host_ctxt, 2), cpu_reg(host_ctxt, 3));
+}
+
+static __noreturn unsigned long psci_forward_noreturn(struct kvm_cpu_context
*host_ctxt)
+{
+ psci_forward(host_ctxt);
+ hyp_panic(); /* unreachable */
+}
+
static unsigned long psci_0_1_handler(u64 func_id, struct kvm_cpu_context
*host_ctxt)
{
- return PSCI_RET_NOT_SUPPORTED;
+ if (func_id == kvm_host_psci_function_id[PSCI_FN_CPU_OFF])
+ return psci_forward(host_ctxt);
+ else if (func_id == kvm_host_psci_function_id[PSCI_FN_MIGRATE])
+ return psci_forward(host_ctxt);
+ else
+ return PSCI_RET_NOT_SUPPORTED;
}
static unsigned long psci_0_2_handler(u64 func_id, struct kvm_cpu_context
*host_ctxt)
{
switch (func_id) {
+ case PSCI_0_2_FN_PSCI_VERSION:
+ case PSCI_0_2_FN_CPU_OFF:
+ case PSCI_0_2_FN64_AFFINITY_INFO:
+ case PSCI_0_2_FN64_MIGRATE:
+ case PSCI_0_2_FN_MIGRATE_INFO_TYPE:
+ case PSCI_0_2_FN64_MIGRATE_INFO_UP_CPU:
+ return psci_forward(host_ctxt);
+ case PSCI_0_2_FN_SYSTEM_OFF:
+ case PSCI_0_2_FN_SYSTEM_RESET:
+ psci_forward_noreturn(host_ctxt);
+ unreachable();
default:
return PSCI_RET_NOT_SUPPORTED;
}
@@ -73,6 +110,10 @@ static unsigned long psci_0_2_handler(u64 func_id, struct
kvm_cpu_context *host_
static unsigned long psci_1_0_handler(u64 func_id, struct kvm_cpu_context
*host_ctxt)
{
switch (func_id) {
+ case PSCI_1_0_FN_PSCI_FEATURES:
+ case PSCI_1_0_FN_SET_SUSPEND_MODE:
+ case PSCI_1_1_FN64_SYSTEM_RESET2:
+ return psci_forward(host_ctxt);
default:
return psci_0_2_handler(func_id, host_ctxt);
}
--
2.29.2.454.gaff20da3a2-goog
[PATCH v3 13/23] kvm: arm64: Add SMC handler in nVHE EL2
Add handler of host SMCs in KVM nVHE trap handler. Forward all SMCs to
EL3 and propagate the result back to EL1. This is done in preparation
for validating host SMCs in KVM nVHE protected mode.
The implementation assumes that firmware uses SMCCC v1.2 or older. That
means x0-x17 can be used both for arguments and results, other GPRs are
preserved.
Signed-off-by: David Brazdil
---
arch/arm64/kvm/hyp/nvhe/host.S | 38 ++
arch/arm64/kvm/hyp/nvhe/hyp-main.c | 35 ---
2 files changed, 70 insertions(+), 3 deletions(-)
diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index fe2740b224cf..2b56f0bdf874 100644
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -180,3 +180,41 @@ SYM_CODE_START(__kvm_hyp_host_vector)
invalid_host_el1_vect // FIQ 32-bit EL1
invalid_host_el1_vect // Error 32-bit EL1
SYM_CODE_END(__kvm_hyp_host_vector)
+
+/*
+ * Forward SMC with arguments in struct kvm_cpu_context, and
+ * store the result into the same struct. Assumes SMCCC 1.2 or older.
+ *
+ * x0: struct kvm_cpu_context*
+ */
+SYM_CODE_START(__kvm_hyp_host_forward_smc)
+ /*
+* Use x18 to keep the pointer to the host context because
+* x18 is callee-saved in SMCCC but not in AAPCS64.
+*/
+ mov x18, x0
+
+ ldp x0, x1, [x18, #CPU_XREG_OFFSET(0)]
+ ldp x2, x3, [x18, #CPU_XREG_OFFSET(2)]
+ ldp x4, x5, [x18, #CPU_XREG_OFFSET(4)]
+ ldp x6, x7, [x18, #CPU_XREG_OFFSET(6)]
+ ldp x8, x9, [x18, #CPU_XREG_OFFSET(8)]
+ ldp x10, x11, [x18, #CPU_XREG_OFFSET(10)]
+ ldp x12, x13, [x18, #CPU_XREG_OFFSET(12)]
+ ldp x14, x15, [x18, #CPU_XREG_OFFSET(14)]
+ ldp x16, x17, [x18, #CPU_XREG_OFFSET(16)]
+
+ smc #0
+
+ stp x0, x1, [x18, #CPU_XREG_OFFSET(0)]
+ stp x2, x3, [x18, #CPU_XREG_OFFSET(2)]
+ stp x4, x5, [x18, #CPU_XREG_OFFSET(4)]
+ stp x6, x7, [x18, #CPU_XREG_OFFSET(6)]
+ stp x8, x9, [x18, #CPU_XREG_OFFSET(8)]
+ stp x10, x11, [x18, #CPU_XREG_OFFSET(10)]
+ stp x12, x13, [x18, #CPU_XREG_OFFSET(12)]
+ stp x14, x15, [x18, #CPU_XREG_OFFSET(14)]
+ stp x16, x17, [x18, #CPU_XREG_OFFSET(16)]
+
+ ret
+SYM_CODE_END(__kvm_hyp_host_forward_smc)
diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index a4f1cac714d7..f25680ede080 100644
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -18,6 +18,8 @@
DEFINE_PER_CPU(struct kvm_nvhe_init_params, kvm_init_params);
+void __kvm_hyp_host_forward_smc(struct kvm_cpu_context *host_ctxt);
+
static void handle___kvm_vcpu_run(struct kvm_cpu_context *host_ctxt)
{
DECLARE_REG(struct kvm_vcpu *, vcpu, host_ctxt, 1);
@@ -152,12 +154,39 @@ static void handle_host_hcall(struct kvm_cpu_context
*host_ctxt)
cpu_reg(host_ctxt, 0) = SMCCC_RET_NOT_SUPPORTED;
}
+static void default_host_smc_handler(struct kvm_cpu_context *host_ctxt)
+{
+ __kvm_hyp_host_forward_smc(host_ctxt);
+}
+
+static void skip_host_instruction(void)
+{
+ write_sysreg_el2(read_sysreg_el2(SYS_ELR) + 4, SYS_ELR);
+}
+
+static void handle_host_smc(struct kvm_cpu_context *host_ctxt)
+{
+ default_host_smc_handler(host_ctxt);
+
+ /*
+* Unlike HVC, the return address of an SMC is the instruction's PC.
+* Move the return address past the instruction.
+*/
+ skip_host_instruction();
+}
+
void handle_trap(struct kvm_cpu_context *host_ctxt)
{
u64 esr = read_sysreg_el2(SYS_ESR);
- if (unlikely(ESR_ELx_EC(esr) != ESR_ELx_EC_HVC64))
+ switch (ESR_ELx_EC(esr)) {
+ case ESR_ELx_EC_HVC64:
+ handle_host_hcall(host_ctxt);
+ break;
+ case ESR_ELx_EC_SMC64:
+ handle_host_smc(host_ctxt);
+ break;
+ default:
hyp_panic();
-
- handle_host_hcall(host_ctxt);
+ }
}
--
2.29.2.454.gaff20da3a2-goog
