Signed-off-by: Yanan Wang <wangyana...@huawei.com>
---
arch/arm64/kvm/hyp/pgtable.c | 73 +++++++++++++++++++++++++-----------
1 file changed, 52 insertions(+), 21 deletions(-)
diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index 23a01dfcb27a..f8b3248cef1c 100644
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -45,6 +45,8 @@
#define KVM_PTE_LEAF_ATTR_HI_S2_XN BIT(54)
+#define KVM_PTE_LEAF_ATTR_PERMS (GENMASK(7, 6) | BIT(54))
+
struct kvm_pgtable_walk_data {
struct kvm_pgtable *pgt;
struct kvm_pgtable_walker *walker;
@@ -170,10 +172,9 @@ static void kvm_set_table_pte(kvm_pte_t *ptep,
kvm_pte_t *childp)
smp_store_release(ptep, pte);
}
-static bool kvm_set_valid_leaf_pte(kvm_pte_t *ptep, u64 pa,
kvm_pte_t attr,
- u32 level)
+static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, u32
level)
{
- kvm_pte_t old = *ptep, pte = kvm_phys_to_pte(pa);
+ kvm_pte_t pte = kvm_phys_to_pte(pa);
u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ?
KVM_PTE_TYPE_PAGE :
KVM_PTE_TYPE_BLOCK;
@@ -181,12 +182,7 @@ static bool kvm_set_valid_leaf_pte(kvm_pte_t
*ptep, u64 pa, kvm_pte_t attr,
pte |= FIELD_PREP(KVM_PTE_TYPE, type);
pte |= KVM_PTE_VALID;
- /* Tolerate KVM recreating the exact same mapping. */
- if (kvm_pte_valid(old))
- return old == pte;
-
- smp_store_release(ptep, pte);
- return true;
+ return pte;
}
static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data
*data, u64 addr,
@@ -341,12 +337,17 @@ static int hyp_map_set_prot_attr(enum
kvm_pgtable_prot prot,
static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level,
kvm_pte_t *ptep, struct hyp_map_data *data)
{
+ kvm_pte_t new, old = *ptep;
u64 granule = kvm_granule_size(level), phys = data->phys;
if (!kvm_block_mapping_supported(addr, end, phys, level))
return false;
- WARN_ON(!kvm_set_valid_leaf_pte(ptep, phys, data->attr, level));
+ /* Tolerate KVM recreating the exact same mapping. */
+ new = kvm_init_valid_leaf_pte(phys, data->attr, level);
+ if (old != new && !WARN_ON(kvm_pte_valid(old)))
+ smp_store_release(ptep, new);
+
data->phys += granule;
return true;
}
@@ -461,25 +462,56 @@ static int stage2_map_set_prot_attr(enum
kvm_pgtable_prot prot,
return 0;
}
+static bool stage2_set_valid_leaf_pte_pre(u64 addr, u32 level,
+ kvm_pte_t *ptep, kvm_pte_t new,
+ struct stage2_map_data *data)
+{
+ kvm_pte_t old = *ptep, old_attr, new_attr;
+
+ if ((old ^ new) & (~KVM_PTE_LEAF_ATTR_PERMS))
+ return false;
+
+ /*
+ * Skip updating if we are trying to recreate exactly the same
mapping
+ * or to reduce the access permissions only. And update the
valid leaf
+ * PTE without break-before-make if we are trying to add more
access
+ * permissions only.
+ */
+ old_attr = (old & KVM_PTE_LEAF_ATTR_PERMS) ^
KVM_PTE_LEAF_ATTR_HI_S2_XN;
+ new_attr = (new & KVM_PTE_LEAF_ATTR_PERMS) ^
KVM_PTE_LEAF_ATTR_HI_S2_XN;
+ if (new_attr <= old_attr)
+ return true;
+
+ WRITE_ONCE(*ptep, new);
+ kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level);