Branch-free code is fun and everybody knows how much Avi loves it,
but last_pte_bitmap takes it a bit to the extreme. A logical | is
still branch-free and simpler than building the whole truth table into
last_pte_bitmap.
Signed-off-by: Paolo Bonzini <pbonz...@redhat.com>
---
arch/x86/include/asm/kvm_host.h | 6 +-----
arch/x86/kvm/mmu.c | 40 +++++++++++++++++++++++++---------------
2 files changed, 26 insertions(+), 20 deletions(-)
diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 7b5459982433..3b4c0b98f285 100644
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -338,11 +338,7 @@ struct kvm_mmu {
struct rsvd_bits_validate guest_rsvd_check;
- /*
- * Bitmap: bit set = last pte in walk
- * index[0:1]: level (zero-based)
- * index[2]: pte.ps
- */
+ /* bit [7-n] = can have large pages at (one-based) level n */
u8 last_pte_bitmap;
bool nx;
diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 6d47b5c43246..6bf8c36b681b 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3566,13 +3566,21 @@ static bool sync_mmio_spte(struct kvm_vcpu *vcpu, u64
*sptep, gfn_t gfn,
return false;
}
-static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned
gpte)
+static inline bool is_last_gpte(struct kvm_mmu *mmu,
+ unsigned level, unsigned gpte)
{
- unsigned index;
+ unsigned m1, m2;
+
+ /* PT_PAGE_TABLE_LEVEL always terminates. m1 has bit 7 set iff
+ * level <= PT_PAGE_TABLE_LEVEL, which for our purpose means
+ * level == PT_PAGE_TABLE_LEVEL (level is never zero). */
+ m1 = level - (PT_PAGE_TABLE_LEVEL + 1);
- index = level - 1;
- index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
- return mmu->last_pte_bitmap & (1 << index);
+ /* last_pte_bitmap is built so that shifting it left by level
+ * produces a 1-bit mask for gpte's PT_PAGE_SIZE_MASK bit.
+ */
+ m2 = gpte & (mmu->last_pte_bitmap << level);
+ return (m1 | m2) & PT_PAGE_SIZE_MASK;
}
#define PTTYPE_EPT 18 /* arbitrary */
@@ -3848,16 +3856,18 @@ static void update_last_pte_bitmap(struct kvm_vcpu
*vcpu, struct kvm_mmu *mmu)
{
u8 map;
unsigned level, root_level = mmu->root_level;
- const unsigned ps_set_index = 1 << 2; /* bit 2 of index: ps */
-
- if (root_level == PT32E_ROOT_LEVEL)
- --root_level;
- /* PT_PAGE_TABLE_LEVEL always terminates */
- map = 1 | (1 << ps_set_index);
- for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
- if (level <= PT_PDPE_LEVEL
- && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
- map |= 1 << (ps_set_index | (level - 1));
+
+ map = 0;
+ if (root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)) {
+ /* Apart from PSE, there are never large pages at the top level
+ * (512GB for 64-bit, 1GB for PAE).
+ */
+ if (root_level != PT32_ROOT_LEVEL)
+ --root_level;
+
+ BUILD_BUG_ON(PT_PAGE_SIZE_MASK > 255);
+ for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level)
+ map |= PT_PAGE_SIZE_MASK >> level;
}
mmu->last_pte_bitmap = map;
}
--
1.8.3.1
