tree: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git master head: 5695e51619745d4fe3ec2506a2f0cd982c5e27a4 commit: 7cca2d0b7e7d9f3cd740d41afdc00051c9b508a0 KVM: x86/mmu: Protect TDP MMU page table memory with RCU date: 3 weeks ago config: x86_64-randconfig-s022-20210228 (attached as .config) compiler: gcc-9 (Debian 9.3.0-15) 9.3.0 reproduce: # apt-get install sparse # sparse version: v0.6.3-241-geaceeafa-dirty # https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=7cca2d0b7e7d9f3cd740d41afdc00051c9b508a0 git remote add linus https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git git fetch --no-tags linus master git checkout 7cca2d0b7e7d9f3cd740d41afdc00051c9b508a0 # save the attached .config to linux build tree make W=1 C=1 CF='-fdiagnostic-prefix -D__CHECK_ENDIAN__' ARCH=x86_64
If you fix the issue, kindly add following tag as appropriate
Reported-by: kernel test robot <l...@intel.com>
"sparse warnings: (new ones prefixed by >>)"
>> arch/x86/kvm/mmu/tdp_mmu.c:388:49: sparse: sparse: incorrect type in
>> argument 2 (different address spaces) @@ expected unsigned long long
>> [usertype] *pt @@ got unsigned long long [noderef] [usertype] __rcu * @@
arch/x86/kvm/mmu/tdp_mmu.c:388:49: sparse: expected unsigned long long
[usertype] *pt
arch/x86/kvm/mmu/tdp_mmu.c:388:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *
>> arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: sparse: incorrect type in
>> argument 2 (different address spaces) @@ expected unsigned long long
>> [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu
>> * @@
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: expected unsigned long long
[usertype] *root_pt
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: got unsigned long long
[noderef] [usertype] __rcu *
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: sparse: incorrect type in
>> argument 2 (different address spaces) @@ expected unsigned long long
>> [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu
>> * @@
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: expected unsigned long long
[usertype] *root_pt
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: got unsigned long long
[noderef] [usertype] __rcu *
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: sparse: incorrect type in
>> argument 2 (different address spaces) @@ expected unsigned long long
>> [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu
>> * @@
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: expected unsigned long long
[usertype] *root_pt
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: got unsigned long long
[noderef] [usertype] __rcu *
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: sparse: incorrect type in
>> argument 2 (different address spaces) @@ expected unsigned long long
>> [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu
>> * @@
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: expected unsigned long long
[usertype] *root_pt
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: got unsigned long long
[noderef] [usertype] __rcu *
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: sparse: incorrect type in
>> argument 2 (different address spaces) @@ expected unsigned long long
>> [usertype] *root_pt @@ got unsigned long long [noderef] [usertype] __rcu
>> * @@
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: expected unsigned long long
[usertype] *root_pt
arch/x86/kvm/mmu/tdp_mmu.c:506:51: sparse: got unsigned long long
[noderef] [usertype] __rcu *
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
>> arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: sparse: incorrect type in
>> argument 1 (different address spaces) @@ expected unsigned long long
>> [usertype] *sptep @@ got unsigned long long [noderef] [usertype] __rcu
>> *[usertype] root_pt @@
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: expected unsigned long long
[usertype] *sptep
arch/x86/kvm/mmu/tdp_mmu.c:421:49: sparse: got unsigned long long
[noderef] [usertype] __rcu *[usertype] root_pt
vim +388 arch/x86/kvm/mmu/tdp_mmu.c
a066e61f13cf4b Ben Gardon 2021-02-02 302
2f2fad0897cbfd Ben Gardon 2020-10-14 303 /**
2f2fad0897cbfd Ben Gardon 2020-10-14 304 * handle_changed_spte -
handle bookkeeping associated with an SPTE change
2f2fad0897cbfd Ben Gardon 2020-10-14 305 * @kvm: kvm instance
2f2fad0897cbfd Ben Gardon 2020-10-14 306 * @as_id: the address
space of the paging structure the SPTE was a part of
2f2fad0897cbfd Ben Gardon 2020-10-14 307 * @gfn: the base GFN that
was mapped by the SPTE
2f2fad0897cbfd Ben Gardon 2020-10-14 308 * @old_spte: The value of
the SPTE before the change
2f2fad0897cbfd Ben Gardon 2020-10-14 309 * @new_spte: The value of
the SPTE after the change
2f2fad0897cbfd Ben Gardon 2020-10-14 310 * @level: the level of the
PT the SPTE is part of in the paging structure
2f2fad0897cbfd Ben Gardon 2020-10-14 311 *
2f2fad0897cbfd Ben Gardon 2020-10-14 312 * Handle bookkeeping that
might result from the modification of a SPTE.
2f2fad0897cbfd Ben Gardon 2020-10-14 313 * This function must be
called for all TDP SPTE modifications.
2f2fad0897cbfd Ben Gardon 2020-10-14 314 */
2f2fad0897cbfd Ben Gardon 2020-10-14 315 static void
__handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
2f2fad0897cbfd Ben Gardon 2020-10-14 316
u64 old_spte, u64 new_spte, int level)
2f2fad0897cbfd Ben Gardon 2020-10-14 317 {
2f2fad0897cbfd Ben Gardon 2020-10-14 318 bool was_present =
is_shadow_present_pte(old_spte);
2f2fad0897cbfd Ben Gardon 2020-10-14 319 bool is_present =
is_shadow_present_pte(new_spte);
2f2fad0897cbfd Ben Gardon 2020-10-14 320 bool was_leaf =
was_present && is_last_spte(old_spte, level);
2f2fad0897cbfd Ben Gardon 2020-10-14 321 bool is_leaf =
is_present && is_last_spte(new_spte, level);
2f2fad0897cbfd Ben Gardon 2020-10-14 322 bool pfn_changed =
spte_to_pfn(old_spte) != spte_to_pfn(new_spte);
2f2fad0897cbfd Ben Gardon 2020-10-14 323
2f2fad0897cbfd Ben Gardon 2020-10-14 324 WARN_ON(level >
PT64_ROOT_MAX_LEVEL);
2f2fad0897cbfd Ben Gardon 2020-10-14 325 WARN_ON(level <
PG_LEVEL_4K);
764388ce598f0c Sean Christopherson 2020-10-23 326 WARN_ON(gfn &
(KVM_PAGES_PER_HPAGE(level) - 1));
2f2fad0897cbfd Ben Gardon 2020-10-14 327
2f2fad0897cbfd Ben Gardon 2020-10-14 328 /*
2f2fad0897cbfd Ben Gardon 2020-10-14 329 * If this warning were
to trigger it would indicate that there was a
2f2fad0897cbfd Ben Gardon 2020-10-14 330 * missing MMU notifier
or a race with some notifier handler.
2f2fad0897cbfd Ben Gardon 2020-10-14 331 * A present, leaf SPTE
should never be directly replaced with another
2f2fad0897cbfd Ben Gardon 2020-10-14 332 * present leaf SPTE
pointing to a differnt PFN. A notifier handler
2f2fad0897cbfd Ben Gardon 2020-10-14 333 * should be zapping
the SPTE before the main MM's page table is
2f2fad0897cbfd Ben Gardon 2020-10-14 334 * changed, or the SPTE
should be zeroed, and the TLBs flushed by the
2f2fad0897cbfd Ben Gardon 2020-10-14 335 * thread before
replacement.
2f2fad0897cbfd Ben Gardon 2020-10-14 336 */
2f2fad0897cbfd Ben Gardon 2020-10-14 337 if (was_leaf && is_leaf
&& pfn_changed) {
2f2fad0897cbfd Ben Gardon 2020-10-14 338 pr_err("Invalid
SPTE change: cannot replace a present leaf\n"
2f2fad0897cbfd Ben Gardon 2020-10-14 339 "SPTE
with another present leaf SPTE mapping a\n"
2f2fad0897cbfd Ben Gardon 2020-10-14 340
"different PFN!\n"
2f2fad0897cbfd Ben Gardon 2020-10-14 341 "as_id:
%d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
2f2fad0897cbfd Ben Gardon 2020-10-14 342 as_id,
gfn, old_spte, new_spte, level);
2f2fad0897cbfd Ben Gardon 2020-10-14 343
2f2fad0897cbfd Ben Gardon 2020-10-14 344 /*
2f2fad0897cbfd Ben Gardon 2020-10-14 345 * Crash the
host to prevent error propagation and guest data
2f2fad0897cbfd Ben Gardon 2020-10-14 346 * courruption.
2f2fad0897cbfd Ben Gardon 2020-10-14 347 */
2f2fad0897cbfd Ben Gardon 2020-10-14 348 BUG();
2f2fad0897cbfd Ben Gardon 2020-10-14 349 }
2f2fad0897cbfd Ben Gardon 2020-10-14 350
2f2fad0897cbfd Ben Gardon 2020-10-14 351 if (old_spte ==
new_spte)
2f2fad0897cbfd Ben Gardon 2020-10-14 352 return;
2f2fad0897cbfd Ben Gardon 2020-10-14 353
b9a98c3437e353 Ben Gardon 2020-10-27 354
trace_kvm_tdp_mmu_spte_changed(as_id, gfn, level, old_spte, new_spte);
b9a98c3437e353 Ben Gardon 2020-10-27 355
2f2fad0897cbfd Ben Gardon 2020-10-14 356 /*
2f2fad0897cbfd Ben Gardon 2020-10-14 357 * The only times a
SPTE should be changed from a non-present to
2f2fad0897cbfd Ben Gardon 2020-10-14 358 * non-present state is
when an MMIO entry is installed/modified/
2f2fad0897cbfd Ben Gardon 2020-10-14 359 * removed. In that
case, there is nothing to do here.
2f2fad0897cbfd Ben Gardon 2020-10-14 360 */
2f2fad0897cbfd Ben Gardon 2020-10-14 361 if (!was_present &&
!is_present) {
2f2fad0897cbfd Ben Gardon 2020-10-14 362 /*
2f2fad0897cbfd Ben Gardon 2020-10-14 363 * If this
change does not involve a MMIO SPTE, it is
2f2fad0897cbfd Ben Gardon 2020-10-14 364 * unexpected.
Log the change, though it should not impact the
2f2fad0897cbfd Ben Gardon 2020-10-14 365 * guest since
both the former and current SPTEs are nonpresent.
2f2fad0897cbfd Ben Gardon 2020-10-14 366 */
2f2fad0897cbfd Ben Gardon 2020-10-14 367 if
(WARN_ON(!is_mmio_spte(old_spte) && !is_mmio_spte(new_spte)))
2f2fad0897cbfd Ben Gardon 2020-10-14 368
pr_err("Unexpected SPTE change! Nonpresent SPTEs\n"
2f2fad0897cbfd Ben Gardon 2020-10-14 369
"should not be replaced with another,\n"
2f2fad0897cbfd Ben Gardon 2020-10-14 370
"different nonpresent SPTE, unless one or both\n"
2f2fad0897cbfd Ben Gardon 2020-10-14 371
"are MMIO SPTEs.\n"
2f2fad0897cbfd Ben Gardon 2020-10-14 372
"as_id: %d gfn: %llx old_spte: %llx new_spte: %llx level: %d",
2f2fad0897cbfd Ben Gardon 2020-10-14 373
as_id, gfn, old_spte, new_spte, level);
2f2fad0897cbfd Ben Gardon 2020-10-14 374 return;
2f2fad0897cbfd Ben Gardon 2020-10-14 375 }
2f2fad0897cbfd Ben Gardon 2020-10-14 376
2f2fad0897cbfd Ben Gardon 2020-10-14 377
2f2fad0897cbfd Ben Gardon 2020-10-14 378 if (was_leaf &&
is_dirty_spte(old_spte) &&
2f2fad0897cbfd Ben Gardon 2020-10-14 379
(!is_dirty_spte(new_spte) || pfn_changed))
2f2fad0897cbfd Ben Gardon 2020-10-14 380
kvm_set_pfn_dirty(spte_to_pfn(old_spte));
2f2fad0897cbfd Ben Gardon 2020-10-14 381
2f2fad0897cbfd Ben Gardon 2020-10-14 382 /*
2f2fad0897cbfd Ben Gardon 2020-10-14 383 * Recursively handle
child PTs if the change removed a subtree from
2f2fad0897cbfd Ben Gardon 2020-10-14 384 * the paging structure.
2f2fad0897cbfd Ben Gardon 2020-10-14 385 */
a066e61f13cf4b Ben Gardon 2021-02-02 386 if (was_present &&
!was_leaf && (pfn_changed || !is_present))
a066e61f13cf4b Ben Gardon 2021-02-02 387
handle_removed_tdp_mmu_page(kvm,
a066e61f13cf4b Ben Gardon 2021-02-02 @388
spte_to_child_pt(old_spte, level));
2f2fad0897cbfd Ben Gardon 2020-10-14 389 }
2f2fad0897cbfd Ben Gardon 2020-10-14 390
2f2fad0897cbfd Ben Gardon 2020-10-14 391 static void
handle_changed_spte(struct kvm *kvm, int as_id, gfn_t gfn,
2f2fad0897cbfd Ben Gardon 2020-10-14 392
u64 old_spte, u64 new_spte, int level)
2f2fad0897cbfd Ben Gardon 2020-10-14 393 {
2f2fad0897cbfd Ben Gardon 2020-10-14 394
__handle_changed_spte(kvm, as_id, gfn, old_spte, new_spte, level);
f8e144971c6834 Ben Gardon 2020-10-14 395
handle_changed_spte_acc_track(old_spte, new_spte, level);
a6a0b05da9f37f Ben Gardon 2020-10-14 396
handle_changed_spte_dirty_log(kvm, as_id, gfn, old_spte,
a6a0b05da9f37f Ben Gardon 2020-10-14 397
new_spte, level);
2f2fad0897cbfd Ben Gardon 2020-10-14 398 }
faaf05b00aecdb Ben Gardon 2020-10-14 399
fe43fa2f407b9d Ben Gardon 2021-02-02 400 /*
fe43fa2f407b9d Ben Gardon 2021-02-02 401 * __tdp_mmu_set_spte - Set
a TDP MMU SPTE and handle the associated bookkeeping
fe43fa2f407b9d Ben Gardon 2021-02-02 402 * @kvm: kvm instance
fe43fa2f407b9d Ben Gardon 2021-02-02 403 * @iter: a tdp_iter
instance currently on the SPTE that should be set
fe43fa2f407b9d Ben Gardon 2021-02-02 404 * @new_spte: The value the
SPTE should be set to
fe43fa2f407b9d Ben Gardon 2021-02-02 405 * @record_acc_track:
Notify the MM subsystem of changes to the accessed state
fe43fa2f407b9d Ben Gardon 2021-02-02 406 * of the
page. Should be set unless handling an MMU
fe43fa2f407b9d Ben Gardon 2021-02-02 407 * notifier
for access tracking. Leaving record_acc_track
fe43fa2f407b9d Ben Gardon 2021-02-02 408 * unset in
that case prevents page accesses from being
fe43fa2f407b9d Ben Gardon 2021-02-02 409 * double
counted.
fe43fa2f407b9d Ben Gardon 2021-02-02 410 * @record_dirty_log:
Record the page as dirty in the dirty bitmap if
fe43fa2f407b9d Ben Gardon 2021-02-02 411 *
appropriate for the change being made. Should be set
fe43fa2f407b9d Ben Gardon 2021-02-02 412 * unless
performing certain dirty logging operations.
fe43fa2f407b9d Ben Gardon 2021-02-02 413 * Leaving
record_dirty_log unset in that case prevents page
fe43fa2f407b9d Ben Gardon 2021-02-02 414 * writes
from being double counted.
fe43fa2f407b9d Ben Gardon 2021-02-02 415 */
f8e144971c6834 Ben Gardon 2020-10-14 416 static inline void
__tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter,
a6a0b05da9f37f Ben Gardon 2020-10-14 417
u64 new_spte, bool record_acc_track,
a6a0b05da9f37f Ben Gardon 2020-10-14 418
bool record_dirty_log)
faaf05b00aecdb Ben Gardon 2020-10-14 419 {
7cca2d0b7e7d9f Ben Gardon 2021-02-02 420 tdp_ptep_t root_pt =
tdp_iter_root_pt(iter);
faaf05b00aecdb Ben Gardon 2020-10-14 @421 struct kvm_mmu_page
*root = sptep_to_sp(root_pt);
faaf05b00aecdb Ben Gardon 2020-10-14 422 int as_id =
kvm_mmu_page_as_id(root);
faaf05b00aecdb Ben Gardon 2020-10-14 423
3a9a4aa5657471 Ben Gardon 2021-02-02 424
lockdep_assert_held(&kvm->mmu_lock);
3a9a4aa5657471 Ben Gardon 2021-02-02 425
7cca2d0b7e7d9f Ben Gardon 2021-02-02 426
WRITE_ONCE(*rcu_dereference(iter->sptep), new_spte);
faaf05b00aecdb Ben Gardon 2020-10-14 427
f8e144971c6834 Ben Gardon 2020-10-14 428
__handle_changed_spte(kvm, as_id, iter->gfn, iter->old_spte, new_spte,
faaf05b00aecdb Ben Gardon 2020-10-14 429
iter->level);
f8e144971c6834 Ben Gardon 2020-10-14 430 if (record_acc_track)
f8e144971c6834 Ben Gardon 2020-10-14 431
handle_changed_spte_acc_track(iter->old_spte, new_spte,
f8e144971c6834 Ben Gardon 2020-10-14 432
iter->level);
a6a0b05da9f37f Ben Gardon 2020-10-14 433 if (record_dirty_log)
a6a0b05da9f37f Ben Gardon 2020-10-14 434
handle_changed_spte_dirty_log(kvm, as_id, iter->gfn,
a6a0b05da9f37f Ben Gardon 2020-10-14 435
iter->old_spte, new_spte,
a6a0b05da9f37f Ben Gardon 2020-10-14 436
iter->level);
f8e144971c6834 Ben Gardon 2020-10-14 437 }
f8e144971c6834 Ben Gardon 2020-10-14 438
f8e144971c6834 Ben Gardon 2020-10-14 439 static inline void
tdp_mmu_set_spte(struct kvm *kvm, struct tdp_iter *iter,
f8e144971c6834 Ben Gardon 2020-10-14 440
u64 new_spte)
f8e144971c6834 Ben Gardon 2020-10-14 441 {
a6a0b05da9f37f Ben Gardon 2020-10-14 442 __tdp_mmu_set_spte(kvm,
iter, new_spte, true, true);
f8e144971c6834 Ben Gardon 2020-10-14 443 }
f8e144971c6834 Ben Gardon 2020-10-14 444
f8e144971c6834 Ben Gardon 2020-10-14 445 static inline void
tdp_mmu_set_spte_no_acc_track(struct kvm *kvm,
f8e144971c6834 Ben Gardon 2020-10-14 446
struct tdp_iter *iter,
f8e144971c6834 Ben Gardon 2020-10-14 447
u64 new_spte)
f8e144971c6834 Ben Gardon 2020-10-14 448 {
a6a0b05da9f37f Ben Gardon 2020-10-14 449 __tdp_mmu_set_spte(kvm,
iter, new_spte, false, true);
a6a0b05da9f37f Ben Gardon 2020-10-14 450 }
a6a0b05da9f37f Ben Gardon 2020-10-14 451
a6a0b05da9f37f Ben Gardon 2020-10-14 452 static inline void
tdp_mmu_set_spte_no_dirty_log(struct kvm *kvm,
a6a0b05da9f37f Ben Gardon 2020-10-14 453
struct tdp_iter *iter,
a6a0b05da9f37f Ben Gardon 2020-10-14 454
u64 new_spte)
a6a0b05da9f37f Ben Gardon 2020-10-14 455 {
a6a0b05da9f37f Ben Gardon 2020-10-14 456 __tdp_mmu_set_spte(kvm,
iter, new_spte, true, false);
faaf05b00aecdb Ben Gardon 2020-10-14 457 }
faaf05b00aecdb Ben Gardon 2020-10-14 458
faaf05b00aecdb Ben Gardon 2020-10-14 459 #define
tdp_root_for_each_pte(_iter, _root, _start, _end) \
faaf05b00aecdb Ben Gardon 2020-10-14 460 for_each_tdp_pte(_iter,
_root->spt, _root->role.level, _start, _end)
faaf05b00aecdb Ben Gardon 2020-10-14 461
f8e144971c6834 Ben Gardon 2020-10-14 462 #define
tdp_root_for_each_leaf_pte(_iter, _root, _start, _end) \
f8e144971c6834 Ben Gardon 2020-10-14 463
tdp_root_for_each_pte(_iter, _root, _start, _end) \
f8e144971c6834 Ben Gardon 2020-10-14 464 if
(!is_shadow_present_pte(_iter.old_spte) || \
f8e144971c6834 Ben Gardon 2020-10-14 465
!is_last_spte(_iter.old_spte, _iter.level)) \
f8e144971c6834 Ben Gardon 2020-10-14 466
continue; \
f8e144971c6834 Ben Gardon 2020-10-14 467 else
f8e144971c6834 Ben Gardon 2020-10-14 468
bb18842e21111a Ben Gardon 2020-10-14 469 #define
tdp_mmu_for_each_pte(_iter, _mmu, _start, _end) \
bb18842e21111a Ben Gardon 2020-10-14 470 for_each_tdp_pte(_iter,
__va(_mmu->root_hpa), \
bb18842e21111a Ben Gardon 2020-10-14 471
_mmu->shadow_root_level, _start, _end)
bb18842e21111a Ben Gardon 2020-10-14 472
e28a436ca4f653 Ben Gardon 2021-02-02 473 /*
e28a436ca4f653 Ben Gardon 2021-02-02 474 * Yield if the MMU lock is
contended or this thread needs to return control
e28a436ca4f653 Ben Gardon 2021-02-02 475 * to the scheduler.
e28a436ca4f653 Ben Gardon 2021-02-02 476 *
e139a34ef9d562 Ben Gardon 2021-02-02 477 * If this function should
yield and flush is set, it will perform a remote
e139a34ef9d562 Ben Gardon 2021-02-02 478 * TLB flush before
yielding.
e139a34ef9d562 Ben Gardon 2021-02-02 479 *
e28a436ca4f653 Ben Gardon 2021-02-02 480 * If this function yields,
it will also reset the tdp_iter's walk over the
ed5e484b79e8a9 Ben Gardon 2021-02-02 481 * paging structure and the
calling function should skip to the next
ed5e484b79e8a9 Ben Gardon 2021-02-02 482 * iteration to allow the
iterator to continue its traversal from the
ed5e484b79e8a9 Ben Gardon 2021-02-02 483 * paging structure root.
e28a436ca4f653 Ben Gardon 2021-02-02 484 *
e28a436ca4f653 Ben Gardon 2021-02-02 485 * Return true if this
function yielded and the iterator's traversal was reset.
e28a436ca4f653 Ben Gardon 2021-02-02 486 * Return false if a yield
was not needed.
e28a436ca4f653 Ben Gardon 2021-02-02 487 */
e139a34ef9d562 Ben Gardon 2021-02-02 488 static inline bool
tdp_mmu_iter_cond_resched(struct kvm *kvm,
e139a34ef9d562 Ben Gardon 2021-02-02 489
struct tdp_iter *iter, bool flush)
a6a0b05da9f37f Ben Gardon 2020-10-14 490 {
ed5e484b79e8a9 Ben Gardon 2021-02-02 491 /* Ensure forward
progress has been made before yielding. */
ed5e484b79e8a9 Ben Gardon 2021-02-02 492 if
(iter->next_last_level_gfn == iter->yielded_gfn)
ed5e484b79e8a9 Ben Gardon 2021-02-02 493 return false;
ed5e484b79e8a9 Ben Gardon 2021-02-02 494
a6a0b05da9f37f Ben Gardon 2020-10-14 495 if (need_resched() ||
spin_needbreak(&kvm->mmu_lock)) {
7cca2d0b7e7d9f Ben Gardon 2021-02-02 496
rcu_read_unlock();
7cca2d0b7e7d9f Ben Gardon 2021-02-02 497
e139a34ef9d562 Ben Gardon 2021-02-02 498 if (flush)
e139a34ef9d562 Ben Gardon 2021-02-02 499
kvm_flush_remote_tlbs(kvm);
e139a34ef9d562 Ben Gardon 2021-02-02 500
a6a0b05da9f37f Ben Gardon 2020-10-14 501
cond_resched_lock(&kvm->mmu_lock);
7cca2d0b7e7d9f Ben Gardon 2021-02-02 502 rcu_read_lock();
ed5e484b79e8a9 Ben Gardon 2021-02-02 503
ed5e484b79e8a9 Ben Gardon 2021-02-02 504
WARN_ON(iter->gfn > iter->next_last_level_gfn);
ed5e484b79e8a9 Ben Gardon 2021-02-02 505
ed5e484b79e8a9 Ben Gardon 2021-02-02 @506
tdp_iter_start(iter, iter->pt_path[iter->root_level - 1],
ed5e484b79e8a9 Ben Gardon 2021-02-02 507
iter->root_level, iter->min_level,
ed5e484b79e8a9 Ben Gardon 2021-02-02 508
iter->next_last_level_gfn);
ed5e484b79e8a9 Ben Gardon 2021-02-02 509
e28a436ca4f653 Ben Gardon 2021-02-02 510 return true;
a6a0b05da9f37f Ben Gardon 2020-10-14 511 }
e28a436ca4f653 Ben Gardon 2021-02-02 512
e28a436ca4f653 Ben Gardon 2021-02-02 513 return false;
a6a0b05da9f37f Ben Gardon 2020-10-14 514 }
a6a0b05da9f37f Ben Gardon 2020-10-14 515
:::::: The code at line 388 was first introduced by commit
:::::: a066e61f13cf4b17d043ad8bea0cdde2b1e5ee49 KVM: x86/mmu: Factor out
handling of removed page tables
:::::: TO: Ben Gardon <bgar...@google.com>
:::::: CC: Paolo Bonzini <pbonz...@redhat.com>
---
0-DAY CI Kernel Test Service, Intel Corporation
https://lists.01.org/hyperkitty/list/kbuild-...@lists.01.org
.config.gz
Description: application/gzip
