[tip:x86/timers] x86/kvmclock: Switch kvmclock data to a PER_CPU variable
Commit-ID: 95a3d4454bb1cf5bfd666c27fdd2dc188e17c14d
Gitweb: https://git.kernel.org/tip/95a3d4454bb1cf5bfd666c27fdd2dc188e17c14d
Author: Thomas Gleixner
AuthorDate: Thu, 19 Jul 2018 16:55:26 -0400
Committer: Thomas Gleixner
CommitDate: Fri, 20 Jul 2018 00:02:38 +0200
x86/kvmclock: Switch kvmclock data to a PER_CPU variable
The previous removal of the memblock dependency from kvmclock introduced a
static data array sized 64bytes * CONFIG_NR_CPUS. That's wasteful on large
systems when kvmclock is not used.
Replace it with:
- A static page sized array of pvclock data. It's page sized because the
pvclock data of the boot cpu is mapped into the VDSO so otherwise random
other data would be exposed to the vDSO
- A PER_CPU variable of pvclock data pointers. This is used to access the
pcvlock data storage on each CPU.
The setup is done in two stages:
- Early boot stores the pointer to the static page for the boot CPU in
the per cpu data.
- In the preparatory stage of CPU hotplug assign either an element of
the static array (when the CPU number is in that range) or allocate
memory and initialize the per cpu pointer.
Signed-off-by: Thomas Gleixner
Signed-off-by: Pavel Tatashin
Acked-by: Paolo Bonzini
Cc: steven.sist...@oracle.com
Cc: daniel.m.jor...@oracle.com
Cc: li...@armlinux.org.uk
Cc: schwidef...@de.ibm.com
Cc: heiko.carst...@de.ibm.com
Cc: john.stu...@linaro.org
Cc: sb...@codeaurora.org
Cc: h...@zytor.com
Cc: douly.f...@cn.fujitsu.com
Cc: pet...@infradead.org
Cc: pra...@redhat.com
Cc: feng.t...@intel.com
Cc: pmla...@suse.com
Cc: gno...@lxorguk.ukuu.org.uk
Cc: linux-s...@vger.kernel.org
Cc: boris.ostrov...@oracle.com
Cc: jgr...@suse.com
Link: https://lkml.kernel.org/r/20180719205545.16512-8-pasha.tatas...@oracle.com
---
arch/x86/kernel/kvmclock.c | 99 +-
1 file changed, 62 insertions(+), 37 deletions(-)
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index 7d690d2238f8..91b94c0ae4e3 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -23,6 +23,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -55,12 +56,23 @@ early_param("no-kvmclock-vsyscall",
parse_no_kvmclock_vsyscall);
/* Aligned to page sizes to match whats mapped via vsyscalls to userspace */
#define HV_CLOCK_SIZE (sizeof(struct pvclock_vsyscall_time_info) * NR_CPUS)
+#define HVC_BOOT_ARRAY_SIZE \
+ (PAGE_SIZE / sizeof(struct pvclock_vsyscall_time_info))
-static u8 hv_clock_mem[PAGE_ALIGN(HV_CLOCK_SIZE)] __aligned(PAGE_SIZE);
-
-/* The hypervisor will put information about time periodically here */
-static struct pvclock_vsyscall_time_info *hv_clock __ro_after_init;
+static struct pvclock_vsyscall_time_info
+ hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __aligned(PAGE_SIZE);
static struct pvclock_wall_clock wall_clock;
+static DEFINE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu);
+
+static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void)
+{
+ return _cpu_read(hv_clock_per_cpu)->pvti;
+}
+
+static inline struct pvclock_vsyscall_time_info *this_cpu_hvclock(void)
+{
+ return this_cpu_read(hv_clock_per_cpu);
+}
/*
* The wallclock is the time of day when we booted. Since then, some time may
@@ -69,17 +81,10 @@ static struct pvclock_wall_clock wall_clock;
*/
static void kvm_get_wallclock(struct timespec64 *now)
{
- struct pvclock_vcpu_time_info *vcpu_time;
- int cpu;
-
wrmsrl(msr_kvm_wall_clock, slow_virt_to_phys(_clock));
-
- cpu = get_cpu();
-
- vcpu_time = _clock[cpu].pvti;
- pvclock_read_wallclock(_clock, vcpu_time, now);
-
- put_cpu();
+ preempt_disable();
+ pvclock_read_wallclock(_clock, this_cpu_pvti(), now);
+ preempt_enable();
}
static int kvm_set_wallclock(const struct timespec64 *now)
@@ -89,14 +94,10 @@ static int kvm_set_wallclock(const struct timespec64 *now)
static u64 kvm_clock_read(void)
{
- struct pvclock_vcpu_time_info *src;
u64 ret;
- int cpu;
preempt_disable_notrace();
- cpu = smp_processor_id();
- src = _clock[cpu].pvti;
- ret = pvclock_clocksource_read(src);
+ ret = pvclock_clocksource_read(this_cpu_pvti());
preempt_enable_notrace();
return ret;
}
@@ -141,7 +142,7 @@ static inline void kvm_sched_clock_init(bool stable)
static unsigned long kvm_get_tsc_khz(void)
{
setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
- return pvclock_tsc_khz(_clock[0].pvti);
+ return pvclock_tsc_khz(this_cpu_pvti());
}
static void kvm_get_preset_lpj(void)
@@ -158,15 +159,14 @@ static void kvm_get_preset_lpj(void)
bool kvm_check_and_clear_guest_paused(void)
{
- struct pvclock_vcpu_time_info *src;
+ struct pvclock_vsyscall_time_info *src = this_cpu_hvclock();
bool ret = false;
- if (!hv_clock)
+ if (!src)
[tip:x86/timers] x86/kvmclock: Switch kvmclock data to a PER_CPU variable
Commit-ID: 95a3d4454bb1cf5bfd666c27fdd2dc188e17c14d
Gitweb: https://git.kernel.org/tip/95a3d4454bb1cf5bfd666c27fdd2dc188e17c14d
Author: Thomas Gleixner
AuthorDate: Thu, 19 Jul 2018 16:55:26 -0400
Committer: Thomas Gleixner
CommitDate: Fri, 20 Jul 2018 00:02:38 +0200
x86/kvmclock: Switch kvmclock data to a PER_CPU variable
The previous removal of the memblock dependency from kvmclock introduced a
static data array sized 64bytes * CONFIG_NR_CPUS. That's wasteful on large
systems when kvmclock is not used.
Replace it with:
- A static page sized array of pvclock data. It's page sized because the
pvclock data of the boot cpu is mapped into the VDSO so otherwise random
other data would be exposed to the vDSO
- A PER_CPU variable of pvclock data pointers. This is used to access the
pcvlock data storage on each CPU.
The setup is done in two stages:
- Early boot stores the pointer to the static page for the boot CPU in
the per cpu data.
- In the preparatory stage of CPU hotplug assign either an element of
the static array (when the CPU number is in that range) or allocate
memory and initialize the per cpu pointer.
Signed-off-by: Thomas Gleixner
Signed-off-by: Pavel Tatashin
Acked-by: Paolo Bonzini
Cc: steven.sist...@oracle.com
Cc: daniel.m.jor...@oracle.com
Cc: li...@armlinux.org.uk
Cc: schwidef...@de.ibm.com
Cc: heiko.carst...@de.ibm.com
Cc: john.stu...@linaro.org
Cc: sb...@codeaurora.org
Cc: h...@zytor.com
Cc: douly.f...@cn.fujitsu.com
Cc: pet...@infradead.org
Cc: pra...@redhat.com
Cc: feng.t...@intel.com
Cc: pmla...@suse.com
Cc: gno...@lxorguk.ukuu.org.uk
Cc: linux-s...@vger.kernel.org
Cc: boris.ostrov...@oracle.com
Cc: jgr...@suse.com
Link: https://lkml.kernel.org/r/20180719205545.16512-8-pasha.tatas...@oracle.com
---
arch/x86/kernel/kvmclock.c | 99 +-
1 file changed, 62 insertions(+), 37 deletions(-)
diff --git a/arch/x86/kernel/kvmclock.c b/arch/x86/kernel/kvmclock.c
index 7d690d2238f8..91b94c0ae4e3 100644
--- a/arch/x86/kernel/kvmclock.c
+++ b/arch/x86/kernel/kvmclock.c
@@ -23,6 +23,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -55,12 +56,23 @@ early_param("no-kvmclock-vsyscall",
parse_no_kvmclock_vsyscall);
/* Aligned to page sizes to match whats mapped via vsyscalls to userspace */
#define HV_CLOCK_SIZE (sizeof(struct pvclock_vsyscall_time_info) * NR_CPUS)
+#define HVC_BOOT_ARRAY_SIZE \
+ (PAGE_SIZE / sizeof(struct pvclock_vsyscall_time_info))
-static u8 hv_clock_mem[PAGE_ALIGN(HV_CLOCK_SIZE)] __aligned(PAGE_SIZE);
-
-/* The hypervisor will put information about time periodically here */
-static struct pvclock_vsyscall_time_info *hv_clock __ro_after_init;
+static struct pvclock_vsyscall_time_info
+ hv_clock_boot[HVC_BOOT_ARRAY_SIZE] __aligned(PAGE_SIZE);
static struct pvclock_wall_clock wall_clock;
+static DEFINE_PER_CPU(struct pvclock_vsyscall_time_info *, hv_clock_per_cpu);
+
+static inline struct pvclock_vcpu_time_info *this_cpu_pvti(void)
+{
+ return _cpu_read(hv_clock_per_cpu)->pvti;
+}
+
+static inline struct pvclock_vsyscall_time_info *this_cpu_hvclock(void)
+{
+ return this_cpu_read(hv_clock_per_cpu);
+}
/*
* The wallclock is the time of day when we booted. Since then, some time may
@@ -69,17 +81,10 @@ static struct pvclock_wall_clock wall_clock;
*/
static void kvm_get_wallclock(struct timespec64 *now)
{
- struct pvclock_vcpu_time_info *vcpu_time;
- int cpu;
-
wrmsrl(msr_kvm_wall_clock, slow_virt_to_phys(_clock));
-
- cpu = get_cpu();
-
- vcpu_time = _clock[cpu].pvti;
- pvclock_read_wallclock(_clock, vcpu_time, now);
-
- put_cpu();
+ preempt_disable();
+ pvclock_read_wallclock(_clock, this_cpu_pvti(), now);
+ preempt_enable();
}
static int kvm_set_wallclock(const struct timespec64 *now)
@@ -89,14 +94,10 @@ static int kvm_set_wallclock(const struct timespec64 *now)
static u64 kvm_clock_read(void)
{
- struct pvclock_vcpu_time_info *src;
u64 ret;
- int cpu;
preempt_disable_notrace();
- cpu = smp_processor_id();
- src = _clock[cpu].pvti;
- ret = pvclock_clocksource_read(src);
+ ret = pvclock_clocksource_read(this_cpu_pvti());
preempt_enable_notrace();
return ret;
}
@@ -141,7 +142,7 @@ static inline void kvm_sched_clock_init(bool stable)
static unsigned long kvm_get_tsc_khz(void)
{
setup_force_cpu_cap(X86_FEATURE_TSC_KNOWN_FREQ);
- return pvclock_tsc_khz(_clock[0].pvti);
+ return pvclock_tsc_khz(this_cpu_pvti());
}
static void kvm_get_preset_lpj(void)
@@ -158,15 +159,14 @@ static void kvm_get_preset_lpj(void)
bool kvm_check_and_clear_guest_paused(void)
{
- struct pvclock_vcpu_time_info *src;
+ struct pvclock_vsyscall_time_info *src = this_cpu_hvclock();
bool ret = false;
- if (!hv_clock)
+ if (!src)
