Module Name:    src
Committed By:   ad
Date:           Thu May 21 21:12:31 UTC 2020

Modified Files:
        src/sys/arch/x86/acpi: acpi_wakeup.c
        src/sys/arch/x86/include: i82489var.h
        src/sys/arch/x86/x86: cpu.c lapic.c x86_machdep.c
        src/sys/arch/xen/x86: cpu.c
        src/sys/arch/xen/xen: hypervisor.c xen_clock.c

Log Message:
- Recalibrate the APIC timer using the TSC, once the TSC has in turn been
  recalibrated using the HPET.  This gets the clock interrupt firing more
  closely to HZ.

- Undo change with recent Xen merge and go back to starting the clocks in
  initclocks() on the boot CPU, and in cpu_hatch() on secondary CPUs.

- On reflection don't use HPET delay any more, it works very well but means
  going over the bus.  It's enough to use HPET to calibrate the TSC and
  APIC.

Tested on amd64 native, xen and xen PVH.


To generate a diff of this commit:
cvs rdiff -u -r1.52 -r1.53 src/sys/arch/x86/acpi/acpi_wakeup.c
cvs rdiff -u -r1.20 -r1.21 src/sys/arch/x86/include/i82489var.h
cvs rdiff -u -r1.191 -r1.192 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.80 -r1.81 src/sys/arch/x86/x86/lapic.c
cvs rdiff -u -r1.142 -r1.143 src/sys/arch/x86/x86/x86_machdep.c
cvs rdiff -u -r1.135 -r1.136 src/sys/arch/xen/x86/cpu.c
cvs rdiff -u -r1.84 -r1.85 src/sys/arch/xen/xen/hypervisor.c
cvs rdiff -u -r1.5 -r1.6 src/sys/arch/xen/xen/xen_clock.c

Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.

Modified files:

Index: src/sys/arch/x86/acpi/acpi_wakeup.c
diff -u src/sys/arch/x86/acpi/acpi_wakeup.c:1.52 src/sys/arch/x86/acpi/acpi_wakeup.c:1.53
--- src/sys/arch/x86/acpi/acpi_wakeup.c:1.52	Sat Feb 22 19:49:11 2020
+++ src/sys/arch/x86/acpi/acpi_wakeup.c	Thu May 21 21:12:30 2020
@@ -1,4 +1,4 @@
-/*	$NetBSD: acpi_wakeup.c,v 1.52 2020/02/22 19:49:11 chs Exp $	*/
+/*	$NetBSD: acpi_wakeup.c,v 1.53 2020/05/21 21:12:30 ad Exp $	*/
 
 /*-
  * Copyright (c) 2002, 2011 The NetBSD Foundation, Inc.
@@ -59,7 +59,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: acpi_wakeup.c,v 1.52 2020/02/22 19:49:11 chs Exp $");
+__KERNEL_RCSID(0, "$NetBSD: acpi_wakeup.c,v 1.53 2020/05/21 21:12:30 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -277,7 +277,7 @@ acpi_cpu_sleep(struct cpu_info *ci)
 #if NLAPIC > 0
 	lapic_enable();
 	lapic_set_lvt();
-	lapic_initclocks();
+	lapic_reset();
 #endif
 
 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);
@@ -351,7 +351,7 @@ acpi_md_sleep(int state)
 #if NLAPIC > 0
 	lapic_enable();
 	lapic_set_lvt();
-	lapic_initclocks();
+	lapic_reset();
 #endif
 #if NIOAPIC > 0
 	ioapic_reenable();

Index: src/sys/arch/x86/include/i82489var.h
diff -u src/sys/arch/x86/include/i82489var.h:1.20 src/sys/arch/x86/include/i82489var.h:1.21
--- src/sys/arch/x86/include/i82489var.h:1.20	Sun Dec  1 08:23:09 2019
+++ src/sys/arch/x86/include/i82489var.h	Thu May 21 21:12:30 2020
@@ -1,4 +1,4 @@
-/*	$NetBSD: i82489var.h,v 1.20 2019/12/01 08:23:09 maxv Exp $	*/
+/*	$NetBSD: i82489var.h,v 1.21 2020/05/21 21:12:30 ad Exp $	*/
 
 /*-
  * Copyright (c) 1998 The NetBSD Foundation, Inc.
@@ -88,8 +88,8 @@ struct cpu_info;
 extern void lapic_boot_init(paddr_t);
 extern void lapic_set_lvt(void);
 extern void lapic_enable(void);
-extern void lapic_calibrate_timer(struct cpu_info *ci);
-extern void lapic_initclocks(void);
+extern void lapic_calibrate_timer(bool);
+extern void lapic_reset(void);
 
 extern uint32_t lapic_readreg(u_int);
 extern void lapic_writereg(u_int, uint32_t);

Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.191 src/sys/arch/x86/x86/cpu.c:1.192
--- src/sys/arch/x86/x86/cpu.c:1.191	Tue May 12 06:32:05 2020
+++ src/sys/arch/x86/x86/cpu.c	Thu May 21 21:12:30 2020
@@ -1,4 +1,4 @@
-/*	$NetBSD: cpu.c,v 1.191 2020/05/12 06:32:05 msaitoh Exp $	*/
+/*	$NetBSD: cpu.c,v 1.192 2020/05/21 21:12:30 ad Exp $	*/
 
 /*
  * Copyright (c) 2000-2020 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.191 2020/05/12 06:32:05 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.192 2020/05/21 21:12:30 ad Exp $");
 
 #include "opt_ddb.h"
 #include "opt_mpbios.h"		/* for MPDEBUG */
@@ -456,7 +456,7 @@ cpu_attach(device_t parent, device_t sel
 			lapic_enable();
 			lapic_set_lvt();
 			if (!vm_guest_is_xenpvh_or_pvhvm())
-				lapic_calibrate_timer(ci);
+				lapic_calibrate_timer(false);
 		}
 #endif
 		kcsan_cpu_init(ci);
@@ -471,7 +471,6 @@ cpu_attach(device_t parent, device_t sel
 		cpu_identify(ci);
 		x86_errata();
 		x86_cpu_idle_init();
-		(*x86_cpu_initclock_func)();
 #ifdef XENPVHVM
 		xen_hvm_init_cpu(ci);
 #endif
@@ -485,7 +484,6 @@ cpu_attach(device_t parent, device_t sel
 #ifdef XENPVHVM
 		xen_hvm_init_cpu(ci);
 #endif
-		(*x86_cpu_initclock_func)();
 		break;
 
 #ifdef MULTIPROCESSOR
@@ -739,14 +737,6 @@ cpu_boot_secondary_processors(void)
 	kcpuset_t *cpus;
 	u_long i;
 
-#if NHPET > 0
-	/* Use HPET delay, and re-calibrate TSC on boot CPU using HPET. */
-	if (hpet_delay_p() && x86_delay == i8254_delay) {
-		delay_func = x86_delay = hpet_delay;
-		cpu_get_tsc_freq(curcpu());
-	}
-#endif
-
 	/* Now that we know the number of CPUs, patch the text segment. */
 	x86_patch(false);
 
@@ -1010,7 +1000,7 @@ cpu_hatch(void *v)
 #ifdef XENPVHVM
 	xen_hvm_init_cpu(ci);
 #endif
-	(*x86_cpu_initclock_func)();
+	(*x86_initclock_func)();
 	cpu_get_tsc_freq(ci);
 
 	s = splhigh();
@@ -1346,7 +1336,7 @@ cpu_get_tsc_freq(struct cpu_info *ci)
 			overhead = 0;
 			for (int i = 0; i <= 8; i++) {
 				t0 = cpu_counter();
-				x86_delay(0);
+				delay_func(0);
 				t1 = cpu_counter();
 				if (i > 0) {
 					overhead += (t1 - t0);
@@ -1356,7 +1346,7 @@ cpu_get_tsc_freq(struct cpu_info *ci)
 
 			/* Now do the calibration. */
 			t0 = cpu_counter();
-			x86_delay(100000);
+			delay_func(100000);
 			t1 = cpu_counter();
 			freq = (t1 - t0 - overhead) * 10;
 		}

Index: src/sys/arch/x86/x86/lapic.c
diff -u src/sys/arch/x86/x86/lapic.c:1.80 src/sys/arch/x86/x86/lapic.c:1.81
--- src/sys/arch/x86/x86/lapic.c:1.80	Wed May 20 02:23:35 2020
+++ src/sys/arch/x86/x86/lapic.c	Thu May 21 21:12:30 2020
@@ -1,4 +1,4 @@
-/*	$NetBSD: lapic.c,v 1.80 2020/05/20 02:23:35 msaitoh Exp $	*/
+/*	$NetBSD: lapic.c,v 1.81 2020/05/21 21:12:30 ad Exp $	*/
 
 /*-
  * Copyright (c) 2000, 2008, 2020 The NetBSD Foundation, Inc.
@@ -32,7 +32,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.80 2020/05/20 02:23:35 msaitoh Exp $");
+__KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.81 2020/05/21 21:12:30 ad Exp $");
 
 #include "acpica.h"
 #include "ioapic.h"
@@ -580,11 +580,10 @@ lapic_clockintr(void *arg, struct intrfr
 }
 
 void
-lapic_initclocks(void)
+lapic_reset(void)
 {
+
 	/*
-	 * Start local apic countdown timer running, in repeated mode.
-	 *
 	 * Mask the clock interrupt and set mode,
 	 * then set divisor,
 	 * then unmask and set the vector.
@@ -598,6 +597,29 @@ lapic_initclocks(void)
 	lapic_writereg(LAPIC_EOI, 0);
 }
 
+static void
+lapic_initclock(void)
+{
+
+	if (curcpu() == &cpu_info_primary) {
+		/*
+		 * Recalibrate the timer using the cycle counter, now that
+		 * the cycle counter itself has been recalibrated.
+		 */
+		lapic_calibrate_timer(true);
+
+		/*
+		 * Hook up time counter.  This assume that all LAPICs have
+		 * the same frequency.
+		 */
+		lapic_timecounter.tc_frequency = lapic_per_second;
+		tc_init(&lapic_timecounter);
+	}
+
+	/* Start local apic countdown timer running, in repeated mode. */
+	lapic_reset();
+}
+
 /*
  * Calibrate the local apic count-down timer (which is running at
  * bus-clock speed) vs. the i8254 counter/timer (which is running at
@@ -610,50 +632,71 @@ lapic_initclocks(void)
  * We're actually using the IRQ0 timer.  Hmm.
  */
 void
-lapic_calibrate_timer(struct cpu_info *ci)
+lapic_calibrate_timer(bool secondpass)
 {
-	unsigned int seen, delta, initial_i8254, initial_lapic;
-	unsigned int cur_i8254, cur_lapic;
+	struct cpu_info *ci = curcpu();
 	uint64_t tmp;
 	int i;
 	char tbuf[9];
 
-	if (lapic_per_second != 0)
-		goto calibrate_done;
+	KASSERT(ci == &cpu_info_primary);
 
-	aprint_debug_dev(ci->ci_dev, "calibrating local timer\n");
+	aprint_debug_dev(ci->ci_dev, "[re]calibrating local timer\n");
 
 	/*
 	 * Configure timer to one-shot, interrupt masked,
 	 * large positive number.
 	 */
+	x86_disable_intr();
 	lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_MASKED);
 	lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
 	lapic_writereg(LAPIC_ICR_TIMER, 0x80000000);
+	(void)lapic_gettick();
 
-	x86_disable_intr();
+	if (secondpass && cpu_hascounter()) {
+		/*
+		 * Second pass calibration, using the TSC which has ideally
+		 * been calibrated using the HPET or information gleaned
+		 * from MSRs by this point.
+		 */
+		uint64_t l0, l1, t0, t1;
 
-	initial_lapic = lapic_gettick();
-	initial_i8254 = gettick();
+		(void)cpu_counter();
+		t0 = cpu_counter();
+		l0 = lapic_gettick();
+		t0 += cpu_counter();
+		DELAY(50000);
+		t1 = cpu_counter();
+		l1 = lapic_gettick();
+		t1 += cpu_counter();
+
+		tmp = (l0 - l1) * cpu_frequency(ci) / ((t1 - t0 + 1) / 2);
+		lapic_per_second = rounddown(tmp + 500, 1000);
+	} else if (lapic_per_second == 0) {
+		/*
+		 * Inaccurate first pass calibration using the i8254.
+		 */
+		unsigned int seen, delta, initial_i8254, initial_lapic;
+		unsigned int cur_i8254, cur_lapic;
 
-	for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) {
-		cur_i8254 = gettick();
-		if (cur_i8254 > initial_i8254)
-			delta = x86_rtclock_tval - (cur_i8254 - initial_i8254);
-		else
-			delta = initial_i8254 - cur_i8254;
-		initial_i8254 = cur_i8254;
+		(void)gettick();
+		initial_lapic = lapic_gettick();
+		initial_i8254 = gettick();
+		for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) {
+			cur_i8254 = gettick();
+			if (cur_i8254 > initial_i8254)
+				delta = x86_rtclock_tval - (cur_i8254 - initial_i8254);
+			else
+				delta = initial_i8254 - cur_i8254;
+			initial_i8254 = cur_i8254;
+		}
+		cur_lapic = lapic_gettick();
+		tmp = initial_lapic - cur_lapic;
+		lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen;
 	}
-	cur_lapic = lapic_gettick();
-
 	x86_enable_intr();
 
-	tmp = initial_lapic - cur_lapic;
-	lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen;
-
-calibrate_done:
 	humanize_number(tbuf, sizeof(tbuf), lapic_per_second, "Hz", 1000);
-
 	aprint_debug_dev(ci->ci_dev, "apic clock running at %s\n", tbuf);
 
 	if (lapic_per_second != 0) {
@@ -703,18 +746,10 @@ calibrate_done:
 		 * Now that the timer's calibrated, use the apic timer routines
 		 * for all our timing needs..
 		 */
-		delay_func = lapic_delay;
-		x86_cpu_initclock_func = lapic_initclocks;
-		x86_initclock_func = x86_dummy_initclock;
-		initrtclock(0);
-
-		if (lapic_timecounter.tc_frequency == 0) {
-			/*
-			 * Hook up time counter.
-			 * This assume that all LAPICs have the same frequency.
-			 */
-			lapic_timecounter.tc_frequency = lapic_per_second;
-			tc_init(&lapic_timecounter);
+		if (!secondpass) {
+			delay_func = lapic_delay;
+			x86_initclock_func = lapic_initclock;
+			initrtclock(0);
 		}
 	}
 }

Index: src/sys/arch/x86/x86/x86_machdep.c
diff -u src/sys/arch/x86/x86/x86_machdep.c:1.142 src/sys/arch/x86/x86/x86_machdep.c:1.143
--- src/sys/arch/x86/x86/x86_machdep.c:1.142	Sun May  3 17:22:03 2020
+++ src/sys/arch/x86/x86/x86_machdep.c	Thu May 21 21:12:30 2020
@@ -1,4 +1,4 @@
-/*	$NetBSD: x86_machdep.c,v 1.142 2020/05/03 17:22:03 bouyer Exp $	*/
+/*	$NetBSD: x86_machdep.c,v 1.143 2020/05/21 21:12:30 ad Exp $	*/
 
 /*-
  * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.142 2020/05/03 17:22:03 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.143 2020/05/21 21:12:30 ad Exp $");
 
 #include "opt_modular.h"
 #include "opt_physmem.h"
@@ -108,11 +108,9 @@ char module_machine_i386pae_xen[] = "i38
 #ifndef XENPV
 void (*delay_func)(unsigned int) = i8254_delay;
 void (*x86_initclock_func)(void) = i8254_initclocks;
-void (*x86_cpu_initclock_func)(void) = x86_dummy_initclock;
 #else /* XENPV */
 void (*delay_func)(unsigned int) = xen_delay;
 void (*x86_initclock_func)(void) = xen_initclocks;
-void (*x86_cpu_initclock_func)(void) = xen_cpu_initclocks;
 #endif
 
 
@@ -1500,10 +1498,12 @@ void
 cpu_initclocks(void)
 {
 
-	(*x86_initclock_func)();
-}
+	/*
+	 * Re-calibrate TSC on boot CPU using most accurate time source,
+	 * thus making accurate TSC available for x86_initclock_func().
+	 */
+	cpu_get_tsc_freq(curcpu());
 
-void
-x86_dummy_initclock(void)
-{
+	/* Now start the clocks on this CPU (the boot CPU). */
+	(*x86_initclock_func)();
 }

Index: src/sys/arch/xen/x86/cpu.c
diff -u src/sys/arch/xen/x86/cpu.c:1.135 src/sys/arch/xen/x86/cpu.c:1.136
--- src/sys/arch/xen/x86/cpu.c:1.135	Sat Apr 25 15:26:17 2020
+++ src/sys/arch/xen/x86/cpu.c	Thu May 21 21:12:31 2020
@@ -1,4 +1,4 @@
-/*	$NetBSD: cpu.c,v 1.135 2020/04/25 15:26:17 bouyer Exp $	*/
+/*	$NetBSD: cpu.c,v 1.136 2020/05/21 21:12:31 ad Exp $	*/
 
 /*-
  * Copyright (c) 2000 The NetBSD Foundation, Inc.
@@ -65,7 +65,7 @@
  */
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.135 2020/04/25 15:26:17 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.136 2020/05/21 21:12:31 ad Exp $");
 
 #include "opt_ddb.h"
 #include "opt_multiprocessor.h"
@@ -456,14 +456,12 @@ cpu_attach_common(device_t parent, devic
 		atomic_or_32(&ci->ci_flags, CPUF_SP);
 		cpu_identify(ci);
 		x86_cpu_idle_init();
-		xen_cpu_initclocks();
 		break;
 
 	case CPU_ROLE_BP:
 		atomic_or_32(&ci->ci_flags, CPUF_BSP);
 		cpu_identify(ci);
 		x86_cpu_idle_init();
-		xen_cpu_initclocks();
 		break;
 
 	case CPU_ROLE_AP:
@@ -726,7 +724,7 @@ cpu_hatch(void *v)
 
 	xen_ipi_init();
 
-	xen_cpu_initclocks();
+	xen_initclocks();
 
 #ifdef __x86_64__
 	fpuinit(ci);

Index: src/sys/arch/xen/xen/hypervisor.c
diff -u src/sys/arch/xen/xen/hypervisor.c:1.84 src/sys/arch/xen/xen/hypervisor.c:1.85
--- src/sys/arch/xen/xen/hypervisor.c:1.84	Sat May  9 08:01:38 2020
+++ src/sys/arch/xen/xen/hypervisor.c	Thu May 21 21:12:31 2020
@@ -1,4 +1,4 @@
-/* $NetBSD: hypervisor.c,v 1.84 2020/05/09 08:01:38 bouyer Exp $ */
+/* $NetBSD: hypervisor.c,v 1.85 2020/05/21 21:12:31 ad Exp $ */
 
 /*
  * Copyright (c) 2005 Manuel Bouyer.
@@ -53,7 +53,7 @@
 
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: hypervisor.c,v 1.84 2020/05/09 08:01:38 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: hypervisor.c,v 1.85 2020/05/21 21:12:31 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -260,7 +260,6 @@ init_xen_early(void)
 	}
 	delay_func = x86_delay = xen_delay;
 	x86_initclock_func = xen_initclocks;
-	x86_cpu_initclock_func = xen_cpu_initclocks;
 	if (hvm_start_info->cmdline_paddr != 0) {
 		cmd_line =
 		    (void *)((uintptr_t)hvm_start_info->cmdline_paddr + KERNBASE);
@@ -436,7 +435,6 @@ xen_hvm_init(void)
 
 	delay_func = x86_delay = xen_delay;
 	x86_initclock_func = xen_initclocks;
-	x86_cpu_initclock_func = xen_cpu_initclocks;
 
 	vm_guest = VM_GUEST_XENPVHVM; /* Be more specific */
 	return 1;

Index: src/sys/arch/xen/xen/xen_clock.c
diff -u src/sys/arch/xen/xen/xen_clock.c:1.5 src/sys/arch/xen/xen/xen_clock.c:1.6
--- src/sys/arch/xen/xen/xen_clock.c:1.5	Thu May  7 19:48:57 2020
+++ src/sys/arch/xen/xen/xen_clock.c	Thu May 21 21:12:31 2020
@@ -1,4 +1,4 @@
-/*	$NetBSD: xen_clock.c,v 1.5 2020/05/07 19:48:57 bouyer Exp $	*/
+/*	$NetBSD: xen_clock.c,v 1.6 2020/05/21 21:12:31 ad Exp $	*/
 
 /*-
  * Copyright (c) 2017, 2018 The NetBSD Foundation, Inc.
@@ -36,7 +36,7 @@
 #endif
 
 #include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: xen_clock.c,v 1.5 2020/05/07 19:48:57 bouyer Exp $");
+__KERNEL_RCSID(0, "$NetBSD: xen_clock.c,v 1.6 2020/05/21 21:12:31 ad Exp $");
 
 #include <sys/param.h>
 #include <sys/types.h>
@@ -729,12 +729,12 @@ again:
 }
 
 /*
- * xen_cpu_initclocks()
+ * xen_initclocks()
  *
  *	Initialize the Xen clocks on the current CPU.
  */
 void
-xen_cpu_initclocks(void)
+xen_initclocks(void)
 {
 	struct cpu_info *ci = curcpu();
 
@@ -766,22 +766,13 @@ xen_cpu_initclocks(void)
 
 	/* Fire up the clocks.  */
 	xen_resumeclocks(ci);
-}
 
-/*
- * xen_initclocks()
- *
- *	Initialize the Xen global clock
- */
-void
-xen_initclocks(void)
-{
 #ifdef DOM0OPS
 	/*
 	 * If this is a privileged dom0, start pushing the wall
 	 * clock time back to the Xen hypervisor.
 	 */
-	if (xendomain_is_privileged())
+	if (ci == &cpu_info_primary && xendomain_is_privileged())
 		xen_timepush_init();
 #endif
 }

Reply via email to