This patch switches arm64 to clockintr(9).
This has survived about a dozen parallel kernel builds, parallel
release builds, and bsd.rd upgrades on my RPi4b.
kettenis@ said he tried it on one of his M1 machines, too.
It could use more testing, particularly on arm64 machines that aren't
a Raspberry Pi.
Notes:
- The spec is a little unclear about whether or not the
values written to CNTV_TVAL_EL0 are interpretted as signed
or unsigned. To be safe, I have capped the cycle count at
INT32_MAX.
Index: sys/arch/arm64/dev/agtimer.c
===================================================================
RCS file: /cvs/src/sys/arch/arm64/dev/agtimer.c,v
retrieving revision 1.19
diff -u -p -r1.19 agtimer.c
--- sys/arch/arm64/dev/agtimer.c 24 Oct 2021 17:52:28 -0000 1.19
+++ sys/arch/arm64/dev/agtimer.c 6 Nov 2022 18:48:22 -0000
@@ -18,10 +18,12 @@
#include <sys/param.h>
#include <sys/systm.h>
+#include <sys/clockintr.h>
#include <sys/queue.h>
#include <sys/malloc.h>
#include <sys/device.h>
#include <sys/kernel.h>
+#include <sys/stdint.h>
#include <sys/timetc.h>
#include <sys/evcount.h>
@@ -53,28 +55,13 @@ static struct timecounter agtimer_timeco
.tc_user = TC_AGTIMER,
};
-struct agtimer_pcpu_softc {
- uint64_t pc_nexttickevent;
- uint64_t pc_nextstatevent;
- u_int32_t pc_ticks_err_sum;
-};
-
struct agtimer_softc {
struct device sc_dev;
int sc_node;
- struct agtimer_pcpu_softc sc_pstat[MAXCPUS];
-
- u_int32_t sc_ticks_err_cnt;
u_int32_t sc_ticks_per_second;
- u_int32_t sc_ticks_per_intr;
- u_int32_t sc_statvar;
- u_int32_t sc_statmin;
-
-#ifdef AMPTIMER_DEBUG
- struct evcount sc_clk_count;
- struct evcount sc_stat_count;
-#endif
+ uint64_t sc_nsec_cycle_ratio;
+ uint64_t sc_nsec_max;
void *sc_ih;
};
@@ -84,9 +71,11 @@ uint64_t agtimer_readcnt64(void);
int agtimer_intr(void *);
void agtimer_cpu_initclocks(void);
void agtimer_delay(u_int);
+void agtimer_rearm(void *, uint64_t);
void agtimer_setstatclockrate(int stathz);
void agtimer_set_clockrate(int32_t new_frequency);
void agtimer_startclock(void);
+void agtimer_trigger(void *);
const struct cfattach agtimer_ca = {
sizeof (struct agtimer_softc), agtimer_match, agtimer_attach
@@ -96,6 +85,11 @@ struct cfdriver agtimer_cd = {
NULL, "agtimer", DV_DULL
};
+struct intrclock agtimer_intrclock = {
+ .ic_rearm = agtimer_rearm,
+ .ic_trigger = agtimer_trigger
+};
+
uint64_t
agtimer_readcnt64(void)
{
@@ -173,13 +167,11 @@ agtimer_attach(struct device *parent, st
agtimer_frequency =
OF_getpropint(sc->sc_node, "clock-frequency", agtimer_frequency);
sc->sc_ticks_per_second = agtimer_frequency;
+ sc->sc_nsec_cycle_ratio =
+ sc->sc_ticks_per_second * (1ULL << 32) / 1000000000;
+ sc->sc_nsec_max = UINT64_MAX / sc->sc_nsec_cycle_ratio;
- printf(": %d kHz\n", sc->sc_ticks_per_second / 1000);
-
-#ifdef AMPTIMER_DEBUG
- evcount_attach(&sc->sc_clk_count, "clock", NULL);
- evcount_attach(&sc->sc_stat_count, "stat", NULL);
-#endif
+ printf(": %u kHz\n", sc->sc_ticks_per_second / 1000);
/*
* private timer and interrupts not enabled until
@@ -191,8 +183,9 @@ agtimer_attach(struct device *parent, st
agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second;
agtimer_timecounter.tc_priv = sc;
-
tc_init(&agtimer_timecounter);
+
+ agtimer_intrclock.ic_cookie = sc;
}
u_int
@@ -209,72 +202,30 @@ agtimer_get_timecount(struct timecounter
return (val & 0xffffffff);
}
-int
-agtimer_intr(void *frame)
+void
+agtimer_rearm(void *cookie, uint64_t nsecs)
{
- struct agtimer_softc *sc = agtimer_cd.cd_devs[0];
- struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
- uint64_t now;
- uint64_t nextevent;
- uint32_t r;
-#if defined(USE_GTIMER_CMP)
- int skip = 1;
-#else
- int64_t delay;
-#endif
- int rc = 0;
-
- /*
- * DSR - I know that the tick timer is 64 bits, but the following
- * code deals with rollover, so there is no point in dealing
- * with the 64 bit math, just let the 32 bit rollover
- * do the right thing
- */
-
- now = agtimer_readcnt64();
+ struct agtimer_softc *sc = cookie;
+ uint32_t cycles;
- while (pc->pc_nexttickevent <= now) {
- pc->pc_nexttickevent += sc->sc_ticks_per_intr;
- pc->pc_ticks_err_sum += sc->sc_ticks_err_cnt;
-
- /* looping a few times is faster than divide */
- while (pc->pc_ticks_err_sum > hz) {
- pc->pc_nexttickevent += 1;
- pc->pc_ticks_err_sum -= hz;
- }
-
-#ifdef AMPTIMER_DEBUG
- sc->sc_clk_count.ec_count++;
-#endif
- rc = 1;
- hardclock(frame);
- }
- while (pc->pc_nextstatevent <= now) {
- do {
- r = random() & (sc->sc_statvar -1);
- } while (r == 0); /* random == 0 not allowed */
- pc->pc_nextstatevent += sc->sc_statmin + r;
-
- /* XXX - correct nextstatevent? */
-#ifdef AMPTIMER_DEBUG
- sc->sc_stat_count.ec_count++;
-#endif
- rc = 1;
- statclock(frame);
- }
-
- if (pc->pc_nexttickevent < pc->pc_nextstatevent)
- nextevent = pc->pc_nexttickevent;
- else
- nextevent = pc->pc_nextstatevent;
-
- delay = nextevent - now;
- if (delay < 0)
- delay = 1;
+ if (nsecs > sc->sc_nsec_max)
+ nsecs = sc->sc_nsec_max;
+ cycles = (nsecs * sc->sc_nsec_cycle_ratio) >> 32;
+ if (cycles > INT32_MAX)
+ cycles = INT32_MAX;
+ agtimer_set_tval(cycles);
+}
- agtimer_set_tval(delay);
+void
+agtimer_trigger(void *unused)
+{
+ agtimer_set_tval(0);
+}
- return (rc);
+int
+agtimer_intr(void *frame)
+{
+ return clockintr_dispatch(frame);
}
void
@@ -288,8 +239,13 @@ agtimer_set_clockrate(int32_t new_freque
return;
sc->sc_ticks_per_second = agtimer_frequency;
+ sc->sc_nsec_cycle_ratio =
+ sc->sc_ticks_per_second * (1ULL << 32) / 1000000000;
+ sc->sc_nsec_max = UINT64_MAX / sc->sc_nsec_cycle_ratio;
+
agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second;
- printf("agtimer0: adjusting clock: new tick rate %d kHz\n",
+
+ printf("agtimer0: adjusting clock: new tick rate %u kHz\n",
sc->sc_ticks_per_second / 1000);
}
@@ -297,37 +253,31 @@ void
agtimer_cpu_initclocks(void)
{
struct agtimer_softc *sc = agtimer_cd.cd_devs[0];
- struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
uint32_t reg;
- uint64_t next;
uint64_t kctl;
stathz = hz;
- profhz = hz * 10;
+ profhz = stathz * 10;
+ clockintr_init(CL_RNDSTAT);
if (sc->sc_ticks_per_second != agtimer_frequency) {
agtimer_set_clockrate(agtimer_frequency);
}
- agtimer_setstatclockrate(stathz);
-
- sc->sc_ticks_per_intr = sc->sc_ticks_per_second / hz;
- sc->sc_ticks_err_cnt = sc->sc_ticks_per_second % hz;
- pc->pc_ticks_err_sum = 0;
-
/* configure virtual timer interrupt */
sc->sc_ih = arm_intr_establish_fdt_idx(sc->sc_node, 2,
IPL_CLOCK|IPL_MPSAFE, agtimer_intr, NULL, "tick");
- next = agtimer_readcnt64() + sc->sc_ticks_per_intr;
- pc->pc_nexttickevent = pc->pc_nextstatevent = next;
+ clockintr_cpu_init(&agtimer_intrclock);
reg = agtimer_get_ctrl();
reg &= ~GTIMER_CNTV_CTL_IMASK;
reg |= GTIMER_CNTV_CTL_ENABLE;
- agtimer_set_tval(sc->sc_ticks_per_second);
+ agtimer_set_tval(INT32_MAX);
agtimer_set_ctrl(reg);
+ clockintr_trigger();
+
/* enable userland access to virtual counter */
kctl = READ_SPECIALREG(CNTKCTL_EL1);
WRITE_SPECIALREG(CNTKCTL_EL1, kctl | CNTKCTL_EL0VCTEN);
@@ -367,48 +317,27 @@ agtimer_delay(u_int usecs)
void
agtimer_setstatclockrate(int newhz)
{
- struct agtimer_softc *sc = agtimer_cd.cd_devs[0];
- int minint, statint;
- int s;
-
- s = splclock();
-
- statint = sc->sc_ticks_per_second / newhz;
- /* calculate largest 2^n which is smaller that just over half statint */
- sc->sc_statvar = 0x40000000; /* really big power of two */
- minint = statint / 2 + 100;
- while (sc->sc_statvar > minint)
- sc->sc_statvar >>= 1;
-
- sc->sc_statmin = statint - (sc->sc_statvar >> 1);
-
- splx(s);
-
- /*
- * XXX this allows the next stat timer to occur then it switches
- * to the new frequency. Rather than switching instantly.
- */
+ clockintr_setstatclockrate(newhz);
}
void
agtimer_startclock(void)
{
struct agtimer_softc *sc = agtimer_cd.cd_devs[0];
- struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
- uint64_t nextevent;
uint64_t kctl;
uint32_t reg;
- nextevent = agtimer_readcnt64() + sc->sc_ticks_per_intr;
- pc->pc_nexttickevent = pc->pc_nextstatevent = nextevent;
-
arm_intr_route(sc->sc_ih, 1, curcpu());
+ clockintr_cpu_init(&agtimer_intrclock);
+
reg = agtimer_get_ctrl();
reg &= ~GTIMER_CNTV_CTL_IMASK;
reg |= GTIMER_CNTV_CTL_ENABLE;
- agtimer_set_tval(sc->sc_ticks_per_second);
+ agtimer_set_tval(INT32_MAX);
agtimer_set_ctrl(reg);
+
+ clockintr_trigger();
/* enable userland access to virtual counter */
kctl = READ_SPECIALREG(CNTKCTL_EL1);
Index: sys/arch/arm64/include/cpu.h
===================================================================
RCS file: /cvs/src/sys/arch/arm64/include/cpu.h,v
retrieving revision 1.28
diff -u -p -r1.28 cpu.h
--- sys/arch/arm64/include/cpu.h 29 Aug 2022 02:01:18 -0000 1.28
+++ sys/arch/arm64/include/cpu.h 6 Nov 2022 18:48:22 -0000
@@ -81,6 +81,7 @@ void arm32_vector_init(vaddr_t, int);
* Per-CPU information. For now we assume one CPU.
*/
+#include <sys/clockintr.h>
#include <sys/device.h>
#include <sys/sched.h>
#include <sys/srp.h>
@@ -142,7 +143,7 @@ struct cpu_info {
#ifdef GPROF
struct gmonparam *ci_gmon;
#endif
-
+ struct clockintr_queue ci_queue;
char ci_panicbuf[512];
};
Index: sys/arch/arm64/include/_types.h
===================================================================
RCS file: /cvs/src/sys/arch/arm64/include/_types.h,v
retrieving revision 1.4
diff -u -p -r1.4 _types.h
--- sys/arch/arm64/include/_types.h 5 Mar 2018 01:15:25 -0000 1.4
+++ sys/arch/arm64/include/_types.h 6 Nov 2022 18:48:22 -0000
@@ -34,6 +34,8 @@
#ifndef _MACHINE__TYPES_H_
#define _MACHINE__TYPES_H_
+#define __HAVE_CLOCKINTR
+
#if defined(_KERNEL)
typedef struct label_t {
long val[13];
