Author: jeff Date: Tue Aug 13 04:54:02 2019 New Revision: 350972 URL: https://svnweb.freebsd.org/changeset/base/350972
Log: Move scheduler state into the per-cpu area where it can be allocated on the correct NUMA domain. Reviewed by: markj, gallatin Sponsored by: Netflix Differential Revision: https://reviews.freebsd.org/D19315 Modified: head/sys/kern/sched_ule.c head/sys/sys/pcpu.h Modified: head/sys/kern/sched_ule.c ============================================================================== --- head/sys/kern/sched_ule.c Tue Aug 13 03:49:53 2019 (r350971) +++ head/sys/kern/sched_ule.c Tue Aug 13 04:54:02 2019 (r350972) @@ -247,6 +247,7 @@ struct tdq { u_char tdq_ipipending; /* IPI pending. */ u_char tdq_idx; /* Current insert index. */ u_char tdq_ridx; /* Current removal index. */ + int tdq_id; /* cpuid. */ struct runq tdq_realtime; /* real-time run queue. */ struct runq tdq_timeshare; /* timeshare run queue. */ struct runq tdq_idle; /* Queue of IDLE threads. */ @@ -280,14 +281,14 @@ static int trysteal_limit = 2; /* * One thread queue per processor. */ -static struct tdq tdq_cpu[MAXCPU]; static struct tdq *balance_tdq; static int balance_ticks; +DPCPU_DEFINE_STATIC(struct tdq, tdq); DPCPU_DEFINE_STATIC(uint32_t, randomval); -#define TDQ_SELF() (&tdq_cpu[PCPU_GET(cpuid)]) -#define TDQ_CPU(x) (&tdq_cpu[(x)]) -#define TDQ_ID(x) ((int)((x) - tdq_cpu)) +#define TDQ_SELF() ((struct tdq *)PCPU_GET(sched)) +#define TDQ_CPU(x) (DPCPU_ID_PTR((x), tdq)) +#define TDQ_ID(x) ((x)->tdq_id) #else /* !SMP */ static struct tdq tdq_cpu; @@ -311,7 +312,7 @@ static void sched_pctcpu_update(struct td_sched *, int /* Operations on per processor queues */ static struct thread *tdq_choose(struct tdq *); -static void tdq_setup(struct tdq *); +static void tdq_setup(struct tdq *, int i); static void tdq_load_add(struct tdq *, struct thread *); static void tdq_load_rem(struct tdq *, struct thread *); static __inline void tdq_runq_add(struct tdq *, struct thread *, int); @@ -838,6 +839,7 @@ sched_highest(const struct cpu_group *cg, cpuset_t mas static void sched_balance_group(struct cpu_group *cg) { + struct tdq *tdq; cpuset_t hmask, lmask; int high, low, anylow; @@ -853,9 +855,9 @@ sched_balance_group(struct cpu_group *cg) if (CPU_EMPTY(&lmask)) break; anylow = 1; + tdq = TDQ_CPU(high); nextlow: - low = sched_lowest(cg, lmask, -1, - TDQ_CPU(high)->tdq_load - 1, high); + low = sched_lowest(cg, lmask, -1, tdq->tdq_load - 1, high); /* Stop if we looked well and found no less loaded CPU. */ if (anylow && low == -1) break; @@ -863,7 +865,7 @@ nextlow: if (low == -1) continue; /* Transfer thread from high to low. */ - if (sched_balance_pair(TDQ_CPU(high), TDQ_CPU(low))) { + if (sched_balance_pair(tdq, TDQ_CPU(low))) { /* CPU that got thread can no longer be a donor. */ CPU_CLR(low, &hmask); } else { @@ -1271,7 +1273,7 @@ sched_pickcpu(struct thread *td, int flags) curthread->td_intr_nesting_level && ts->ts_cpu != self) { SCHED_STAT_INC(pickcpu_intrbind); ts->ts_cpu = self; - if (TDQ_CPU(self)->tdq_lowpri > pri) { + if (TDQ_SELF()->tdq_lowpri > pri) { SCHED_STAT_INC(pickcpu_affinity); return (ts->ts_cpu); } @@ -1329,9 +1331,10 @@ sched_pickcpu(struct thread *td, int flags) /* * Compare the lowest loaded cpu to current cpu. */ - if (THREAD_CAN_SCHED(td, self) && TDQ_CPU(self)->tdq_lowpri > pri && - TDQ_CPU(cpu)->tdq_lowpri < PRI_MIN_IDLE && - TDQ_CPU(self)->tdq_load <= TDQ_CPU(cpu)->tdq_load + 1) { + tdq = TDQ_CPU(cpu); + if (THREAD_CAN_SCHED(td, self) && TDQ_SELF()->tdq_lowpri > pri && + tdq->tdq_lowpri < PRI_MIN_IDLE && + TDQ_SELF()->tdq_load <= tdq->tdq_load + 1) { SCHED_STAT_INC(pickcpu_local); cpu = self; } else @@ -1376,14 +1379,15 @@ tdq_choose(struct tdq *tdq) * Initialize a thread queue. */ static void -tdq_setup(struct tdq *tdq) +tdq_setup(struct tdq *tdq, int id) { if (bootverbose) - printf("ULE: setup cpu %d\n", TDQ_ID(tdq)); + printf("ULE: setup cpu %d\n", id); runq_init(&tdq->tdq_realtime); runq_init(&tdq->tdq_timeshare); runq_init(&tdq->tdq_idle); + tdq->tdq_id = id; snprintf(tdq->tdq_name, sizeof(tdq->tdq_name), "sched lock %d", (int)TDQ_ID(tdq)); mtx_init(&tdq->tdq_lock, tdq->tdq_name, "sched lock", @@ -1403,12 +1407,13 @@ sched_setup_smp(void) cpu_top = smp_topo(); CPU_FOREACH(i) { - tdq = TDQ_CPU(i); - tdq_setup(tdq); + tdq = DPCPU_ID_PTR(i, tdq); + tdq_setup(tdq, i); tdq->tdq_cg = smp_topo_find(cpu_top, i); if (tdq->tdq_cg == NULL) panic("Can't find cpu group for %d\n", i); } + PCPU_SET(sched, DPCPU_PTR(tdq)); balance_tdq = TDQ_SELF(); } #endif @@ -1422,12 +1427,12 @@ sched_setup(void *dummy) { struct tdq *tdq; - tdq = TDQ_SELF(); #ifdef SMP sched_setup_smp(); #else - tdq_setup(tdq); + tdq_setup(TDQ_SELF(), 0); #endif + tdq = TDQ_SELF(); /* Add thread0's load since it's running. */ TDQ_LOCK(tdq); @@ -2036,7 +2041,7 @@ sched_switch(struct thread *td, struct thread *newtd, KASSERT(newtd == NULL, ("sched_switch: Unsupported newtd argument")); cpuid = PCPU_GET(cpuid); - tdq = TDQ_CPU(cpuid); + tdq = TDQ_SELF(); ts = td_get_sched(td); mtx = td->td_lock; sched_pctcpu_update(ts, 1); @@ -2131,7 +2136,7 @@ sched_switch(struct thread *td, struct thread *newtd, * run queue lock. */ cpuid = PCPU_GET(cpuid); - tdq = TDQ_CPU(cpuid); + tdq = TDQ_SELF(); lock_profile_obtain_lock_success( &TDQ_LOCKPTR(tdq)->lock_object, 0, 0, __FILE__, __LINE__); @@ -2871,14 +2876,18 @@ sched_throw(struct thread *td) struct thread *newtd; struct tdq *tdq; - tdq = TDQ_SELF(); if (td == NULL) { +#ifdef SMP + PCPU_SET(sched, DPCPU_PTR(tdq)); +#endif /* Correct spinlock nesting and acquire the correct lock. */ + tdq = TDQ_SELF(); TDQ_LOCK(tdq); spinlock_exit(); PCPU_SET(switchtime, cpu_ticks()); PCPU_SET(switchticks, ticks); } else { + tdq = TDQ_SELF(); MPASS(td->td_lock == TDQ_LOCKPTR(tdq)); tdq_load_rem(tdq, td); lock_profile_release_lock(&TDQ_LOCKPTR(tdq)->lock_object); @@ -2906,7 +2915,7 @@ sched_fork_exit(struct thread *td) * non-nested critical section with the scheduler lock held. */ cpuid = PCPU_GET(cpuid); - tdq = TDQ_CPU(cpuid); + tdq = TDQ_SELF(); if (TD_IS_IDLETHREAD(td)) td->td_lock = TDQ_LOCKPTR(tdq); MPASS(td->td_lock == TDQ_LOCKPTR(tdq)); Modified: head/sys/sys/pcpu.h ============================================================================== --- head/sys/sys/pcpu.h Tue Aug 13 03:49:53 2019 (r350971) +++ head/sys/sys/pcpu.h Tue Aug 13 04:54:02 2019 (r350972) @@ -180,6 +180,7 @@ struct pcpu { struct thread *pc_fpcurthread; /* Fp state owner */ struct thread *pc_deadthread; /* Zombie thread or NULL */ struct pcb *pc_curpcb; /* Current pcb */ + void *pc_sched; /* Scheduler state */ uint64_t pc_switchtime; /* cpu_ticks() at last csw */ int pc_switchticks; /* `ticks' at last csw */ u_int pc_cpuid; /* This cpu number */ _______________________________________________ svn-src-all@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-all To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
