[PATCH 3/5] sched/fair: Make select_idle_cpu() proportional to cores
From: Peter Zijlstra (Intel) Instead of calculating how many (logical) CPUs to scan, compute how many cores to scan. This changes behaviour for anything !SMT2. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Mel Gorman --- kernel/sched/core.c | 18 +- kernel/sched/fair.c | 12 ++-- kernel/sched/sched.h | 2 ++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 15d2562118d1..ada8faac2e4d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7444,11 +7444,19 @@ int sched_cpu_activate(unsigned int cpu) balance_push_set(cpu, false); #ifdef CONFIG_SCHED_SMT - /* -* When going up, increment the number of cores with SMT present. -*/ - if (cpumask_weight(cpu_smt_mask(cpu)) == 2) - static_branch_inc_cpuslocked(_smt_present); + do { + int weight = cpumask_weight(cpu_smt_mask(cpu)); + + if (weight > sched_smt_weight) + sched_smt_weight = weight; + + /* +* When going up, increment the number of cores with SMT present. +*/ + if (weight == 2) + static_branch_inc_cpuslocked(_smt_present); + + } while (0); #endif set_cpu_active(cpu, true); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c8d8e185cf3b..0811e2fe4f19 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6010,6 +6010,8 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p DEFINE_STATIC_KEY_FALSE(sched_smt_present); EXPORT_SYMBOL_GPL(sched_smt_present); +int sched_smt_weight __read_mostly = 1; + static inline void set_idle_cores(int cpu, int val) { struct sched_domain_shared *sds; @@ -6124,6 +6126,8 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t #else /* CONFIG_SCHED_SMT */ +#define sched_smt_weight 1 + static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) { return -1; @@ -6136,6 +6140,8 @@ static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd #endif /* CONFIG_SCHED_SMT */ +#define sis_min_cores 2 + /* * Scan the LLC domain for idle CPUs; this is dynamically regulated by * comparing the average scan cost (tracked in sd->avg_scan_cost) against the @@ -6166,10 +6172,12 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t avg_cost = this_sd->avg_scan_cost + 1; span_avg = sd->span_weight * avg_idle; - if (span_avg > 4*avg_cost) + if (span_avg > sis_min_cores*avg_cost) nr = div_u64(span_avg, avg_cost); else - nr = 4; + nr = sis_min_cores; + + nr *= sched_smt_weight; time = cpu_clock(this); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 12ada79d40f3..29aabe98dd1d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1107,6 +1107,8 @@ static inline void update_idle_core(struct rq *rq) __update_idle_core(rq); } +extern int sched_smt_weight; + #else static inline void update_idle_core(struct rq *rq) { } #endif -- 2.26.2
Re: [PATCH 3/5] sched/fair: Make select_idle_cpu() proportional to cores
On Mon, Jan 18, 2021 at 04:14:36PM +0800, Li, Aubrey wrote: > > > > @@ -6124,6 +6126,8 @@ static int select_idle_smt(struct task_struct *p, > > struct sched_domain *sd, int t > > > > #else /* CONFIG_SCHED_SMT */ > > > > +#define sched_smt_weight 1 > > + > > static inline int select_idle_core(struct task_struct *p, struct > > sched_domain *sd, int target) > > { > > return -1; > > > > > > > > @@ -6166,10 +6172,12 @@ static int select_idle_cpu(struct task_struct *p, > > struct sched_domain *sd, int t > > avg_cost = this_sd->avg_scan_cost + 1; > > > > span_avg = sd->span_weight * avg_idle; > > - if (span_avg > 4*avg_cost) > > + if (span_avg > sis_min_cores*avg_cost) > > nr = div_u64(span_avg, avg_cost); > > else > > - nr = 4; > > + nr = sis_min_cores; > > + > > + nr *= sched_smt_weight; > > Is it better to put this into an inline wrapper to hide sched_smt_weight if > !CONFIG_SCHED_SMT? > There already is a #define sched_smt_weight for !CONFIG_SCHED_SMT and I do not think an inline wrapper would make it more readable or maintainable. -- Mel Gorman SUSE Labs
Re: [PATCH 3/5] sched/fair: Make select_idle_cpu() proportional to cores
On 2021/1/15 18:08, Mel Gorman wrote: > From: Peter Zijlstra (Intel) > > Instead of calculating how many (logical) CPUs to scan, compute how > many cores to scan. > > This changes behaviour for anything !SMT2. > > Signed-off-by: Peter Zijlstra (Intel) > Signed-off-by: Mel Gorman > --- > kernel/sched/core.c | 18 +- > kernel/sched/fair.c | 12 ++-- > kernel/sched/sched.h | 2 ++ > 3 files changed, 25 insertions(+), 7 deletions(-) > > diff --git a/kernel/sched/core.c b/kernel/sched/core.c > index 15d2562118d1..ada8faac2e4d 100644 > --- a/kernel/sched/core.c > +++ b/kernel/sched/core.c > @@ -7444,11 +7444,19 @@ int sched_cpu_activate(unsigned int cpu) > balance_push_set(cpu, false); > > #ifdef CONFIG_SCHED_SMT > - /* > - * When going up, increment the number of cores with SMT present. > - */ > - if (cpumask_weight(cpu_smt_mask(cpu)) == 2) > - static_branch_inc_cpuslocked(_smt_present); > + do { > + int weight = cpumask_weight(cpu_smt_mask(cpu)); > + > + if (weight > sched_smt_weight) > + sched_smt_weight = weight; > + > + /* > + * When going up, increment the number of cores with SMT > present. > + */ > + if (weight == 2) > + static_branch_inc_cpuslocked(_smt_present); > + > + } while (0); > #endif > set_cpu_active(cpu, true); > > diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c > index c8d8e185cf3b..0811e2fe4f19 100644 > --- a/kernel/sched/fair.c > +++ b/kernel/sched/fair.c > @@ -6010,6 +6010,8 @@ static inline int find_idlest_cpu(struct sched_domain > *sd, struct task_struct *p > DEFINE_STATIC_KEY_FALSE(sched_smt_present); > EXPORT_SYMBOL_GPL(sched_smt_present); > > +int sched_smt_weight __read_mostly = 1; > + > static inline void set_idle_cores(int cpu, int val) > { > struct sched_domain_shared *sds; > @@ -6124,6 +6126,8 @@ static int select_idle_smt(struct task_struct *p, > struct sched_domain *sd, int t > > #else /* CONFIG_SCHED_SMT */ > > +#define sched_smt_weight 1 > + > static inline int select_idle_core(struct task_struct *p, struct > sched_domain *sd, int target) > { > return -1; > @@ -6136,6 +6140,8 @@ static inline int select_idle_smt(struct task_struct > *p, struct sched_domain *sd > > #endif /* CONFIG_SCHED_SMT */ > > +#define sis_min_cores2 > + > /* > * Scan the LLC domain for idle CPUs; this is dynamically regulated by > * comparing the average scan cost (tracked in sd->avg_scan_cost) against the > @@ -6166,10 +6172,12 @@ static int select_idle_cpu(struct task_struct *p, > struct sched_domain *sd, int t > avg_cost = this_sd->avg_scan_cost + 1; > > span_avg = sd->span_weight * avg_idle; > - if (span_avg > 4*avg_cost) > + if (span_avg > sis_min_cores*avg_cost) > nr = div_u64(span_avg, avg_cost); > else > - nr = 4; > + nr = sis_min_cores; > + > + nr *= sched_smt_weight; Is it better to put this into an inline wrapper to hide sched_smt_weight if !CONFIG_SCHED_SMT? Thanks, -Aubrey
[PATCH 3/5] sched/fair: Make select_idle_cpu() proportional to cores
From: Peter Zijlstra (Intel) Instead of calculating how many (logical) CPUs to scan, compute how many cores to scan. This changes behaviour for anything !SMT2. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Mel Gorman --- kernel/sched/core.c | 18 +- kernel/sched/fair.c | 12 ++-- kernel/sched/sched.h | 2 ++ 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 15d2562118d1..ada8faac2e4d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7444,11 +7444,19 @@ int sched_cpu_activate(unsigned int cpu) balance_push_set(cpu, false); #ifdef CONFIG_SCHED_SMT - /* -* When going up, increment the number of cores with SMT present. -*/ - if (cpumask_weight(cpu_smt_mask(cpu)) == 2) - static_branch_inc_cpuslocked(_smt_present); + do { + int weight = cpumask_weight(cpu_smt_mask(cpu)); + + if (weight > sched_smt_weight) + sched_smt_weight = weight; + + /* +* When going up, increment the number of cores with SMT present. +*/ + if (weight == 2) + static_branch_inc_cpuslocked(_smt_present); + + } while (0); #endif set_cpu_active(cpu, true); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c8d8e185cf3b..0811e2fe4f19 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6010,6 +6010,8 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p DEFINE_STATIC_KEY_FALSE(sched_smt_present); EXPORT_SYMBOL_GPL(sched_smt_present); +int sched_smt_weight __read_mostly = 1; + static inline void set_idle_cores(int cpu, int val) { struct sched_domain_shared *sds; @@ -6124,6 +6126,8 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t #else /* CONFIG_SCHED_SMT */ +#define sched_smt_weight 1 + static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) { return -1; @@ -6136,6 +6140,8 @@ static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd #endif /* CONFIG_SCHED_SMT */ +#define sis_min_cores 2 + /* * Scan the LLC domain for idle CPUs; this is dynamically regulated by * comparing the average scan cost (tracked in sd->avg_scan_cost) against the @@ -6166,10 +6172,12 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t avg_cost = this_sd->avg_scan_cost + 1; span_avg = sd->span_weight * avg_idle; - if (span_avg > 4*avg_cost) + if (span_avg > sis_min_cores*avg_cost) nr = div_u64(span_avg, avg_cost); else - nr = 4; + nr = sis_min_cores; + + nr *= sched_smt_weight; time = cpu_clock(this); } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 12ada79d40f3..29aabe98dd1d 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1107,6 +1107,8 @@ static inline void update_idle_core(struct rq *rq) __update_idle_core(rq); } +extern int sched_smt_weight; + #else static inline void update_idle_core(struct rq *rq) { } #endif -- 2.26.2
Re: [PATCH 3/5] sched/fair: Make select_idle_cpu() proportional to cores
On Wed, Jan 13, 2021 at 05:49:58PM +0100, Vincent Guittot wrote: > > @@ -7444,11 +7444,20 @@ int sched_cpu_activate(unsigned int cpu) > > balance_push_set(cpu, false); > > > > #ifdef CONFIG_SCHED_SMT > > - /* > > -* When going up, increment the number of cores with SMT present. > > -*/ > > - if (cpumask_weight(cpu_smt_mask(cpu)) == 2) > > - static_branch_inc_cpuslocked(_smt_present); > > + do { > > + int weight = cpumask_weight(cpu_smt_mask(cpu)); > > + extern int sched_smt_weight; > > coding style problem > Presumably you are referring to an extern defined in a C file. That can move to kernel/sched/sched.h in this patch. > > > > /* > > * Scan the LLC domain for idle CPUs; this is dynamically regulated by > > * comparing the average scan cost (tracked in sd->avg_scan_cost) against > > the > > @@ -6166,10 +6172,12 @@ static int select_idle_cpu(struct task_struct *p, > > struct sched_domain *sd, int t > > avg_cost = this_sd->avg_scan_cost + 1; > > > > span_avg = sd->span_weight * avg_idle; > > - if (span_avg > 4*avg_cost) > > + if (span_avg > sis_min_cores*avg_cost) > > nr = div_u64(span_avg, avg_cost); > > else > > - nr = 4; > > + nr = sis_min_cores; > > + > > + nr *= sched_smt_weight; > > Also, patch 5 will look at all CPUs of a core in select_idle_core so > nr will decrement by 1 per core so i don't see the need to multiply by > sched_smt_weight one patch 5 is applied > It makes sense in the context of this patch but can be removed again in the last patch and then I think sched_smt_weight only exists in core.c -- Mel Gorman SUSE Labs
Re: [PATCH 3/5] sched/fair: Make select_idle_cpu() proportional to cores
On Mon, 11 Jan 2021 at 16:50, Mel Gorman wrote: > > From: Peter Zijlstra (Intel) > > Instead of calculating how many (logical) CPUs to scan, compute how > many cores to scan. > > This changes behaviour for anything !SMT2. > > Signed-off-by: Peter Zijlstra (Intel) > Signed-off-by: Mel Gorman > --- > kernel/sched/core.c | 19 ++- > kernel/sched/fair.c | 12 ++-- > 2 files changed, 24 insertions(+), 7 deletions(-) > > diff --git a/kernel/sched/core.c b/kernel/sched/core.c > index 15d2562118d1..7bfa73de6a8d 100644 > --- a/kernel/sched/core.c > +++ b/kernel/sched/core.c > @@ -7444,11 +7444,20 @@ int sched_cpu_activate(unsigned int cpu) > balance_push_set(cpu, false); > > #ifdef CONFIG_SCHED_SMT > - /* > -* When going up, increment the number of cores with SMT present. > -*/ > - if (cpumask_weight(cpu_smt_mask(cpu)) == 2) > - static_branch_inc_cpuslocked(_smt_present); > + do { > + int weight = cpumask_weight(cpu_smt_mask(cpu)); > + extern int sched_smt_weight; coding style problem > + > + if (weight > sched_smt_weight) > + sched_smt_weight = weight; > + > + /* > +* When going up, increment the number of cores with SMT > present. > +*/ > + if (weight == 2) > + static_branch_inc_cpuslocked(_smt_present); > + > + } while (0); > #endif > set_cpu_active(cpu, true); > > diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c > index c8d8e185cf3b..0811e2fe4f19 100644 > --- a/kernel/sched/fair.c > +++ b/kernel/sched/fair.c > @@ -6010,6 +6010,8 @@ static inline int find_idlest_cpu(struct sched_domain > *sd, struct task_struct *p > DEFINE_STATIC_KEY_FALSE(sched_smt_present); > EXPORT_SYMBOL_GPL(sched_smt_present); > > +int sched_smt_weight __read_mostly = 1; > + > static inline void set_idle_cores(int cpu, int val) > { > struct sched_domain_shared *sds; > @@ -6124,6 +6126,8 @@ static int select_idle_smt(struct task_struct *p, > struct sched_domain *sd, int t > > #else /* CONFIG_SCHED_SMT */ > > +#define sched_smt_weight 1 > + > static inline int select_idle_core(struct task_struct *p, struct > sched_domain *sd, int target) > { > return -1; > @@ -6136,6 +6140,8 @@ static inline int select_idle_smt(struct task_struct > *p, struct sched_domain *sd > > #endif /* CONFIG_SCHED_SMT */ > > +#define sis_min_cores 2 > + > /* > * Scan the LLC domain for idle CPUs; this is dynamically regulated by > * comparing the average scan cost (tracked in sd->avg_scan_cost) against the > @@ -6166,10 +6172,12 @@ static int select_idle_cpu(struct task_struct *p, > struct sched_domain *sd, int t > avg_cost = this_sd->avg_scan_cost + 1; > > span_avg = sd->span_weight * avg_idle; > - if (span_avg > 4*avg_cost) > + if (span_avg > sis_min_cores*avg_cost) > nr = div_u64(span_avg, avg_cost); > else > - nr = 4; > + nr = sis_min_cores; > + > + nr *= sched_smt_weight; Also, patch 5 will look at all CPUs of a core in select_idle_core so nr will decrement by 1 per core so i don't see the need to multiply by sched_smt_weight one patch 5 is applied > > time = cpu_clock(this); > } > -- > 2.26.2 >
[PATCH 3/5] sched/fair: Make select_idle_cpu() proportional to cores
From: Peter Zijlstra (Intel) Instead of calculating how many (logical) CPUs to scan, compute how many cores to scan. This changes behaviour for anything !SMT2. Signed-off-by: Peter Zijlstra (Intel) Signed-off-by: Mel Gorman --- kernel/sched/core.c | 19 ++- kernel/sched/fair.c | 12 ++-- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/kernel/sched/core.c b/kernel/sched/core.c index 15d2562118d1..7bfa73de6a8d 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -7444,11 +7444,20 @@ int sched_cpu_activate(unsigned int cpu) balance_push_set(cpu, false); #ifdef CONFIG_SCHED_SMT - /* -* When going up, increment the number of cores with SMT present. -*/ - if (cpumask_weight(cpu_smt_mask(cpu)) == 2) - static_branch_inc_cpuslocked(_smt_present); + do { + int weight = cpumask_weight(cpu_smt_mask(cpu)); + extern int sched_smt_weight; + + if (weight > sched_smt_weight) + sched_smt_weight = weight; + + /* +* When going up, increment the number of cores with SMT present. +*/ + if (weight == 2) + static_branch_inc_cpuslocked(_smt_present); + + } while (0); #endif set_cpu_active(cpu, true); diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index c8d8e185cf3b..0811e2fe4f19 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -6010,6 +6010,8 @@ static inline int find_idlest_cpu(struct sched_domain *sd, struct task_struct *p DEFINE_STATIC_KEY_FALSE(sched_smt_present); EXPORT_SYMBOL_GPL(sched_smt_present); +int sched_smt_weight __read_mostly = 1; + static inline void set_idle_cores(int cpu, int val) { struct sched_domain_shared *sds; @@ -6124,6 +6126,8 @@ static int select_idle_smt(struct task_struct *p, struct sched_domain *sd, int t #else /* CONFIG_SCHED_SMT */ +#define sched_smt_weight 1 + static inline int select_idle_core(struct task_struct *p, struct sched_domain *sd, int target) { return -1; @@ -6136,6 +6140,8 @@ static inline int select_idle_smt(struct task_struct *p, struct sched_domain *sd #endif /* CONFIG_SCHED_SMT */ +#define sis_min_cores 2 + /* * Scan the LLC domain for idle CPUs; this is dynamically regulated by * comparing the average scan cost (tracked in sd->avg_scan_cost) against the @@ -6166,10 +6172,12 @@ static int select_idle_cpu(struct task_struct *p, struct sched_domain *sd, int t avg_cost = this_sd->avg_scan_cost + 1; span_avg = sd->span_weight * avg_idle; - if (span_avg > 4*avg_cost) + if (span_avg > sis_min_cores*avg_cost) nr = div_u64(span_avg, avg_cost); else - nr = 4; + nr = sis_min_cores; + + nr *= sched_smt_weight; time = cpu_clock(this); } -- 2.26.2