Expose some scheduler statistics to userland via sysctl
Moving this to tech@ Hi, I was studying the scheduler code after watching tedu's talk, I'd like to expose this statistics to userland so that I can try playing with cache affinity in the future: gimli:src: sysctl kern.schedstat kern.schedstat.nmigrations=23744 kern.schedstat.noidle=0 kern.schedstat.stolen=9170 kern.schedstat.choose=834843 kern.schedstat.wasidle=808711 kern.schedstat.nomigrations=2388 Opinions ? Index: sys/sys//sched.h === RCS file: /cvs/src/sys/sys/sched.h,v retrieving revision 1.29 diff -d -u -p -w -r1.29 sched.h --- sys/sys//sched.h7 Jul 2011 18:00:33 - 1.29 +++ sys/sys//sched.h12 Nov 2011 13:51:04 - @@ -75,6 +75,34 @@ * Posix defines a sched.h which may want to include sys/sched.h */ +struct schedstat { + u_int64_t scs_nmigrations; + u_int64_t scs_noidle; + u_int64_t scs_stolen; + + u_int64_t scs_choose; + u_int64_t scs_wasidle; + u_int64_t scs_nomigrations; +}; + +/* These sysctl names are only really used by sysctl(8) */ +#define KERN_SCHEDSTAT_NMIGRATIONS 1 +#define KERN_SCHEDSTAT_NOIDLE 2 +#define KERN_SCHEDSTAT_STOLEN 3 +#define KERN_SCHEDSTAT_CHOOSE 4 +#define KERN_SCHEDSTAT_WASIDLE 5 +#define KERN_SCHEDSTAT_NOMIGRATIONS6 +#define KERN_SCHEDSTAT_MAXID 7 + +#define CTL_KERN_SCHEDSTAT_NAMES { \ + { 0, 0 }, \ + { nmigrations, CTLTYPE_QUAD },\ + { noidle, CTLTYPE_QUAD }, \ + { stolen, CTLTYPE_QUAD }, \ + { choose, CTLTYPE_QUAD }, \ + { wasidle, CTLTYPE_QUAD },\ + { nomigrations, CTLTYPE_QUAD }\ +} /* * CPU states. * XXX Not really scheduler state, but no other good place to put Index: sys/sys//sysctl.h === RCS file: /cvs/src/sys/sys/sysctl.h,v retrieving revision 1.117 diff -d -u -p -w -r1.117 sysctl.h --- sys/sys//sysctl.h 30 Aug 2011 01:09:29 - 1.117 +++ sys/sys//sysctl.h 12 Nov 2011 13:40:45 - @@ -189,7 +189,8 @@ struct ctlname { #defineKERN_CONSDEV75 /* dev_t: console terminal device */ #defineKERN_NETLIVELOCKS 76 /* int: number of network livelocks */ #defineKERN_POOL_DEBUG 77 /* int: enable pool_debug */ -#defineKERN_MAXID 78 /* number of valid kern ids */ +#define KERN_SCHEDSTAT 78 /* struct: sched statistics */ +#defineKERN_MAXID 79 /* number of valid kern ids */ #defineCTL_KERN_NAMES { \ { 0, 0 }, \ @@ -270,6 +271,7 @@ struct ctlname { { consdev, CTLTYPE_STRUCT }, \ { netlivelocks, CTLTYPE_INT }, \ { pool_debug, CTLTYPE_INT }, \ + { schedstat, CTLTYPE_STRUCT }, \ } /* Index: sys/kern//kern_sched.c === RCS file: /cvs/src/sys/kern/kern_sched.c,v retrieving revision 1.24 diff -d -u -p -w -r1.24 kern_sched.c --- sys/kern//kern_sched.c 12 Oct 2011 18:30:09 - 1.24 +++ sys/kern//kern_sched.c 12 Nov 2011 14:41:59 - @@ -35,6 +35,8 @@ void sched_kthreads_create(void *); int sched_proc_to_cpu_cost(struct cpu_info *ci, struct proc *p); struct proc *sched_steal_proc(struct cpu_info *); +struct schedstat schedstat; + /* * To help choosing which cpu should run which process we keep track * of cpus which are currently idle and which cpus have processes @@ -301,14 +303,6 @@ again: return (p); } -uint64_t sched_nmigrations; -uint64_t sched_noidle; -uint64_t sched_stolen; - -uint64_t sched_choose; -uint64_t sched_wasidle; -uint64_t sched_nomigrations; - struct cpu_info * sched_choosecpu_fork(struct proc *parent, int flags) { @@ -374,7 +368,7 @@ sched_choosecpu(struct proc *p) if (p-p_flag P_CPUPEG) return (p-p_cpu); - sched_choose++; + schedstat.scs_choose++; /* * Look at all cpus that are currently idle and have nothing queued. @@ -393,7 +387,7 @@ sched_choosecpu(struct proc *p) if (cpuset_isset(set, p-p_cpu) || (p-p_cpu == curcpu() p-p_cpu-ci_schedstate.spc_nrun == 0 curproc == p)) { - sched_wasidle++; + schedstat.scs_wasidle++; return (p-p_cpu); } @@ -411,9 +405,9 @@ sched_choosecpu(struct proc *p) } if (p-p_cpu != choice) - sched_nmigrations++; + schedstat.scs_nmigrations++; else - sched_nomigrations++; + schedstat.scs_nomigrations++; return (choice); } @@ -461,7 +455,7 @@ sched_steal_proc(struct cpu_info *self)
Re: Expose some scheduler statistics to userland via sysctl
On 2011 Nov 17 (Thu) at 21:18:24 -0200 (-0200), Christiano F. Haesbaert wrote: Moving this to tech@ Hi, I was studying the scheduler code after watching tedu's talk, I'd like to expose this statistics to userland so that I can try playing with cache affinity in the future: gimli:src: sysctl kern.schedstat kern.schedstat.nmigrations=23744 kern.schedstat.noidle=0 kern.schedstat.stolen=9170 kern.schedstat.choose=834843 kern.schedstat.wasidle=808711 kern.schedstat.nomigrations=2388 Opinions ? I see no point in exporting this out. They are essentially pointless knobs that people _will_ fiddle with without a clue. This belongs in a developers tree IMO. Index: sys/sys//sched.h === RCS file: /cvs/src/sys/sys/sched.h,v retrieving revision 1.29 diff -d -u -p -w -r1.29 sched.h --- sys/sys//sched.h 7 Jul 2011 18:00:33 - 1.29 +++ sys/sys//sched.h 12 Nov 2011 13:51:04 - @@ -75,6 +75,34 @@ * Posix defines a sched.h which may want to include sys/sched.h */ +struct schedstat { + u_int64_t scs_nmigrations; + u_int64_t scs_noidle; + u_int64_t scs_stolen; + + u_int64_t scs_choose; + u_int64_t scs_wasidle; + u_int64_t scs_nomigrations; +}; + +/* These sysctl names are only really used by sysctl(8) */ +#define KERN_SCHEDSTAT_NMIGRATIONS 1 +#define KERN_SCHEDSTAT_NOIDLE2 +#define KERN_SCHEDSTAT_STOLEN3 +#define KERN_SCHEDSTAT_CHOOSE4 +#define KERN_SCHEDSTAT_WASIDLE 5 +#define KERN_SCHEDSTAT_NOMIGRATIONS 6 +#define KERN_SCHEDSTAT_MAXID 7 + +#define CTL_KERN_SCHEDSTAT_NAMES { \ + { 0, 0 }, \ + { nmigrations, CTLTYPE_QUAD },\ + { noidle, CTLTYPE_QUAD }, \ + { stolen, CTLTYPE_QUAD }, \ + { choose, CTLTYPE_QUAD }, \ + { wasidle, CTLTYPE_QUAD },\ + { nomigrations, CTLTYPE_QUAD }\ +} /* * CPU states. * XXX Not really scheduler state, but no other good place to put Index: sys/sys//sysctl.h === RCS file: /cvs/src/sys/sys/sysctl.h,v retrieving revision 1.117 diff -d -u -p -w -r1.117 sysctl.h --- sys/sys//sysctl.h 30 Aug 2011 01:09:29 - 1.117 +++ sys/sys//sysctl.h 12 Nov 2011 13:40:45 - @@ -189,7 +189,8 @@ struct ctlname { #define KERN_CONSDEV75 /* dev_t: console terminal device */ #define KERN_NETLIVELOCKS 76 /* int: number of network livelocks */ #define KERN_POOL_DEBUG 77 /* int: enable pool_debug */ -#define KERN_MAXID 78 /* number of valid kern ids */ +#define KERN_SCHEDSTAT 78 /* struct: sched statistics */ +#define KERN_MAXID 79 /* number of valid kern ids */ #define CTL_KERN_NAMES { \ { 0, 0 }, \ @@ -270,6 +271,7 @@ struct ctlname { { consdev, CTLTYPE_STRUCT }, \ { netlivelocks, CTLTYPE_INT }, \ { pool_debug, CTLTYPE_INT }, \ + { schedstat, CTLTYPE_STRUCT }, \ } /* Index: sys/kern//kern_sched.c === RCS file: /cvs/src/sys/kern/kern_sched.c,v retrieving revision 1.24 diff -d -u -p -w -r1.24 kern_sched.c --- sys/kern//kern_sched.c12 Oct 2011 18:30:09 - 1.24 +++ sys/kern//kern_sched.c12 Nov 2011 14:41:59 - @@ -35,6 +35,8 @@ void sched_kthreads_create(void *); int sched_proc_to_cpu_cost(struct cpu_info *ci, struct proc *p); struct proc *sched_steal_proc(struct cpu_info *); +struct schedstat schedstat; + /* * To help choosing which cpu should run which process we keep track * of cpus which are currently idle and which cpus have processes @@ -301,14 +303,6 @@ again: return (p); } -uint64_t sched_nmigrations; -uint64_t sched_noidle; -uint64_t sched_stolen; - -uint64_t sched_choose; -uint64_t sched_wasidle; -uint64_t sched_nomigrations; - struct cpu_info * sched_choosecpu_fork(struct proc *parent, int flags) { @@ -374,7 +368,7 @@ sched_choosecpu(struct proc *p) if (p-p_flag P_CPUPEG) return (p-p_cpu); - sched_choose++; + schedstat.scs_choose++; /* * Look at all cpus that are currently idle and have nothing queued. @@ -393,7 +387,7 @@ sched_choosecpu(struct proc *p) if (cpuset_isset(set, p-p_cpu) || (p-p_cpu == curcpu() p-p_cpu-ci_schedstate.spc_nrun == 0 curproc == p)) { - sched_wasidle++; + schedstat.scs_wasidle++; return (p-p_cpu); } @@ -411,9 +405,9 @@ sched_choosecpu(struct proc
Re: Expose some scheduler statistics to userland via sysctl
On Fri, Nov 18, 2011, Thordur Bjornsson wrote: On 2011 Nov 17 (Thu) at 21:18:24 -0200 (-0200), Christiano F. Haesbaert wrote: Moving this to tech@ Hi, I was studying the scheduler code after watching tedu's talk, I'd like to expose this statistics to userland so that I can try playing with cache affinity in the future: gimli:src: sysctl kern.schedstat kern.schedstat.nmigrations=23744 kern.schedstat.noidle=0 kern.schedstat.stolen=9170 kern.schedstat.choose=834843 kern.schedstat.wasidle=808711 kern.schedstat.nomigrations=2388 Opinions ? I see no point in exporting this out. They are essentially pointless knobs that people _will_ fiddle with without a clue. ??? They're readonly.
Re: Expose some scheduler statistics to userland via sysctl
On 17 November 2011 23:39, Ted Unangst t...@tedunangst.com wrote: On Fri, Nov 18, 2011, Thordur Bjornsson wrote: On 2011 Nov 17 (Thu) at 21:18:24 -0200 (-0200), Christiano F. Haesbaert wrote: Moving this to tech@ Hi, I was studying the scheduler code after watching tedu's talk, I'd like to expose this statistics to userland so that I can try playing with cache affinity in the future: gimli:src: sysctl kern.schedstat kern.schedstat.nmigrations=23744 kern.schedstat.noidle=0 kern.schedstat.stolen=9170 kern.schedstat.choose=834843 kern.schedstat.wasidle=808711 kern.schedstat.nomigrations=2388 Opinions ? I see no point in exporting this out. They are essentially pointless knobs that people _will_ fiddle with without a clue. ??? They're readonly. Mike just pointed out I can get those with pstat, so that solves my problem. Still maybe there is a place for sysctl, but I'm not too strong about it anymore. What is the line between sysctl vs globals ?
Re: Expose some scheduler statistics to userland via sysctl
On 18 November 2011 00:59, Christiano F. Haesbaert haesba...@haesbaert.org wrote: On 17 November 2011 23:39, Ted Unangst t...@tedunangst.com wrote: On Fri, Nov 18, 2011, Thordur Bjornsson wrote: On 2011 Nov 17 (Thu) at 21:18:24 -0200 (-0200), Christiano F. Haesbaert wrote: Moving this to tech@ Hi, I was studying the scheduler code after watching tedu's talk, I'd like to expose this statistics to userland so that I can try playing with cache affinity in the future: gimli:src: sysctl kern.schedstat kern.schedstat.nmigrations=23744 kern.schedstat.noidle=0 kern.schedstat.stolen=9170 kern.schedstat.choose=834843 kern.schedstat.wasidle=808711 kern.schedstat.nomigrations=2388 Opinions ? I see no point in exporting this out. They are essentially pointless knobs that people _will_ fiddle with without a clue. ??? They're readonly. Mike just pointed out I can get those with pstat, so that solves my problem. Still maybe there is a place for sysctl, but I'm not too strong about it anymore. What is the line between sysctl vs globals ? Checking again, it makes sense having the sysctl IMHO. We have something very similar in kern.forkstat, kern.nchstats, and the protocol stats.
Re: Expose some scheduler statistics to userland via sysctl
On Fri, Nov 18, 2011, Christiano F. Haesbaert wrote: What is the line between sysctl vs globals ? Checking again, it makes sense having the sysctl IMHO. We have something very similar in kern.forkstat, kern.nchstats, and the protocol stats. I think those stats are useful for a sysadmin to monitor performance, and there actions one can take to affect them. The scheduler is not really controllable. If you just want to peek at some values for debugging, pstat is the way to go. That's why I added it. ;)