On a multiprocessor system, setting the CPU affinity mask can improve performance. We already tested it on a NUMA machine, and had obtained performance benifits. Since that, I provided a configurable way to set the CPU affinity of sender threads. People using nfs-ganesha can arbitrarily tune the bind status of sender threads through amending these configurations below:
* Sender_Cpus_Allowed: set CPU affinity, it's a comma-delimited list of CPU numbers the sender threads may run on * Sender_Cpumask: the same as Sender_Cpus_Allowed, but it allows a comma-hexadecimal value of allowed cpus * Sender_Cpus_Allowed_Policy: Set the policy of how ntirpc distributes the CPUs, split or shared is allowed V2: * use a more explicit name about those params. Signed-off-by: Wei Fang <fangw...@huawei.com> --- The patches about parsing these config items have been posted on gerrithub: https://review.gerrithub.io/247979 https://review.gerrithub.io/247980 https://review.gerrithub.io/247983 TODO list: - I use thr_cnt to calculate the cpu in the split way. This may cause the problem that all the workers are running on the same CPU, since the threads are created and destroyed on the running time. So the better way is to trace the number of the threads on each cpu by realtime, and then bind the new thread to the cpu on which the number of threads is less than all the other cpus. Further more, we can tune the bind status by a background thread every several seconds. This smart affinity set function would be implement in the future. - Introduce the configuration of the scheduler class and priority for the work threads. --- ntirpc/rpc/svc.h | 7 +++++++ ntirpc/rpc/work_pool.h | 3 +++ src/svc.c | 9 +++++++-- src/svc_internal.h | 2 ++ src/work_pool.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 2 deletions(-) diff --git a/ntirpc/rpc/svc.h b/ntirpc/rpc/svc.h index b38cc96..8cb8875 100644 --- a/ntirpc/rpc/svc.h +++ b/ntirpc/rpc/svc.h @@ -121,6 +121,11 @@ #define SVC_VCCR_NONE 0x0000 +enum svc_cpus_allowed_policy { + SVC_CPUS_SHARED, + SVC_CPUS_SPLIT +}; + /* Svc event strategy */ enum svc_event_type { SVC_EVENT_FDSET /* trad. using select and poll (currently unhooked) */ , @@ -139,6 +144,8 @@ typedef struct svc_init_params { u_int gss_max_idle_gen; u_int gss_max_gc; u_int ioq_thrd_max; + cpu_set_t sender_cpu_mask; + u_int sender_policy; } svc_init_params; /* Svc param flags */ diff --git a/ntirpc/rpc/work_pool.h b/ntirpc/rpc/work_pool.h index 9bb7fde..ef3bfb9 100644 --- a/ntirpc/rpc/work_pool.h +++ b/ntirpc/rpc/work_pool.h @@ -53,6 +53,8 @@ struct work_pool_entry { struct work_pool_params { int32_t thrd_max; int32_t thrd_min; + cpu_set_t cpu_mask; + uint32_t policy; }; struct work_pool { @@ -61,6 +63,7 @@ struct work_pool { pthread_attr_t attr; struct work_pool_params params; uint32_t n_threads; + uint32_t work_thrd_cnt; }; struct work_pool_thread { diff --git a/src/svc.c b/src/svc.c index c91386c..22adb5c 100644 --- a/src/svc.c +++ b/src/svc.c @@ -121,7 +121,9 @@ svc_work_pool_init() { struct work_pool_params params = { .thrd_max = __svc_params->ioq.thrd_max, - .thrd_min = 2 + .thrd_min = 2, + .cpu_mask = __svc_params->ioq.cpu_mask, + .policy = __svc_params->ioq.policy }; return work_pool_init(&svc_work_pool, "svc_work_pool", ¶ms); @@ -187,7 +189,10 @@ svc_init(svc_init_params *params) else __svc_params->ioq.thrd_max = 200; - /* uses ioq.thrd_max */ + __svc_params->ioq.cpu_mask = params->sender_cpu_mask; + __svc_params->ioq.policy = params->sender_policy; + + /* uses ioq params */ if (svc_work_pool_init()) { mutex_unlock(&__svc_params->mtx); return false; diff --git a/src/svc_internal.h b/src/svc_internal.h index 2318fb5..fea9f53 100644 --- a/src/svc_internal.h +++ b/src/svc_internal.h @@ -70,6 +70,8 @@ struct svc_params { struct { u_int thrd_max; + u_int policy; + cpu_set_t cpu_mask; } ioq; }; diff --git a/src/work_pool.c b/src/work_pool.c index f9ef6aa..56dab8e 100644 --- a/src/work_pool.c +++ b/src/work_pool.c @@ -54,6 +54,9 @@ #include <string.h> #include <errno.h> #include <intrinsic.h> +#include <unistd.h> +#include <syscall.h> +#include <sched.h> #include <rpc/work_pool.h> @@ -160,6 +163,28 @@ work_pool_wait(struct work_pool *pool, struct work_pool_thread *wpt) return (0); } +static inline void +cpus_split(cpu_set_t *mask, uint32_t cpu_index, cpu_set_t *per_cpu_mask) +{ + uint32_t i, index, cpus_in_mask; + + cpus_in_mask = CPU_COUNT(mask); + cpu_index = cpu_index % cpus_in_mask; + + index = 0; + for (i = 0; ; i++) { + if (!CPU_ISSET(i, mask)) + continue; + + if (cpu_index == index) { + CPU_SET(i, per_cpu_mask); + break; + } + + index++; + } +} + /** * @brief The worker thread * @@ -174,8 +199,30 @@ work_pool_thread(void *arg) { struct work_pool_thread *wpt = arg; struct work_pool *pool = wpt->pool; + struct work_pool_params *params = &pool->params; struct poolq_entry *have; + if (CPU_COUNT(¶ms->cpu_mask) > 0) { + int tid; + uint32_t thrd_cnt; + cpu_set_t mask; + + CPU_ZERO(&mask); + tid = (int)syscall(SYS_gettid); + + __warnx(TIRPC_DEBUG_FLAG_EVENT, + "Send thread %d set CPU affinity", tid); + + thrd_cnt = atomic_postinc_uint32_t(&pool->work_thrd_cnt); + if (params->policy == SVC_CPUS_SPLIT) { + cpus_split(¶ms->cpu_mask, thrd_cnt, &mask); + sched_setaffinity(tid, sizeof(mask), &mask); + } else { + sched_setaffinity(tid, sizeof(params->cpu_mask), + ¶ms->cpu_mask); + } + } + atomic_inc_uint32_t(&pool->n_threads); pthread_cond_init(&wpt->pqcond, NULL); -- 2.5.0 ------------------------------------------------------------------------------ Monitor Your Dynamic Infrastructure at Any Scale With Datadog! Get real-time metrics from all of your servers, apps and tools in one place. SourceForge users - Click here to start your Free Trial of Datadog now! http://pubads.g.doubleclick.net/gampad/clk?id=241902991&iu=/4140 _______________________________________________ Nfs-ganesha-devel mailing list Nfs-ganesha-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/nfs-ganesha-devel
