From: Wei Fang <fangw...@huawei.com>
On a multiprocessor system, setting the CPU affinity mask can
improve performance. We already tested it on a NUMA machine, and
had obtained performance benifits. Since that, I provided a
configurable way to set the CPU affinity of sender threads. People
using nfs-ganesha can arbitrarily tune the bind status of sender
threads through amending these configurations below:
* Sender_Cpus_Allowed: set CPU affinity, it's a comma-delimited list
of CPU numbers the sender threads may run on
* Sender_Cpumask: the same as Sender_Cpus_Allowed, but it allows a
comma-hexadecimal value of allowed cpus
* Sender_Policy: Set the policy of how ntirpc distributes the CPUs,
split or shared is allowed
Signed-off-by: Wei Fang <fangw...@huawei.com>
---
The patches about parsing these config items will be posted soon.
TODO list:
- I use thr_cnt to calculate the cpu in the split way. This may
cause the problem that all the workers are running on the same
CPU, since the threads are created and destroyed on the running
time. So the better way is to trace the number of the threads on
each cpu by realtime, and then bind the new thread to the cpu on
which the number of threads is less than all the other cpus. Further
more, we can tune the bind status by a background thread every
several seconds. This smart affinity set function would be implement
in the future.
- Introduce the configuration of the scheduler class and priority for
the work threads.
---
ntirpc/rpc/svc.h | 7 +++++++
ntirpc/rpc/work_pool.h | 3 +++
src/svc.c | 9 +++++++--
src/svc_internal.h | 2 ++
src/work_pool.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++
5 files changed, 66 insertions(+), 2 deletions(-)
diff --git a/ntirpc/rpc/svc.h b/ntirpc/rpc/svc.h
index b38cc96..4b6e078 100644
--- a/ntirpc/rpc/svc.h
+++ b/ntirpc/rpc/svc.h
@@ -121,6 +121,11 @@
#define SVC_VCCR_NONE 0x0000
+enum svc_cpus_allowed_policy {
+ SVC_CPUS_SHARED,
+ SVC_CPUS_SPLIT
+};
+
/* Svc event strategy */
enum svc_event_type {
SVC_EVENT_FDSET /* trad. using select and poll (currently unhooked) */ ,
@@ -139,6 +144,8 @@ typedef struct svc_init_params {
u_int gss_max_idle_gen;
u_int gss_max_gc;
u_int ioq_thrd_max;
+ cpu_set_t cpu_mask;
+ u_int policy;
} svc_init_params;
/* Svc param flags */
diff --git a/ntirpc/rpc/work_pool.h b/ntirpc/rpc/work_pool.h
index 9bb7fde..ef3bfb9 100644
--- a/ntirpc/rpc/work_pool.h
+++ b/ntirpc/rpc/work_pool.h
@@ -53,6 +53,8 @@ struct work_pool_entry {
struct work_pool_params {
int32_t thrd_max;
int32_t thrd_min;
+ cpu_set_t cpu_mask;
+ uint32_t policy;
};
struct work_pool {
@@ -61,6 +63,7 @@ struct work_pool {
pthread_attr_t attr;
struct work_pool_params params;
uint32_t n_threads;
+ uint32_t work_thrd_cnt;
};
struct work_pool_thread {
diff --git a/src/svc.c b/src/svc.c
index c91386c..ee93c1c 100644
--- a/src/svc.c
+++ b/src/svc.c
@@ -121,7 +121,9 @@ svc_work_pool_init()
{
struct work_pool_params params = {
.thrd_max = __svc_params->ioq.thrd_max,
- .thrd_min = 2
+ .thrd_min = 2,
+ .cpu_mask = __svc_params->ioq.cpu_mask,
+ .policy = __svc_params->ioq.policy
};
return work_pool_init(&svc_work_pool, "svc_work_pool", ¶ms);
@@ -187,7 +189,10 @@ svc_init(svc_init_params *params)
else
__svc_params->ioq.thrd_max = 200;
- /* uses ioq.thrd_max */
+ __svc_params->ioq.cpu_mask = params->cpu_mask;
+ __svc_params->ioq.policy = params->policy;
+
+ /* uses ioq params */
if (svc_work_pool_init()) {
mutex_unlock(&__svc_params->mtx);
return false;
diff --git a/src/svc_internal.h b/src/svc_internal.h
index 2318fb5..fea9f53 100644
--- a/src/svc_internal.h
+++ b/src/svc_internal.h
@@ -70,6 +70,8 @@ struct svc_params {
struct {
u_int thrd_max;
+ u_int policy;
+ cpu_set_t cpu_mask;
} ioq;
};
diff --git a/src/work_pool.c b/src/work_pool.c
index f9ef6aa..56dab8e 100644
--- a/src/work_pool.c
+++ b/src/work_pool.c
@@ -54,6 +54,9 @@
#include <string.h>
#include <errno.h>
#include <intrinsic.h>
+#include <unistd.h>
+#include <syscall.h>
+#include <sched.h>
#include <rpc/work_pool.h>
@@ -160,6 +163,28 @@ work_pool_wait(struct work_pool *pool, struct
work_pool_thread *wpt)
return (0);
}
+static inline void
+cpus_split(cpu_set_t *mask, uint32_t cpu_index, cpu_set_t *per_cpu_mask)
+{
+ uint32_t i, index, cpus_in_mask;
+
+ cpus_in_mask = CPU_COUNT(mask);
+ cpu_index = cpu_index % cpus_in_mask;
+
+ index = 0;
+ for (i = 0; ; i++) {
+ if (!CPU_ISSET(i, mask))
+ continue;
+
+ if (cpu_index == index) {
+ CPU_SET(i, per_cpu_mask);
+ break;
+ }
+
+ index++;
+ }
+}
+
/**
* @brief The worker thread
*
@@ -174,8 +199,30 @@ work_pool_thread(void *arg)
{
struct work_pool_thread *wpt = arg;
struct work_pool *pool = wpt->pool;
+ struct work_pool_params *params = &pool->params;
struct poolq_entry *have;
+ if (CPU_COUNT(¶ms->cpu_mask) > 0) {
+ int tid;
+ uint32_t thrd_cnt;
+ cpu_set_t mask;
+
+ CPU_ZERO(&mask);
+ tid = (int)syscall(SYS_gettid);
+
+ __warnx(TIRPC_DEBUG_FLAG_EVENT,
+ "Send thread %d set CPU affinity", tid);
+
+ thrd_cnt = atomic_postinc_uint32_t(&pool->work_thrd_cnt);
+ if (params->policy == SVC_CPUS_SPLIT) {
+ cpus_split(¶ms->cpu_mask, thrd_cnt, &mask);
+ sched_setaffinity(tid, sizeof(mask), &mask);
+ } else {
+ sched_setaffinity(tid, sizeof(params->cpu_mask),
+ ¶ms->cpu_mask);
+ }
+ }
+
atomic_inc_uint32_t(&pool->n_threads);
pthread_cond_init(&wpt->pqcond, NULL);
--
1.7.1
------------------------------------------------------------------------------
_______________________________________________
Nfs-ganesha-devel mailing list
Nfs-ganesha-devel@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/nfs-ganesha-devel
