From: Matthew Malcomson <[email protected]>
N.b. I've not tested this patch as thoroughly as my other upstream
libgomp patches. I'm in the middle of doing so. However I thought it
useful to get the patch visible early.
This patch goes on top of my earlier patch series (hence why sent "in
reply to" the last patch in that series).
This removes a large amount of overhead in starting a new parallel
region. On one machine with 352 virtual cores this patch reduced the
cost of repeated parallel regions from ~90us to ~8us.
-------- 8< -------------- 8< ------------- 8< ----------
I'm working on reducing the overhead of the creation of a parallel
region. My previous patches have focussed mostly on the barrier that is
a fundamental part of ending/starting a parallel region.
After those patches the majority of the remaining overhead from a
parallel region is inside gomp_team_start. This function is executed by
the primary thread when all secondary threads are paused waiting to be
released from the barrier. This means that it is highly performance
sensitive.
At high thread counts the largest cost in this function is the primary
thread iterating over all threads and setting variables on their
gomp_thread structure.
In an earlier patch I reduced the cost of this function by putting some
data in a shared location from where each of the secondary threads can
fetch it once they are released (hence parallelising the work). This
patch attempts to do the same for all the data where it can be done.
The fields adjusted are as follows:
- nthr->ts.level
This field is set to `team->prev_ts.level`, but the `if` clause that
we are inside has already checked that `team->prev_ts.level == 0` and
since we're re-using this thread it must have come from a non-nested
team before (hence its `nthr->ts.level` must already be `1`).
- nthr->ts.active_level
By a similar argument, this must already be `1` and
`thr->ts.active_level` must also be `1`.
- Move `def_allocator`, `num_teams` and `team_num` into shared
initialisation. These are set the same on every thread, hence storing
them once in the team data structure and having each thread take from
there.
- Mark exiting threads instead of those continuing.
The existing convention is that `gomp_thread_start` sets its own
`thr->fn` to `NULL` and `gomp_team_start` sets the `nthr->fn` of each
thread it wants to continue to the function it needs that thread to
execute. This means the primary thread needs to perform a write on
each thread that is continuing.
Here I switch the convention so that the primary thread sets
`thr->exit` on each of the threads it wants to exit. Those threads
without this boolean flag set read the function that they are to
execute from their team structure. This means the primary thread
writes once per exiting thread instead of once per reused thread. In
the case where the number of threads and affinity setup is not
changing between parallel regions this means no work instead of O(N)
work.
There is still a cause in the main loop of `gomp_thread_start`
checking for `thr->fn`, this is in order to let `gomp_pause_host` and
`gomp_free_thread` continue to work as intended. I could have changed
those functions to adjust the `pool->last_team` instead of setting
`thr->fn` but it didn't seem any better either way.
- Avoid re-calculation of places when relevant place ICV's are
unchanged. If we have calculated the places for each thread in a
team, we're re-using that team, and all the inputs to that calculation
are the same, then we can avoid the calculation and simply re-start
the team.
libgomp/ChangeLog:
* libgomp.h (struct gomp_team): New `ti` member.
New `ppv` member.
New `reused` boolean.
(struct gomp_thread): New `exit` member.
* team.c (gomp_thread_start): Read from `team->ti` for
initialisation.
(get_last_team): Set `team->reused` member when reusing.
(gomp_new_team): Unset `team->reused` member when creating new
team.
(gomp_team_start): Populate `team->ti` instead of each thread
individually, skip re-calculation of places on a reused team
when inputs are the same, avoid reinitialisation of
`nthr->ts.level` and `nthr->ts.active_level` when unnecessary.
Signed-off-by: Matthew Malcomson <[email protected]>
---
libgomp/libgomp.h | 20 ++++++--
libgomp/team.c | 114 +++++++++++++++++++++++++++++++++++-----------
2 files changed, 105 insertions(+), 29 deletions(-)
diff --git a/libgomp/libgomp.h b/libgomp/libgomp.h
index c854743c1c1..1dfdfae64df 100644
--- a/libgomp/libgomp.h
+++ b/libgomp/libgomp.h
@@ -870,9 +870,22 @@ struct gomp_team
team very many times. This allows each secondary thread to perform the
re-initialisation instead of having the primary thread need to perform
that initialisation. */
- struct gomp_task_icv task_icv_init;
- struct gomp_task *task_parent_init;
- struct gomp_taskgroup *task_taskgroup;
+ struct thread_init {
+ struct gomp_task_icv task_icv_init;
+ struct gomp_task *task_parent_init;
+ struct gomp_taskgroup *task_taskgroup;
+ uintptr_t def_allocator;
+ unsigned int num_teams;
+ unsigned int team_num;
+ void (*fn) (void *data);
+ void *data;
+ } ti;
+ struct prev_place_vars {
+ unsigned int place_partition_off;
+ unsigned int place_partition_len;
+ char bind;
+ } ppv;
+ bool reused;
/* This array contains structures for implicit tasks. */
struct gomp_task implicit_task[];
@@ -886,6 +899,7 @@ struct gomp_thread
/* This is the function that the thread should run upon launch. */
void (*fn) (void *data);
void *data;
+ bool exit;
/* This is the current team state for this thread. The ts.team member
is NULL only if the thread is idle. */
diff --git a/libgomp/team.c b/libgomp/team.c
index 08c03fcf949..51758c46e36 100644
--- a/libgomp/team.c
+++ b/libgomp/team.c
@@ -109,6 +109,7 @@ gomp_thread_start (void *xdata)
thr->place = data->place;
thr->num_teams = data->num_teams;
thr->team_num = data->team_num;
+ thr->exit = false;
#ifdef GOMP_NEEDS_THREAD_HANDLE
thr->handle = data->handle;
#endif
@@ -143,6 +144,7 @@ gomp_thread_start (void *xdata)
struct gomp_team *team = thr->ts.team;
struct gomp_task *task = thr->task;
+ thr->fn = NULL;
local_fn (local_data);
gomp_team_barrier_wait_final (&team->barrier, thr->ts.team_id);
gomp_finish_task (task);
@@ -155,14 +157,27 @@ gomp_thread_start (void *xdata)
thread before it frees this team because it will synchronise with
the primary thread in the next iteration of this loop (in
`gomp_team_barrier_wait_final`). */
- if (!gomp_barrier_can_hold (&team->barrier) || thr->fn == NULL)
+ if (!gomp_barrier_can_hold (&team->barrier) || thr->exit)
gomp_simple_barrier_wait (&pool->threads_dock);
/* Early exit because if `thr->fn == NULL` then we can't guarantee
the `thr->ts.team` hasn't been freed by the primary thread racing
ahead. Hence we don't want to write to it. */
- if (thr->fn == NULL)
+ if (thr->exit)
break;
+
+ /* Usually `thr->fn` is not used or set. Is used when secondary
+ threads are "set going" without being assigned to any team.
+ See `gomp_pause_host` and `gomp_free_thread`.
+ In this case we don't want to bother reading the team
+ initialisation things because they've not been set up for us. */
+ if (thr->fn)
+ {
+ local_fn = thr->fn;
+ local_data = thr->data;
+ continue;
+ }
+
team = thr->ts.team;
thr->ts.work_share = &team->work_shares[0];
thr->ts.last_work_share = NULL;
@@ -171,15 +186,16 @@ gomp_thread_start (void *xdata)
#endif
thr->ts.static_trip = 0;
thr->task = &team->implicit_task[thr->ts.team_id];
- gomp_init_task (thr->task, team->task_parent_init,
- &team->task_icv_init);
- thr->task->taskgroup = team->task_taskgroup;
-
- local_fn = thr->fn;
- local_data = thr->data;
- thr->fn = NULL;
+ gomp_init_task (thr->task, team->ti.task_parent_init,
+ &team->ti.task_icv_init);
+ thr->task->taskgroup = team->ti.task_taskgroup;
+ thr->num_teams = team->ti.num_teams;
+ thr->team_num = team->ti.team_num;
+ thr->ts.def_allocator = team->ti.def_allocator;
+ local_fn = team->ti.fn;
+ local_data = team->ti.data;
}
- while (local_fn);
+ while (true);
}
gomp_sem_destroy (&thr->release);
@@ -208,6 +224,7 @@ get_last_team (unsigned nthreads)
"prev_barrier not within cached team: %p != %p",
pool->prev_barrier, &pool->last_team->barrier);
pool->last_team = NULL;
+ last_team->reused = true;
return last_team;
}
}
@@ -241,6 +258,7 @@ gomp_new_team (unsigned nthreads)
gomp_mutex_init (&team->task_lock);
team->nthreads = nthreads;
+ team->reused = false;
}
team->work_share_chunk = 8;
@@ -454,15 +472,6 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned
nthreads,
thr->task->taskgroup = taskgroup;
team->implicit_task[0].icv.nthreads_var = nthreads_var;
team->implicit_task[0].icv.bind_var = bind_var;
- /* Copy entirely rather than copy pointer so that we have an immutable copy
- for the secondary threads to take from for the time-frame that said
- secondary threads are executing on. Can't copy from
- `team->implicit_task[0].icv` because primary thread could have adjusted
- its per-task ICV by the time the secondary thread gets around to
- initialising things. */
- team->task_icv_init = team->implicit_task[0].icv;
- team->task_parent_init = task;
- team->task_taskgroup = taskgroup;
if (nthreads == 1)
return;
@@ -549,6 +558,41 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned
nthreads,
team->prev_ts.team_id);
old_threads_used = pool->threads_used;
+ /* Copy entirely rather than copy pointer so that we have an immutable
+ copy for the secondary threads to take from for the time-frame that
+ said secondary threads are executing on. Can't copy from
+ `team->implicit_task[0].icv` because primary thread could have adjusted
+ its per-task ICV by the time the secondary thread gets around to
+ initialising things. */
+ team->ti.task_icv_init = team->implicit_task[0].icv;
+ team->ti.task_parent_init = task;
+ team->ti.task_taskgroup = taskgroup;
+ team->ti.num_teams = thr->num_teams;
+ team->ti.team_num = thr->team_num;
+ team->ti.fn = fn;
+ team->ti.data = data;
+ team->ti.def_allocator = thr->ts.def_allocator;
+
+ if (team->reused
+ && team->ppv.place_partition_off == thr->ts.place_partition_off
+ && team->ppv.place_partition_len == thr->ts.place_partition_len
+ && team->ppv.bind == bind)
+ {
+ gomp_assert (
+ nthreads == old_threads_used,
+ "odd number of threads when re-using team: nthreads = %u "
+ "old_threads_used = %u\n",
+ nthreads, old_threads_used);
+ goto do_release;
+ }
+ else
+ {
+ team->ppv.place_partition_off = thr->ts.place_partition_off;
+ team->ppv.place_partition_len = thr->ts.place_partition_len;
+ team->ppv.bind = bind;
+ }
+
+
if (nthreads == old_threads_used)
{
n = nthreads;
@@ -712,6 +756,8 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned
nthreads,
pool->threads[j]->data = affinity_thr[l];
affinity_thr[l] = pool->threads[j];
}
+ else
+ pool->threads[j]->exit = true;
pool->threads[j] = NULL;
}
if (nthreads > old_threads_used)
@@ -752,21 +798,34 @@ gomp_team_start (void (*fn) (void *), void *data,
unsigned nthreads,
nthr = pool->threads[i];
__atomic_store_n (&nthr->ts.team, team, MEMMODEL_RELEASE);
nthr->ts.team_id = i;
- nthr->ts.level = team->prev_ts.level + 1;
- nthr->ts.active_level = thr->ts.active_level;
+ gomp_assert (
+ nthr->ts.level == 1 && team->prev_ts.level == 0,
+ "Level indicators surprising: nthr: %u, team->prev_ts: %u\n",
+ nthr->ts.level, team->prev_ts.level);
+ gomp_assert (
+ nthr->ts.active_level == 1 && thr->ts.active_level == 1,
+ "active_level indicators surprising: nthr: %u, thr: %u\n",
+ nthr->ts.active_level, thr->ts.active_level);
nthr->ts.place_partition_off = place_partition_off;
nthr->ts.place_partition_len = place_partition_len;
- nthr->ts.def_allocator = thr->ts.def_allocator;
- nthr->num_teams = thr->num_teams;
- nthr->team_num = thr->team_num;
nthr->place = place;
- nthr->fn = fn;
- nthr->data = data;
team->ordered_release[i] = &nthr->release;
}
if (__builtin_expect (affinity_thr != NULL, 0))
{
+ /* Mark each of the threads that are exiting as such. Hence in
+ `gomp_thread_start` they exit their main loop. */
+ for (unsigned l = 0; l < team->prev_ts.place_partition_len; l++)
+ {
+ struct gomp_thread *t = affinity_thr[l];
+ while (t)
+ {
+ struct gomp_thread *next = (struct gomp_thread *) t->data;
+ t->exit = true;
+ t = next;
+ }
+ }
/* If AFFINITY_THR is non-NULL just because we had to
permute some threads in the pool, but we've managed
to find exactly as many old threads as we'd find
@@ -857,6 +916,9 @@ gomp_team_start (void (*fn) (void *), void *data, unsigned
nthreads,
}
}
}
+ else
+ for (unsigned j = n; j < old_threads_used; j++)
+ pool->threads[j]->exit = true;
if (i == nthreads)
{
--
2.43.0
