Add the -J/--detect-job-slots flag as an shorthand equivalent for -j
with a value of nproc+2. The help message is deliberately left ambiguous
so that we could change it to a different heuristic in future, if
desired.
The -J flag is *not* passed through to sub-makes in MAKEFLAGS, but
instead expanded into -jN (which *does* get passed through).
Signed-off-by: Matt Staveley-Taylor <matt.stav.tay...@gmail.com>
---
Changes in v2:
- Use nproc + 2 rather than nproc. This is what the ninja build system
uses [1], so should not cause any surprises.
- Don't pass -J through to sub-makes in MAKEFLAGS. This was causing
both -J and -jN to be present.
- Added tests and documentation.
[1]
https://github.com/ninja-build/ninja/blob/71b59d858273e14294210c3da62b630351e6ce4f/src/ninja.cc#L251
Thanks all for the discussion. In my opinion, nproc / 2 or nproc * 2 are
much more 'surprising' for users. If someone finds that nproc (+ 2)
works poorly on their system, then they're free to use -j manually. This
argument is supposed to be a convenient shorthand with boring behaviour.
bootstrap.conf | 3 ++-
doc/make.texi | 10 +++++++++
src/main.c | 14 +++++++++++-
tests/scripts/features/parallelism | 34 ++++++++++++++++++++++++++++++
4 files changed, 59 insertions(+), 2 deletions(-)
diff --git a/bootstrap.conf b/bootstrap.conf
index 98a709a5..9716025e 100644
--- a/bootstrap.conf
+++ b/bootstrap.conf
@@ -62,4 +62,5 @@ getloadavg
host-cpu-c-abi
largefile
make-glob
-make-macros"
+make-macros
+nproc"
diff --git a/doc/make.texi b/doc/make.texi
index cb7420d0..4161484f 100644
--- a/doc/make.texi
+++ b/doc/make.texi
@@ -9651,6 +9651,16 @@ If there is more than one @samp{-j} option, the last one
is effective.
@xref{Parallel, ,Parallel Execution}, for more information on how
recipes are run. Note that this option is ignored on MS-DOS.
+@item -J
+@cindex @code{-J}
+@itemx --detect-job-slots
+@cindex @code{--detect-job-slots}
+Like @samp{-j}, but guess a reasonable value for the number of parallel jobs
+based on the number of CPU cores available. The value is obtained through the
+same mechanism used by GNU Coreutils @code{nproc}. Currently, the number of
+jobs is computed as @code{nproc} + 2, but this is subject to change. If both
+@samp{-j} and @samp{-J} are present, the value given to @samp{-j} is used.
+
@item --jobserver-style=[@var{style}]
@cindex @code{--jobserver-style}
Chooses the style of jobserver to use. This option only has effect if
diff --git a/src/main.c b/src/main.c
index 15104212..542da512 100644
--- a/src/main.c
+++ b/src/main.c
@@ -15,6 +15,7 @@ You should have received a copy of the GNU General Public
License along with
this program. If not, see <https://www.gnu.org/licenses/>. */
#include "makeint.h"
+#include "nproc.h"
#include <assert.h>
#if MK_OS_W32
@@ -236,6 +237,9 @@ static const int default_job_slots = INVALID_JOB_SLOTS;
static const int inf_jobs = 0;
+/* If nonzero, detect the number of job slots based on the number of CPUs. */
+static int detect_job_slots_flag = 0;
+
/* Authorization for the jobserver. */
char *jobserver_auth = NULL;
@@ -360,6 +364,8 @@ static const char *const usage[] =
N_("\
--jobserver-style=STYLE Select the style of jobserver to use.\n"),
N_("\
+ -J, --detect-job-slots Detect job slots based on number of CPU
cores.\n"),
+ N_("\
-k, --keep-going Keep going when some targets can't be made.\n"),
N_("\
-l [N], --load-average[=N], --max-load[=N]\n\
@@ -465,6 +471,7 @@ static struct command_switch switches[] =
{ 'E', strlist, &eval_strings, 1, 0, 0, 0, 0, 0, "eval", 0 },
{ 'h', flag, &print_usage_flag, 0, 0, 0, 0, 0, 0, "help", 0 },
{ 'i', flag, &ignore_errors_flag, 1, 1, 0, 0, 0, 0, "ignore-errors", 0 },
+ { 'J', flag, &detect_job_slots_flag, 1, 0, 0, 0, 0, 0, "detect-job-slots",
0 },
{ 'k', flag, &keep_going_flag, 1, 1, 0, 0, 0, &default_keep_going_flag,
"keep-going", &keep_going_origin },
{ 'L', flag, &check_symlink_flag, 1, 1, 0, 0, 0, 0, "check-symlink-times",
0 },
@@ -1616,7 +1623,12 @@ main (int argc, char **argv, char **envp)
argv_slots = arg_job_slots;
if (arg_job_slots == INVALID_JOB_SLOTS)
- arg_job_slots = env_slots;
+ {
+ if (detect_job_slots_flag)
+ arg_job_slots = num_processors(NPROC_CURRENT_OVERRIDABLE) + 2;
+ else
+ arg_job_slots = env_slots;
+ }
}
if (print_usage_flag)
diff --git a/tests/scripts/features/parallelism
b/tests/scripts/features/parallelism
index 8c3e8294..4ea0db5f 100644
--- a/tests/scripts/features/parallelism
+++ b/tests/scripts/features/parallelism
@@ -35,6 +35,40 @@ def_3 : ; @#HELPER# wait ONE file FOUR!,
'-j4', "file ONE\nwait ONE\nfile FOUR\nwait FOUR\nfile
THREE\nwait THREE\nTWO");
rmfiles(qw(ONE TWO THREE FOUR));
+# Check -J takes the value from OMP_NUM_THREADS (and adds two)
+$ENV{'OMP_NUM_THREADS'} = '1';
+run_make_test(q!
+all : show_j def_1 def_2 def_3
+show_j : ; @echo $(firstword ${MAKEFLAGS})
+def_1 : ; @#HELPER# file ONE wait THREE out TWO
+def_2 : ; @#HELPER# wait FOUR file THREE
+def_3 : ; @#HELPER# wait ONE file FOUR!,
+ '-J', "-j3\nfile ONE\nwait ONE\nfile FOUR\nwait FOUR\nfile
THREE\nwait THREE\nTWO");
+rmfiles(qw(ONE TWO THREE FOUR));
+
+# Check -J works in MAKEFLAGS
+$ENV{'OMP_NUM_THREADS'} = '3';
+$ENV{'MAKEFLAGS'} = '-J';
+run_make_test(q!
+all : show_j def_1 def_2 def_3
+show_j : ; @echo $(firstword ${MAKEFLAGS})
+def_1 : ; @#HELPER# file ONE wait THREE out TWO
+def_2 : ; @#HELPER# wait FOUR file THREE
+def_3 : ; @#HELPER# wait ONE file FOUR!,
+ '-J', "-j5\nfile ONE\nwait ONE\nfile FOUR\nwait FOUR\nfile
THREE\nwait THREE\nTWO");
+rmfiles(qw(ONE TWO THREE FOUR));
+
+# Check -j should take precedence over -J
+$ENV{'OMP_NUM_THREADS'} = '5';
+run_make_test(q!
+all : show_j def_1 def_2 def_3
+show_j : ; @echo $(firstword ${MAKEFLAGS})
+def_1 : ; @#HELPER# file ONE wait THREE out TWO
+def_2 : ; @#HELPER# wait FOUR file THREE
+def_3 : ; @#HELPER# wait ONE file FOUR!,
+ '-J -j3', "-j3\nfile ONE\nwait ONE\nfile FOUR\nwait FOUR\nfile
THREE\nwait THREE\nTWO");
+rmfiles(qw(ONE TWO THREE FOUR));
+
# Test parallelism with included files. Here we sleep/echo while
# building the included files, to test that they are being built in
# parallel.
--
2.44.0
