From: Kyrylo Tkachov <[email protected]>
The stage 2 and stage 3 object comparison runs one command per object in a
serial shell loop. Everything after it waits while an otherwise idle machine
compares thousands of objects.
Generate one make target per object and run the comparisons through $(MAKE),
so they share the original jobserver. Keep the generated makefile and result
shards invocation-specific, clean them on normal and signal exits, and sort
the shards before creating the traditional .bad_compare diagnostic. Files in
the exclusion list still only warn. Comparator statuses other than zero or
one now report an operational error instead of being treated as equal.
The tail-based fallback for cmp implementations without byte skipping used
fixed temporary names. Give those files process-specific names, clean them
with a trap, and preserve the same status contract for tail failures and
signals.
On highly parallel AArch64 and x86_64 systems this reduces comparison wall
time by about 95%.
ChangeLog:
* Makefile.tpl ([+compare-target+]): Run object comparisons through an
invocation-specific parallel sub-make. Collect sorted failure shards
and clean temporary state on exit.
* Makefile.in: Regenerate.
* configure: Regenerate.
config/ChangeLog:
* acx.m4 (ACX_PROG_CMP_IGNORE_INITIAL): Use process-specific temporary
files and clean them on exit. Diagnose tail failures as errors.
Signed-off-by: Kyrylo Tkachov <[email protected]>
---
Makefile.in | 116 ++++++++++++++++++++++++++++++++++++--------------
Makefile.tpl | 58 ++++++++++++++++++-------
config/acx.m4 | 11 ++++-
configure | 11 ++++-
4 files changed, 146 insertions(+), 50 deletions(-)
diff --git a/Makefile.in b/Makefile.in
index f34142cc077..5cca69c3f52 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -66022,6 +66022,9 @@ do-clean: clean-stage3
# only possibility, but now it conflicts with no-bootstrap rules
@if gcc-bootstrap
+# Run the comparisons in parallel through a generated sub-makefile under
+# the jobserver. Each failing recipe writes a separate result shard that
+# is collected after the sub-make finishes.
compare:
@r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
@@ -66030,29 +66033,52 @@ compare:
exit 0; \
fi; \
: $(MAKE); $(stage); \
- rm -f .bad_compare; \
+ compare_id=$$$$; \
+ bad_compare=.bad_compare.$$compare_id; export bad_compare; \
+ compare_makefile=compare.$$compare_id.mk; \
+ trap 'st=$$?; rm -f "$$compare_makefile" "$$bad_compare" \
+ "$$bad_compare".*; trap - 0; exit $$st' 0; \
+ trap 'exit 1' 1 2 3 15; \
echo Comparing stages 2 and 3; \
sed=`echo stage3 | sed 's,^stage,,;s,.,.,g'`; \
files=`find stage3-* -name "*$(objext)" -print | \
sed -n s,^stage$$sed-,,p`; \
- for file in $${files} ${extra-compare}; do \
- f1=$$r/stage2-$$file; f2=$$r/stage3-$$file; \
- if test ! -f $$f1; then continue; fi; \
- $(do-compare) > /dev/null 2>&1; \
- if test $$? -eq 1; then \
- case $$file in \
- @compare_exclusions@) \
- echo warning: $$file differs ;; \
- *) \
- echo $$file differs >> .bad_compare ;; \
- esac; \
- fi; \
- done; \
- if [ -f .bad_compare ]; then \
+ cmp_raw='$(do-compare)'; \
+ { \
+ echo 'all:'; \
+ echo '.PHONY: all FORCE'; \
+ echo 'FORCE:'; \
+ printf 'compare/%%: FORCE ; @'; \
+ printf 'f1=$$$$r/stage2-$$*; '; \
+ printf 'f2=$$$$r/stage3-$$*; '; \
+ printf '%s' "$$cmp_raw" | sed 's,\$$,$$$$,g'; \
+ printf ' > /dev/null 2>&1; st=$$$$?; '; \
+ printf 'if test $$$$st -eq 1; then '; \
+ printf 'case $$* in '; \
+ printf '@compare_exclusions@) echo warning: $$* differs ;; '; \
+ printf '*) echo $$* differs >> "$$$$bad_compare.$$$$$$$$" ;; '; \
+ printf 'esac; '; \
+ printf 'elif test $$$$st -ne 0; then '; \
+ printf 'echo "$$* compare: error status $$$$st" '; \
+ printf '>> "$$$$bad_compare.$$$$$$$$"; fi\n'; \
+ for file in $${files} ${extra-compare}; do \
+ if test ! -f $$r/stage2-$$file; then continue; fi; \
+ echo "all: compare/$$file"; \
+ done; \
+ } > "$$compare_makefile"; \
+ $(MAKE) -s -f "$$compare_makefile" all; compare_status=$$?; \
+ if test $$compare_status -ne 0; then \
+ exit $$compare_status; \
+ fi; \
+ set -- "$$bad_compare".*; \
+ if test -f "$$1"; then \
echo "Bootstrap comparison failure!"; \
- cat .bad_compare; \
+ LC_ALL=C sort "$$bad_compare".* > "$$bad_compare" || exit 1; \
+ cat "$$bad_compare"; \
+ mv -f "$$bad_compare" .bad_compare; \
exit 1; \
else \
+ rm -f .bad_compare; \
echo Comparison successful.; \
fi; \
$(STAMP) compare
@@ -66456,6 +66482,9 @@ do-clean: clean-stage4
# only possibility, but now it conflicts with no-bootstrap rules
@if gcc-bootstrap
+# Run the comparisons in parallel through a generated sub-makefile under
+# the jobserver. Each failing recipe writes a separate result shard that
+# is collected after the sub-make finishes.
compare3:
@r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
@@ -66464,29 +66493,52 @@ compare3:
exit 0; \
fi; \
: $(MAKE); $(stage); \
- rm -f .bad_compare; \
+ compare_id=$$$$; \
+ bad_compare=.bad_compare.$$compare_id; export bad_compare; \
+ compare_makefile=compare3.$$compare_id.mk; \
+ trap 'st=$$?; rm -f "$$compare_makefile" "$$bad_compare" \
+ "$$bad_compare".*; trap - 0; exit $$st' 0; \
+ trap 'exit 1' 1 2 3 15; \
echo Comparing stages 3 and 4; \
sed=`echo stage4 | sed 's,^stage,,;s,.,.,g'`; \
files=`find stage4-* -name "*$(objext)" -print | \
sed -n s,^stage$$sed-,,p`; \
- for file in $${files} ${extra-compare}; do \
- f1=$$r/stage3-$$file; f2=$$r/stage4-$$file; \
- if test ! -f $$f1; then continue; fi; \
- $(do-compare3) > /dev/null 2>&1; \
- if test $$? -eq 1; then \
- case $$file in \
- @compare_exclusions@) \
- echo warning: $$file differs ;; \
- *) \
- echo $$file differs >> .bad_compare ;; \
- esac; \
- fi; \
- done; \
- if [ -f .bad_compare ]; then \
+ cmp_raw='$(do-compare3)'; \
+ { \
+ echo 'all:'; \
+ echo '.PHONY: all FORCE'; \
+ echo 'FORCE:'; \
+ printf 'compare3/%%: FORCE ; @'; \
+ printf 'f1=$$$$r/stage3-$$*; '; \
+ printf 'f2=$$$$r/stage4-$$*; '; \
+ printf '%s' "$$cmp_raw" | sed 's,\$$,$$$$,g'; \
+ printf ' > /dev/null 2>&1; st=$$$$?; '; \
+ printf 'if test $$$$st -eq 1; then '; \
+ printf 'case $$* in '; \
+ printf '@compare_exclusions@) echo warning: $$* differs ;; '; \
+ printf '*) echo $$* differs >> "$$$$bad_compare.$$$$$$$$" ;; '; \
+ printf 'esac; '; \
+ printf 'elif test $$$$st -ne 0; then '; \
+ printf 'echo "$$* compare: error status $$$$st" '; \
+ printf '>> "$$$$bad_compare.$$$$$$$$"; fi\n'; \
+ for file in $${files} ${extra-compare}; do \
+ if test ! -f $$r/stage3-$$file; then continue; fi; \
+ echo "all: compare3/$$file"; \
+ done; \
+ } > "$$compare_makefile"; \
+ $(MAKE) -s -f "$$compare_makefile" all; compare_status=$$?; \
+ if test $$compare_status -ne 0; then \
+ exit $$compare_status; \
+ fi; \
+ set -- "$$bad_compare".*; \
+ if test -f "$$1"; then \
echo "Bootstrap comparison failure!"; \
- cat .bad_compare; \
+ LC_ALL=C sort "$$bad_compare".* > "$$bad_compare" || exit 1; \
+ cat "$$bad_compare"; \
+ mv -f "$$bad_compare" .bad_compare; \
exit 1; \
else \
+ rm -f .bad_compare; \
echo Comparison successful.; \
fi; \
$(STAMP) compare3
diff --git a/Makefile.tpl b/Makefile.tpl
index 5891b67b697..98bd03e9a33 100644
--- a/Makefile.tpl
+++ b/Makefile.tpl
@@ -1821,6 +1821,9 @@ do-clean: clean-stage[+id+]
# only possibility, but now it conflicts with no-bootstrap rules
@if gcc-bootstrap
[+ IF compare-target +]
+# Run the comparisons in parallel through a generated sub-makefile under
+# the jobserver. Each failing recipe writes a separate result shard that
+# is collected after the sub-make finishes.
[+compare-target+]:
@r=`${PWD_COMMAND}`; export r; \
s=`cd $(srcdir); ${PWD_COMMAND}`; export s; \
@@ -1829,29 +1832,52 @@ do-clean: clean-stage[+id+]
exit 0; \
fi; \
: $(MAKE); $(stage); \
- rm -f .bad_compare; \
+ compare_id=$$$$; \
+ bad_compare=.bad_compare.$$compare_id; export bad_compare; \
+ compare_makefile=[+compare-target+].$$compare_id.mk; \
+ trap 'st=$$?; rm -f "$$compare_makefile" "$$bad_compare" \
+ "$$bad_compare".*; trap - 0; exit $$st' 0; \
+ trap 'exit 1' 1 2 3 15; \
echo Comparing stages [+prev+] and [+id+]; \
sed=`echo stage[+id+] | sed 's,^stage,,;s,.,.,g'`; \
files=`find stage[+id+]-* -name "*$(objext)" -print | \
sed -n s,^stage$$sed-,,p`; \
- for file in $${files} ${extra-compare}; do \
- f1=$$r/stage[+prev+]-$$file; f2=$$r/stage[+id+]-$$file; \
- if test ! -f $$f1; then continue; fi; \
- $(do-[+compare-target+]) > /dev/null 2>&1; \
- if test $$? -eq 1; then \
- case $$file in \
- @compare_exclusions@) \
- echo warning: $$file differs ;; \
- *) \
- echo $$file differs >> .bad_compare ;; \
- esac; \
- fi; \
- done; \
- if [ -f .bad_compare ]; then \
+ cmp_raw='$(do-[+compare-target+])'; \
+ { \
+ echo 'all:'; \
+ echo '.PHONY: all FORCE'; \
+ echo 'FORCE:'; \
+ printf '[+compare-target+]/%%: FORCE ; @'; \
+ printf 'f1=$$$$r/stage[+prev+]-$$*; '; \
+ printf 'f2=$$$$r/stage[+id+]-$$*; '; \
+ printf '%s' "$$cmp_raw" | sed 's,\$$,$$$$,g'; \
+ printf ' > /dev/null 2>&1; st=$$$$?; '; \
+ printf 'if test $$$$st -eq 1; then '; \
+ printf 'case $$* in '; \
+ printf '@compare_exclusions@) echo warning: $$* differs ;; '; \
+ printf '*) echo $$* differs >> "$$$$bad_compare.$$$$$$$$" ;; '; \
+ printf 'esac; '; \
+ printf 'elif test $$$$st -ne 0; then '; \
+ printf 'echo "$$* compare: error status $$$$st" '; \
+ printf '>> "$$$$bad_compare.$$$$$$$$"; fi\n'; \
+ for file in $${files} ${extra-compare}; do \
+ if test ! -f $$r/stage[+prev+]-$$file; then continue; fi; \
+ echo "all: [+compare-target+]/$$file"; \
+ done; \
+ } > "$$compare_makefile"; \
+ $(MAKE) -s -f "$$compare_makefile" all; compare_status=$$?; \
+ if test $$compare_status -ne 0; then \
+ exit $$compare_status; \
+ fi; \
+ set -- "$$bad_compare".*; \
+ if test -f "$$1"; then \
echo "Bootstrap comparison failure!"; \
- cat .bad_compare; \
+ LC_ALL=C sort "$$bad_compare".* > "$$bad_compare" || exit 1; \
+ cat "$$bad_compare"; \
+ mv -f "$$bad_compare" .bad_compare; \
exit 1; \
else \
+ rm -f .bad_compare; \
echo Comparison successful.; \
fi; \
$(STAMP) [+compare-target+][+ IF prev +]
diff --git a/config/acx.m4 b/config/acx.m4
index ef5468bdd35..5547cbeb9b8 100644
--- a/config/acx.m4
+++ b/config/acx.m4
@@ -477,7 +477,16 @@ AC_DEFUN([ACX_PROG_CMP_IGNORE_INITIAL],
[AC_CACHE_CHECK([how to compare bootstrapped objects], gcc_cv_prog_cmp_skip,
[ echo abfoo >t1
echo cdfoo >t2
- gcc_cv_prog_cmp_skip='tail -c +17 $$f1 > tmp-foo1; tail -c +17 $$f2 >
tmp-foo2; cmp tmp-foo1 tmp-foo2'
+ gcc_cv_prog_cmp_skip='(trap "st=\$$?; rm -f tmp-foo1.$$$$ '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tmp-foo2.$$$$; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'trap - 0; exit \$$st" 0; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'trap "exit 2" 1 2 3 15; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tail -c +17 $$f1 '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'> tmp-foo1.$$$$ || exit 2; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tail -c +17 $$f2 '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'> tmp-foo2.$$$$ || exit 2; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'cmp tmp-foo1.$$$$ '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tmp-foo2.$$$$)'
if cmp t1 t2 2 2 > /dev/null 2>&1; then
if cmp t1 t2 1 1 > /dev/null 2>&1; then
:
diff --git a/configure b/configure
index ad490083741..4db0edbdccd 100755
--- a/configure
+++ b/configure
@@ -6123,7 +6123,16 @@ if ${gcc_cv_prog_cmp_skip+:} false; then :
else
echo abfoo >t1
echo cdfoo >t2
- gcc_cv_prog_cmp_skip='tail -c +17 $$f1 > tmp-foo1; tail -c +17 $$f2 >
tmp-foo2; cmp tmp-foo1 tmp-foo2'
+ gcc_cv_prog_cmp_skip='(trap "st=\$$?; rm -f tmp-foo1.$$$$ '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tmp-foo2.$$$$; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'trap - 0; exit \$$st" 0; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'trap "exit 2" 1 2 3 15; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tail -c +17 $$f1 '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'> tmp-foo1.$$$$ || exit 2; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tail -c +17 $$f2 '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'> tmp-foo2.$$$$ || exit 2; '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'cmp tmp-foo1.$$$$ '
+ gcc_cv_prog_cmp_skip="$gcc_cv_prog_cmp_skip"'tmp-foo2.$$$$)'
if cmp t1 t2 2 2 > /dev/null 2>&1; then
if cmp t1 t2 1 1 > /dev/null 2>&1; then
:
--
2.50.1 (Apple Git-155)
