On 23.06.21 10:55, Peter Eisentraut wrote:
v1-0001-Make-Unicode-makefile-more-parallel-safe.patch
The makefile rule that calls UCS_to_most.pl was written incorrectly for
parallel make. The script writes all output files in one go, but the
rule as written would call the command once for each output file in
parallel.
This could use a comment. At a quick glance, I don't understand what
all the $(wordlist ...) magic does.
Perhaps we should change the script or Makefile so that it doesn't
create all the maps in one go?
I agree, either comment it better or just write one file at a time. I'll
take another look at that.
Here is a patch that does it one file (pair) at a time. The other rules
besides UCS_to_most.pl actually had the same problem, since they produce
two output files, so running in parallel called each script twice. In
this patch, all of that is heavily refactored and works correctly now.
Note that UCS_to_most.pl already accepted a command-line argument to
specify which encoding to work with.
From 974720b0b6c92f42506ae37d8e88368ba279b973 Mon Sep 17 00:00:00 2001
From: Peter Eisentraut <pe...@eisentraut.org>
Date: Tue, 20 Jul 2021 13:47:13 +0200
Subject: [PATCH] Make Unicode makefile parallel-safe
Fix the rules so that each rule is parallel safe, using the same
trickery that we use elsewhere in the tree for rules that produce more
than one output file. Refactor the whole makefile so that there is
less repetition.
---
src/backend/utils/mb/Unicode/Makefile | 134 +++++++++-----------------
1 file changed, 45 insertions(+), 89 deletions(-)
diff --git a/src/backend/utils/mb/Unicode/Makefile
b/src/backend/utils/mb/Unicode/Makefile
index ed6fc07e08..cdcde4fbbd 100644
--- a/src/backend/utils/mb/Unicode/Makefile
+++ b/src/backend/utils/mb/Unicode/Makefile
@@ -12,101 +12,57 @@ subdir = src/backend/utils/mb/Unicode
top_builddir = ../../../../..
include $(top_builddir)/src/Makefile.global
-ISO8859MAPS = iso8859_2_to_utf8.map utf8_to_iso8859_2.map \
- iso8859_3_to_utf8.map utf8_to_iso8859_3.map \
- iso8859_4_to_utf8.map utf8_to_iso8859_4.map \
- iso8859_5_to_utf8.map utf8_to_iso8859_5.map \
- iso8859_6_to_utf8.map utf8_to_iso8859_6.map \
- iso8859_7_to_utf8.map utf8_to_iso8859_7.map \
- iso8859_8_to_utf8.map utf8_to_iso8859_8.map \
- iso8859_9_to_utf8.map utf8_to_iso8859_9.map \
- iso8859_10_to_utf8.map utf8_to_iso8859_10.map \
- iso8859_13_to_utf8.map utf8_to_iso8859_13.map \
- iso8859_14_to_utf8.map utf8_to_iso8859_14.map \
- iso8859_15_to_utf8.map utf8_to_iso8859_15.map \
- iso8859_16_to_utf8.map utf8_to_iso8859_16.map
-
-WINMAPS = win866_to_utf8.map utf8_to_win866.map \
- win874_to_utf8.map utf8_to_win874.map \
- win1250_to_utf8.map utf8_to_win1250.map \
- win1251_to_utf8.map utf8_to_win1251.map \
- win1252_to_utf8.map utf8_to_win1252.map \
- win1253_to_utf8.map utf8_to_win1253.map \
- win1254_to_utf8.map utf8_to_win1254.map \
- win1255_to_utf8.map utf8_to_win1255.map \
- win1256_to_utf8.map utf8_to_win1256.map \
- win1257_to_utf8.map utf8_to_win1257.map \
- win1258_to_utf8.map utf8_to_win1258.map
-
-GENERICMAPS = $(ISO8859MAPS) $(WINMAPS) \
- gbk_to_utf8.map utf8_to_gbk.map \
- koi8r_to_utf8.map utf8_to_koi8r.map \
- koi8u_to_utf8.map utf8_to_koi8u.map
-
-SPECIALMAPS = euc_cn_to_utf8.map utf8_to_euc_cn.map \
- euc_jp_to_utf8.map utf8_to_euc_jp.map \
- euc_kr_to_utf8.map utf8_to_euc_kr.map \
- euc_tw_to_utf8.map utf8_to_euc_tw.map \
- sjis_to_utf8.map utf8_to_sjis.map \
- gb18030_to_utf8.map utf8_to_gb18030.map \
- big5_to_utf8.map utf8_to_big5.map \
- johab_to_utf8.map utf8_to_johab.map \
- uhc_to_utf8.map utf8_to_uhc.map \
- euc_jis_2004_to_utf8.map utf8_to_euc_jis_2004.map \
- shift_jis_2004_to_utf8.map utf8_to_shift_jis_2004.map
-
-MAPS = $(GENERICMAPS) $(SPECIALMAPS)
-
-ISO8859TEXTS = 8859-2.TXT 8859-3.TXT 8859-4.TXT 8859-5.TXT \
- 8859-6.TXT 8859-7.TXT 8859-8.TXT 8859-9.TXT \
- 8859-10.TXT 8859-13.TXT 8859-14.TXT 8859-15.TXT \
- 8859-16.TXT
-
-WINTEXTS = CP866.TXT CP874.TXT CP936.TXT \
- CP1250.TXT CP1251.TXT \
- CP1252.TXT CP1253.TXT CP1254.TXT CP1255.TXT \
- CP1256.TXT CP1257.TXT CP1258.TXT
-
-GENERICTEXTS = $(ISO8859TEXTS) $(WINTEXTS) \
- KOI8-R.TXT KOI8-U.TXT
-all: $(MAPS)
-
-$(GENERICMAPS): UCS_to_most.pl $(GENERICTEXTS)
- $(PERL) -I $(srcdir) $<
-
-johab_to_utf8.map utf8_to_johab.map: UCS_to_JOHAB.pl JOHAB.TXT
- $(PERL) -I $(srcdir) $<
-
-uhc_to_utf8.map utf8_to_uhc.map: UCS_to_UHC.pl windows-949-2000.xml
- $(PERL) -I $(srcdir) $<
-
-euc_jp_to_utf8.map utf8_to_euc_jp.map: UCS_to_EUC_JP.pl CP932.TXT JIS0212.TXT
- $(PERL) -I $(srcdir) $<
+# Define a rule to create to map files from downloaded text input
+# files using a script. Arguments:
+#
+# 1: encoding name used in output files (lower case)
+# 2: script name
+# 3: input text files
+# 4: argument to pass to script (optional)
+#
+# We also collect all the input and output files in variables to
+# define the build and clean rules below.
+#
+# Note that while each script call produces two output files, to be
+# parallel-make safe we need to split this into two rules. (See for
+# example gram.y for more explanation.)
+#
+define map_rule
+MAPS += $(1)_to_utf8.map utf8_to_$(1).map
+ALL_TEXTS += $(3)
-euc_cn_to_utf8.map utf8_to_euc_cn.map: UCS_to_EUC_CN.pl gb-18030-2000.xml
- $(PERL) -I $(srcdir) $<
+$(1)_to_utf8.map: $(2) $(3)
+ $(PERL) -I $$(srcdir) $$< $(4)
-euc_kr_to_utf8.map utf8_to_euc_kr.map: UCS_to_EUC_KR.pl KSX1001.TXT
- $(PERL) -I $(srcdir) $<
+utf8_to_$(1).map: $(1)_to_utf8.map
+ @touch $$@
+endef
-euc_tw_to_utf8.map utf8_to_euc_tw.map: UCS_to_EUC_TW.pl CNS11643.TXT
- $(PERL) -I $(srcdir) $<
+$(foreach n,2 3 4 5 6 7 8 9 10 13 14 15 16,$(eval $(call
map_rule,iso8859_$(n),UCS_to_most.pl,8859-$(n).TXT,ISO8859_$(n))))
-sjis_to_utf8.map utf8_to_sjis.map: UCS_to_SJIS.pl CP932.TXT
- $(PERL) -I $(srcdir) $<
+$(foreach n,866 874 1250 1251 1252 1253 1254 1255 1256 1257 1258,$(eval $(call
map_rule,win$(n),UCS_to_most.pl,CP$(n).TXT,WIN$(n))))
-gb18030_to_utf8.map utf8_to_gb18030.map: UCS_to_GB18030.pl gb-18030-2000.xml
- $(PERL) -I $(srcdir) $<
+$(eval $(call map_rule,koi8r,UCS_to_most.pl,KOI8-R.TXT,KOI8R))
+$(eval $(call map_rule,koi8u,UCS_to_most.pl,KOI8-U.TXT,KOI8U))
+$(eval $(call map_rule,gbk,UCS_to_most.pl,CP936.TXT,GBK))
-big5_to_utf8.map utf8_to_big5.map: UCS_to_BIG5.pl BIG5.TXT CP950.TXT
- $(PERL) -I $(srcdir) $<
+$(eval $(call map_rule,johab,UCS_to_JOHAB.pl,JOHAB.TXT))
+$(eval $(call map_rule,uhc,UCS_to_UHC.pl,windows-949-2000.xml))
+$(eval $(call map_rule,euc_jp,UCS_to_EUC_JP.pl,CP932.TXT JIS0212.TXT))
+$(eval $(call map_rule,euc_cn,UCS_to_EUC_CN.pl,gb-18030-2000.xml))
+$(eval $(call map_rule,euc_kr,UCS_to_EUC_KR.pl,KSX1001.TXT))
+$(eval $(call map_rule,euc_tw,UCS_to_EUC_TW.pl,CNS11643.TXT))
+$(eval $(call map_rule,sjis,UCS_to_SJIS.pl,CP932.TXT))
+$(eval $(call map_rule,gb18030,UCS_to_GB18030.pl,gb-18030-2000.xml))
+$(eval $(call map_rule,big5,UCS_to_BIG5.pl,CP950.TXT BIG5.TXT CP950.TXT))
+$(eval $(call
map_rule,euc_jis_2004,UCS_to_EUC_JIS_2004.pl,euc-jis-2004-std.txt))
+$(eval $(call
map_rule,shift_jis_2004,UCS_to_SHIFT_JIS_2004.pl,sjis-0213-2004-std.txt))
-euc_jis_2004_to_utf8.map utf8_to_euc_jis_2004.map: UCS_to_EUC_JIS_2004.pl
euc-jis-2004-std.txt
- $(PERL) -I $(srcdir) $<
+# remove duplicates
+TEXTS = $(sort ALL_TEXTS)
-shift_jis_2004_to_utf8.map utf8_to_shift_jis_2004.map:
UCS_to_SHIFT_JIS_2004.pl sjis-0213-2004-std.txt
- $(PERL) -I $(srcdir) $<
+all: $(MAPS)
distclean: clean
rm -f $(TEXTS)
@@ -136,11 +92,11 @@ JOHAB.TXT KSX1001.TXT:
KOI8-R.TXT KOI8-U.TXT:
$(DOWNLOAD) https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/$(@F)
-$(ISO8859TEXTS):
+$(filter 8859-%.TXT,$(TEXTS)):
$(DOWNLOAD) https://www.unicode.org/Public/MAPPINGS/ISO8859/$(@F)
-$(filter-out CP8%,$(WINTEXTS)) CP932.TXT CP950.TXT:
+$(filter CP9%.TXT CP12%.TXT,$(TEXTS)):
$(DOWNLOAD)
https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/$(@F)
-$(filter CP8%,$(WINTEXTS)):
+$(filter CP8%.TXT,$(TEXTS)):
$(DOWNLOAD)
https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/$(@F)
--
2.32.0
