config_host.mk.in | 1
configure.ac | 4 +
instsetoo_native/CustomTarget_install.mk | 77 ++++++++++++-------------------
solenv/bin/call_installer.sh | 58 +++++++++++++++++++++++
4 files changed, 94 insertions(+), 46 deletions(-)
New commits:
commit ceb75969695d5132facbde8dafe08d21caada8ea
Author: Thorsten Behrens <thorsten.behr...@allotropia.de>
AuthorDate: Fri Apr 29 12:46:58 2022 +0200
Commit: Andras Timar <andras.ti...@collabora.com>
CommitDate: Thu Jun 9 12:31:02 2022 +0200
Get make_installer calls to run in parallel
If available, use GNU parallel to run N make_installer.pl scripts in
parallel, to scale packaging LibreOffice up with the rest of gbuild.
* fallback if no GNU parallel found - run make_installer sequentially
as before
* push most of the make_installer.pl input param tweaks from gbuild
down into a shared call_installer.sh script
* call gnu parallel with generated number of
"templ:lang:prodname:ext:pkgfmt:strip-flag" tuples, one for each
package to build (empty templ for non-windows, to save on cmd line
length)
* such that we can run all those in parallel (taking into account
the build's PARALLELISM parameter)
* there's still the main package build running epm sequentially for
umpteen sub-packages from within _one_ make_installer.pl instance, but
that's much harder to parallelize from inside Perl (so we punt on
that here)
Change-Id: Ie7d3084ed60d003d587c5e64dc9fb1809b23e409
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/133957
Tested-by: Jenkins
Reviewed-by: Thorsten Behrens <thorsten.behr...@allotropia.de>
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/135346
Tested-by: Jenkins CollaboraOffice <jenkinscollaboraoff...@gmail.com>
Reviewed-by: Andras Timar <andras.ti...@collabora.com>
diff --git a/config_host.mk.in b/config_host.mk.in
index f4edd960ee09..73b367290f65 100644
--- a/config_host.mk.in
+++ b/config_host.mk.in
@@ -240,6 +240,7 @@ export GIO_LIBS=$(gb_SPACE)@GIO_LIBS@
export GIT_REFERENCE_SRC=@GIT_REFERENCE_SRC@
export GIT_LINK_SRC=@GIT_LINK_SRC@
export GIT_NEEDED_SUBMODULES=@GIT_NEEDED_SUBMODULES@
+export GNUPARALLEL=@GNUPARALLEL@
export EPOXY_CFLAGS=$(gb_SPACE)@EPOXY_CFLAGS@
export EPOXY_LIBS=$(gb_SPACE)@EPOXY_LIBS@
export GLM_CFLAGS=$(gb_SPACE)@GLM_CFLAGS@
diff --git a/configure.ac b/configure.ac
index 9a77f638157b..93319018fa4d 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2912,6 +2912,10 @@ if test -z "$COMPRESSIONTOOL"; then
fi
AC_SUBST(COMPRESSIONTOOL)
+# use GNU parallel for packaging, if available
+AC_PATH_PROG(GNUPARALLEL, parallel)
+AC_SUBST(GNUPARALLEL)
+
AC_MSG_CHECKING([for GNU or BSD tar])
for a in $GNUTAR gtar gnutar bsdtar tar /usr/sfw/bin/gtar; do
$a --version 2> /dev/null | egrep "GNU|bsdtar" 2>&1 > /dev/null
diff --git a/instsetoo_native/CustomTarget_install.mk
b/instsetoo_native/CustomTarget_install.mk
index 53c79e102bde..bb34a34450d7 100644
--- a/instsetoo_native/CustomTarget_install.mk
+++ b/instsetoo_native/CustomTarget_install.mk
@@ -62,39 +62,18 @@ export LOCAL_COMMON_OUT := $(instsetoo_OUT)
instsetoo_native_WITH_LANG := en-US $(filter-out en-US,$(gb_WITH_LANG))
-ifeq (WNT,$(OS))
-define instsetoo_native_msitemplates
-
-TEMPLATE_DIR=$(dir $@)msi_templates \
-&& rm -rf $${TEMPLATE_DIR} \
-&& mkdir -p $${TEMPLATE_DIR}/Binary \
-&& for I in $(SRCDIR)/instsetoo_native/inc_$(1)/windows/msi_templates/*.* ; do
$(GREP) -v '^#' "$$I" > $${TEMPLATE_DIR}/`basename $$I` || true ; done \
-&& $(GNUCOPY)
$(SRCDIR)/instsetoo_native/inc_common/windows/msi_templates/Binary/*.*
$${TEMPLATE_DIR}/Binary
-endef
-else
-instsetoo_native_msitemplates :=
-endif
-
define instsetoo_native_install_command
-$(call instsetoo_native_msitemplates,$(1))
-$(call gb_Helper_print_on_error, \
-cd $(dir $@) \
-$(foreach pkgformat,$(5),\
-&& $(PERL) -w $< \
- -f $(BUILDDIR)/instsetoo_native/util/openoffice.lst \
- -l $(subst $(WHITESPACE),$(COMMA),$(strip $(2))) \
- -p $(PRODUCTNAME_WITHOUT_SPACES)$(3) \
- -u $(instsetoo_OUT) \
- -packer $(COMPRESSIONTOOL) \
- -buildid $(if $(filter deb0
rpm0,$(pkgformat)$(LIBO_VERSION_PATCH)),1,$(LIBO_VERSION_PATCH)) \
- $(if $(filter WNT,$(OS)), \
- -msitemplate $(dir $@)msi_templates \
- -msilanguage $(dir $@)win_ulffiles \
- ) \
- $(4) \
- -format $(pkgformat) \
- $(if $(verbose),-verbose,-quiet) \
-),$@.log)
+$(if $(GNUPARALLEL), \
+ $(call gb_Helper_print_on_error, \
+ cd $(dir $@) && \
+ $(GNUPARALLEL) -j $(PARALLELISM) $(SRCDIR)/solenv/bin/call_installer.sh
$(if $(verbose),-verbose,-quiet) -- $(1) \
+ ,$@.log) \
+, \
+ $(call gb_Helper_print_on_error, \
+ cd $(dir $@) \
+ $(foreach curpkg,$(1),\
+ && $(SRCDIR)/solenv/bin/call_installer.sh $(if $(verbose),-verbose,-quiet)
$(curpkg) \
+ ),$@.log))
endef
$(call gb_CustomTarget_get_workdir,instsetoo_native/install)/install.phony:
@@ -102,13 +81,13 @@ $(call
gb_CustomTarget_get_workdir,instsetoo_native/install)/install.phony:
$(call gb_Trace_StartRange,$(subst $(WORKDIR)/,,$@),PRL)
rm -rf $(instsetoo_OUT)
ifeq (TRUE,$(LIBO_TEST_INSTALL))
- $(call instsetoo_native_install_command,openoffice,en-US,,,archive)
+ $(call instsetoo_native_install_command,
"openoffice:en-US:::archive:nostrip")
unzip -q -d $(TESTINSTALLDIR)
$(instsetoo_OUT)/$(PRODUCTNAME_WITHOUT_SPACES)/archive/install/en-US/LibreOffice*_archive.zip
mv $(TESTINSTALLDIR)/LibreOffice*_archive/LibreOffice*/*
$(TESTINSTALLDIR)/
rmdir $(TESTINSTALLDIR)/LibreOffice*_archive/LibreOffice*
rmdir $(TESTINSTALLDIR)/LibreOffice*_archive
ifeq (ODK,$(filter ODK,$(BUILD_TYPE)))
- $(call instsetoo_native_install_command,sdkoo,en-US,_SDK,,archive)
+ $(call instsetoo_native_install_command,
"sdkoo:en-US:_SDK::archive:nostrip")
unzip -q -d $(TESTINSTALLDIR)
$(instsetoo_OUT)/$(PRODUCTNAME_WITHOUT_SPACES)_SDK/archive/install/en-US/LibreOffice*_archive_sdk.zip
mv $(TESTINSTALLDIR)/LibreOffice*_archive_sdk/LibreOffice*_SDK/sdk \
$(TESTINSTALLDIR)/
@@ -116,18 +95,24 @@ ifeq (ODK,$(filter ODK,$(BUILD_TYPE)))
rmdir $(TESTINSTALLDIR)/LibreOffice*_archive_sdk
endif
else # LIBO_TEST_INSTALL
- $(call instsetoo_native_install_command,openoffice,$(if $(filter WNT
MACOSX,$(OS)),$(instsetoo_native_WITH_LANG),en-US),,,$(PKGFORMAT))
-ifeq (ODK,$(filter ODK,$(BUILD_TYPE)))
- $(call instsetoo_native_install_command,sdkoo,en-US,_SDK,,$(PKGFORMAT))
-endif
-ifeq (HELP,$(filter HELP,$(BUILD_TYPE))$(filter MACOSX,$(OS)))
- $(foreach lang,$(gb_HELP_LANGS),\
- $(call
instsetoo_native_install_command,ooohelppack,$(lang),,-helppack,$(PKGFORMAT)))
-endif
-ifeq (,$(filter WNT MACOSX,$(OS)))
- $(foreach lang,$(instsetoo_native_WITH_LANG),\
- $(call
instsetoo_native_install_command,ooolangpack,$(lang),,-languagepack,$(PKGFORMAT)))
-endif
+ $(call instsetoo_native_install_command, \
+ $(foreach pkgformat,$(PKGFORMAT),\
+ $(if $(filter WNT,$(OS)), \
+ "openoffice:$(subst
$(WHITESPACE),$(COMMA),$(strip
$(instsetoo_native_WITH_LANG))):::$(pkgformat):$(if $(filter-out
archive,$(pkgformat)),strip,nostrip)" \
+ $(if $(filter ODK,$(BUILD_TYPE)), \
+
"sdkoo:en-US:_SDK::$(pkgformat):nostrip") \
+ $(if $(filter HELP,$(BUILD_TYPE)), \
+ $(foreach lang,$(gb_HELP_LANGS), \
+
"ooohelppack:$(lang)::-helppack:$(pkgformat):nostrip" )) \
+ , \
+ ":en-US:::$(pkgformat):$(if $(filter-out
archive,$(pkgformat)),strip,nostrip)" \
+ $(if $(filter ODK,$(BUILD_TYPE)), \
+ ":en-US:_SDK::$(pkgformat):nostrip") \
+ $(if $(and $(filter HELP,$(BUILD_TYPE)),
$(filter-out MACOSX,$(OS))), \
+ $(foreach lang,$(gb_HELP_LANGS), \
+
":$(lang)::-helppack:$(pkgformat):nostrip" )) \
+ $(foreach lang,$(instsetoo_native_WITH_LANG), \
+
":$(lang)::-languagepack:$(pkgformat):nostrip" ) )))
endif # LIBO_TEST_INSTALL
touch $@
$(call gb_Trace_EndRange,$(subst $(WORKDIR)/,,$@),PRL)
diff --git a/solenv/bin/call_installer.sh b/solenv/bin/call_installer.sh
new file mode 100755
index 000000000000..f1db809f7faf
--- /dev/null
+++ b/solenv/bin/call_installer.sh
@@ -0,0 +1,58 @@
+#!/bin/sh
+
+# unpack parameters
+VERBOSITY=$1; shift
+MSITEMPL=$(echo "$@" | cut -d ':' -f 1)
+LANG=$(echo "$@" | cut -d ':' -f 2)
+PRODNAME=$(echo "$@" | cut -d ':' -f 3)
+EXTENSION=$(echo "$@" | cut -d ':' -f 4)
+PKGFORMAT=$(echo "$@" | cut -d ':' -f 5)
+STRIP=$(echo "$@" | cut -d ':' -f 6)
+
+# need to hack buildid?
+if [ "${PKGFORMAT}${LIBO_VERSION_PATCH}" = "deb0" ] || \
+ [ "${PKGFORMAT}${LIBO_VERSION_PATCH}" = "rpm0" ] ; then
+ LIBO_VERSION_PATCH=1
+fi
+
+# switch to verbose?
+if [ "${VERBOSITY}" = "-verbose" ] ; then
+ set -x
+fi
+
+# populate MSI template dirs for Windows
+if [ -n "${MSITEMPL}" ]; then
+
TEMPLATE_DIR="${WORKDIR}/CustomTarget/instsetoo_native/install/msi_templates"
+ rm -rf "${TEMPLATE_DIR}" && \
+ mkdir -p "${TEMPLATE_DIR}/Binary" && \
+ for I in
"${SRCDIR}/instsetoo_native/inc_${MSITEMPL}/windows/msi_templates/"*.* ; do \
+ "${GREP}" -v '^#' "$I" > "${TEMPLATE_DIR}/$(basename "$I")" || true ; \
+ done && \
+ "${GNUCOPY}"
"${SRCDIR}/instsetoo_native/inc_common/windows/msi_templates/Binary/"*.*
"${TEMPLATE_DIR}/Binary" || exit 1
+fi
+
+# add extra params for Windows
+EXTRA_PARAMS=
+if [ "${OS}" = "WNT" ] ; then
+ EXTRA_PARAMS="${EXTRA_PARAMS} -msitemplate
${WORKDIR}/CustomTarget/instsetoo_native/install/msi_templates"
+ EXTRA_PARAMS="${EXTRA_PARAMS} -msilanguage
${WORKDIR}/CustomTarget/instsetoo_native/install/win_ulffiles"
+fi
+
+# need to strip?
+if [ "${STRIP}" = "strip" ] ; then
+ export ENABLE_STRIP=1
+fi
+
+# shellcheck disable=SC2086
+# shellcheck disable=SC2154
+${PERL} -w "${SRCDIR}"/solenv/bin/make_installer.pl \
+ -f "${BUILDDIR}"/instsetoo_native/util/openoffice.lst \
+ -l "${LANG}" \
+ -p "${PRODUCTNAME_WITHOUT_SPACES}${PRODNAME}" \
+ -u "${instsetoo_OUT}" \
+ -packer "${COMPRESSIONTOOL}" \
+ -buildid "${LIBO_VERSION_PATCH}" \
+ ${EXTRA_PARAMS:+$EXTRA_PARAMS} \
+ ${EXTENSION:+"$EXTENSION"} \
+ -format "${PKGFORMAT}" \
+ "${VERBOSITY}"
