Makefile.fetch | 355 ++++++++++++++++++++++++++-------------------------- bin/cached-fetch.sh | 71 ++++++++++ 2 files changed, 253 insertions(+), 173 deletions(-)
New commits: commit 699addd6517f6ed5fcbf69d66b6bf7944bd8dc66 Author: Andras Timar <[email protected]> AuthorDate: Mon Feb 16 16:35:31 2026 +0100 Commit: Andras Timar <[email protected]> CommitDate: Mon Feb 16 18:04:26 2026 +0100 Add fetch-names target and cached-fetch.sh for CI tarball caching Extract tarball lists in Makefile.fetch into shared variables and add a fetch-names target that prints the resolved filenames needed for the current build configuration. Add bin/cached-fetch.sh which uses a persistent cache directory to avoid re-downloading ~500MB of external tarballs on every CI source tarball build. On second run, all tarballs are restored from cache and make fetch downloads nothing. Change-Id: I865cefc6189d123257bb6d3888c4774facc36f04 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/199491 Tested-by: Jenkins CollaboraOffice <[email protected]> Reviewed-by: Andras Timar <[email protected]> diff --git a/Makefile.fetch b/Makefile.fetch index 16296c6871bc..162fc3043d8c 100644 --- a/Makefile.fetch +++ b/Makefile.fetch @@ -94,184 +94,193 @@ define fetch_Optional_pack $(if $(strip $($(1))),$(1)) endef +fetch_SRC_TARBALLS := \ + $(call fetch_Optional,ABW,ABW_TARBALL) \ + $(call fetch_Optional,AFDKO,AFDKO_TARBALL) \ + $(call fetch_Optional,AFDKO,ANTLR4CPPRUNTIME_TARBALL) \ + $(call fetch_Optional,ARGON2,ARGON2_TARBALL) \ + $(call fetch_Optional,BOOST,BOOST_TARBALL) \ + $(call fetch_Optional,BOX2D,BOX2D_TARBALL) \ + $(call fetch_Optional,BREAKPAD,BREAKPAD_TARBALL) \ + $(call fetch_Optional,BSH,BSH_TARBALL) \ + $(call fetch_Optional,BZIP2,BZIP2_TARBALL) \ + $(call fetch_Optional,CAIRO,CAIRO_TARBALL) \ + $(call fetch_Optional,CAIRO,PIXMAN_TARBALL) \ + $(call fetch_Optional,CDR,CDR_TARBALL) \ + $(call fetch_Optional,CLUCENE,CLUCENE_TARBALL) \ + $(call fetch_Optional,DRAGONBOX,DRAGONBOX_TARBALL) \ + $(call fetch_Optional,FAST_FLOAT,FAST_FLOAT_TARBALL) \ + $(call fetch_Optional,LIBCMIS,LIBCMIS_TARBALL) \ + $(call fetch_Optional,COINMP,COINMP_TARBALL) \ + $(call fetch_Optional,CPPUNIT,CPPUNIT_TARBALL) \ + $(call fetch_Optional,CURL,CURL_TARBALL) \ + $(call fetch_Optional,EBOOK,EBOOK_TARBALL) \ + $(call fetch_Optional,EPM,EPM_TARBALL) \ + $(call fetch_Optional,EPUBGEN,EPUBGEN_TARBALL) \ + $(call fetch_Optional,ETONYEK,ETONYEK_TARBALL) \ + $(call fetch_Optional,EXPAT,EXPAT_TARBALL) \ + $(call fetch_Optional,FIREBIRD,FIREBIRD_TARBALL) \ + $(call fetch_Optional,FONTCONFIG,FONTCONFIG_TARBALL) \ + $(call fetch_Optional,FREEHAND,FREEHAND_TARBALL) \ + $(call fetch_Optional,FREETYPE,FREETYPE_TARBALL) \ + $(call fetch_Optional,FROZEN,FROZEN_TARBALL) \ + $(call fetch_Optional,EPOXY,EPOXY_TARBALL) \ + $(call fetch_Optional,GLM,GLM_TARBALL) \ + $(call fetch_Optional,GPGMEPP,GPGME_TARBALL) \ + $(call fetch_Optional,GRAPHITE,GRAPHITE_TARBALL) \ + $(call fetch_Optional,HARFBUZZ,HARFBUZZ_TARBALL) \ + $(call fetch_Optional,HSQLDB,HSQLDB_TARBALL) \ + $(call fetch_Optional,HUNSPELL,HUNSPELL_TARBALL) \ + $(call fetch_Optional,HYPHEN,HYPHEN_TARBALL) \ + $(if $(filter WNT,$(OS)),IACCESSIBLE2_TARBALL) \ + $(call fetch_Optional,ICU,ICU_TARBALL) \ + $(call fetch_Optional,ICU,ICU_DATA_TARBALL) \ + $(call fetch_Optional,JAVA_WEBSOCKET,JAVA_WEBSOCKET_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_FLOW_ENGINE_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_FLUTE_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBBASE_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBFONTS_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBFORMULA_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBLAYOUT_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBLOADER_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBREPOSITORY_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBSERIALIZER_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBXML_TARBALL) \ + $(call fetch_Optional,JFREEREPORT,JFREEREPORT_SAC_TARBALL) \ + $(call fetch_Optional,LIBJPEG_TURBO,LIBJPEG_TURBO_TARBALL) \ + $(call fetch_Optional,LCMS2,LCMS2_TARBALL) \ + $(call fetch_Optional,LIBATOMIC_OPS,LIBATOMIC_OPS_TARBALL) \ + $(call fetch_Optional,LIBASSUAN,LIBASSUAN_TARBALL) \ + $(call fetch_Optional,LIBEOT,LIBEOT_TARBALL) \ + $(call fetch_Optional,LIBEXTTEXTCAT,LIBEXTTEXTCAT_TARBALL) \ + $(call fetch_Optional,LIBFFI,LIBFFI_TARBALL) \ + $(call fetch_Optional,LIBGPGERROR,LIBGPGERROR_TARBALL) \ + $(call fetch_Optional,LIBLANGTAG,LANGTAGREG_TARBALL) \ + $(call fetch_Optional,LIBLANGTAG,LIBLANGTAG_TARBALL) \ + $(call fetch_Optional,LIBNUMBERTEXT,LIBNUMBERTEXT_TARBALL) \ + $(call fetch_Optional,LIBPNG,LIBPNG_TARBALL) \ + $(call fetch_Optional,LIBTIFF,LIBTIFF_TARBALL) \ + $(call fetch_Optional,LIBTOMMATH,LIBTOMMATH_TARBALL) \ + $(call fetch_Optional,LIBWEBP,LIBWEBP_TARBALL) \ + $(call fetch_Optional,LIBXML2,LIBXML_TARBALL) \ + $(call fetch_Optional,XMLSEC,XMLSEC_TARBALL) \ + $(call fetch_Optional,LIBXSLT,LIBXSLT_TARBALL) \ + $(call fetch_Optional,LPSOLVE,LPSOLVE_TARBALL) \ + $(call fetch_Optional,LXML,LXML_TARBALL) \ + $(call fetch_Optional,LZMA,LZMA_TARBALL) \ + $(call fetch_Optional,MARIADB_CONNECTOR_C,MARIADB_CONNECTOR_C_TARBALL) \ + $(call fetch_Optional,MD4C,MD4C_TARBALL) \ + $(call fetch_Optional,MDDS,MDDS_TARBALL) \ + $(call fetch_Optional,ZXCVBN,ZXCVBN_C_TARBALL) \ + $(call fetch_Optional,ZXING,ZXING_TARBALL) \ + $(call fetch_Optional,MDNSRESPONDER,MDNSRESPONDER_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_CALADEA_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_CARLITO_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_DEJAVU_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_GENTIUM_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_KARLA_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_LIBERATION_NARROW_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_LIBERATION_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_LINLIBERTINEG_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_SOURCESANS_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_KUFI_ARABIC_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_ARABIC_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_NASKH_ARABIC_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_HEBREW_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_HEBREW_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_ARMENIAN_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_ARMENIAN_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_GEORGIAN_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_GEORGIAN_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_LAO_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_LAO_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_LISU_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_CULMUS_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_LIBRE_HEBREW_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_ALEF_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_AMIRI_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_REEM_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_SCHEHERAZADE_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_OPEN_SANS_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_AGDASIMA_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_BACASIME_ANTIQUE_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_BELANOSIMA_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_CAPRASIMO_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_LUGRASIMO_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_LUMANOSIMO_TARBALL) \ + $(call fetch_Optional,MORE_FONTS,FONT_LUNASIMA_TARBALL) \ + $(call fetch_Optional,MESON,MESON_TARBALL) \ + $(call fetch_Optional,MSPUB,MSPUB_TARBALL) \ + $(call fetch_Optional,MWAW,MWAW_TARBALL) \ + $(call fetch_Optional,MYTHES,MYTHES_TARBALL) \ + $(call fetch_Optional,NSS,NSS_TARBALL) \ + $(call fetch_Optional,ODFGEN,ODFGEN_TARBALL) \ + $(call fetch_Optional,ONLINEUPDATE,ONLINEUPDATE_TARBALL) \ + $(call fetch_Optional,OPENLDAP,OPENLDAP_TARBALL) \ + $(call fetch_Optional,OPENSSL,OPENSSL_TARBALL) \ + $(call fetch_Optional,ORCUS,ORCUS_TARBALL) \ + $(call fetch_Optional,PAGEMAKER,PAGEMAKER_TARBALL) \ + $(call fetch_Optional,PDFIUM,PDFIUM_TARBALL) \ + $(call fetch_Optional,POPPLER,POPPLER_TARBALL) \ + $(call fetch_Optional,POPPLER,POPPLER_DATA_TARBALL) \ + $(call fetch_Optional,POSTGRESQL,POSTGRESQL_TARBALL) \ + $(call fetch_Optional,PYTHON,PYTHON_TARBALL) \ + $(if $(filter WNT,$(OS)),PYTHON_BOOTSTRAP_TARBALL) \ + $(call fetch_Optional,QXP,QXP_TARBALL) \ + $(call fetch_Optional,REDLAND,RAPTOR_TARBALL) \ + $(call fetch_Optional,REDLAND,RASQAL_TARBALL) \ + $(call fetch_Optional,REDLAND,REDLAND_TARBALL) \ + $(call fetch_Optional,REVENGE,REVENGE_TARBALL) \ + $(call fetch_Optional,RHINO,RHINO_TARBALL) \ + $(call fetch_Optional,SKIA,SKIA_TARBALL) \ + $(call fetch_Optional,SQLITE3,SQLITE3_TARBALL) \ + $(call fetch_Optional,STAROFFICE,STAROFFICE_TARBALL) \ + $(if $(filter WNT,$(OS)),TWAIN_DSM_TARBALL) \ + $(call fetch_Optional,VISIO,VISIO_TARBALL) \ + $(call fetch_Optional,WPD,WPD_TARBALL) \ + $(call fetch_Optional,WPG,WPG_TARBALL) \ + $(call fetch_Optional,WPS,WPS_TARBALL) \ + $(call fetch_Optional,XSLTML,XSLTML_TARBALL) \ + $(call fetch_Optional,YRS,YRS_TARBALL) \ + $(call fetch_Optional,ZLIB,ZLIB_TARBALL) \ + $(call fetch_Optional,ZSTD,ZSTD_TARBALL) \ + $(call fetch_Optional,ZMF,ZMF_TARBALL) + +fetch_EXTERN_TARBALLS := \ + $(call fetch_Optional,OPENSYMBOL,OPENSYMBOL_TTF) \ + $(call fetch_Optional,ODFVALIDATOR,ODFVALIDATOR_JAR) \ + $(call fetch_Optional,OFFICEOTRON,OFFICEOTRON_JAR) + +fetch_NOTO_TARBALLS := \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_MORE_TARBALL) \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SANS_JP_TARBALL) \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SANS_KR_TARBALL) \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SANS_SC_TARBALL) \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SANS_TC_TARBALL) \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SERIF_JP_TARBALL) \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SERIF_KR_TARBALL) \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SERIF_SC_TARBALL) \ + $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SERIF_TC_TARBALL) + +fetch_OPENDYSLEXIC_TARBALLS := \ + $(call fetch_Optional,MORE_FONTS,FONT_OPENDYSLEXIC_TARBALL) + +# Print resolved filenames of all tarballs needed for the current build configuration. +# Used by bin/cached-fetch.sh to populate from cache without re-downloading. +fetch-names: $(BUILDDIR)/config_$(gb_Side).mk + @echo $(foreach item,$(fetch_SRC_TARBALLS) $(fetch_EXTERN_TARBALLS) $(fetch_NOTO_TARBALLS) $(fetch_OPENDYSLEXIC_TARBALLS),$($(item))) + download: $(WORKDIR)/download $(WORKDIR)/download: $(BUILDDIR)/config_$(gb_Side).mk $(SRCDIR)/download.lst $(SRCDIR)/Makefile.fetch -@mkdir -p $(TARFILE_LOCATION)/tmp @date >> $(fetch_LOGFILE) - $(foreach item, \ - $(call fetch_Optional,ABW,ABW_TARBALL) \ - $(call fetch_Optional,AFDKO,AFDKO_TARBALL) \ - $(call fetch_Optional,AFDKO,ANTLR4CPPRUNTIME_TARBALL) \ - $(call fetch_Optional,ARGON2,ARGON2_TARBALL) \ - $(call fetch_Optional,BOOST,BOOST_TARBALL) \ - $(call fetch_Optional,BOX2D,BOX2D_TARBALL) \ - $(call fetch_Optional,BREAKPAD,BREAKPAD_TARBALL) \ - $(call fetch_Optional,BSH,BSH_TARBALL) \ - $(call fetch_Optional,BZIP2,BZIP2_TARBALL) \ - $(call fetch_Optional,CAIRO,CAIRO_TARBALL) \ - $(call fetch_Optional,CAIRO,PIXMAN_TARBALL) \ - $(call fetch_Optional,CDR,CDR_TARBALL) \ - $(call fetch_Optional,CLUCENE,CLUCENE_TARBALL) \ - $(call fetch_Optional,DRAGONBOX,DRAGONBOX_TARBALL) \ - $(call fetch_Optional,FAST_FLOAT,FAST_FLOAT_TARBALL) \ - $(call fetch_Optional,LIBCMIS,LIBCMIS_TARBALL) \ - $(call fetch_Optional,COINMP,COINMP_TARBALL) \ - $(call fetch_Optional,CPPUNIT,CPPUNIT_TARBALL) \ - $(call fetch_Optional,CURL,CURL_TARBALL) \ - $(call fetch_Optional,EBOOK,EBOOK_TARBALL) \ - $(call fetch_Optional,EPM,EPM_TARBALL) \ - $(call fetch_Optional,EPUBGEN,EPUBGEN_TARBALL) \ - $(call fetch_Optional,ETONYEK,ETONYEK_TARBALL) \ - $(call fetch_Optional,EXPAT,EXPAT_TARBALL) \ - $(call fetch_Optional,FIREBIRD,FIREBIRD_TARBALL) \ - $(call fetch_Optional,FONTCONFIG,FONTCONFIG_TARBALL) \ - $(call fetch_Optional,FREEHAND,FREEHAND_TARBALL) \ - $(call fetch_Optional,FREETYPE,FREETYPE_TARBALL) \ - $(call fetch_Optional,FROZEN,FROZEN_TARBALL) \ - $(call fetch_Optional,EPOXY,EPOXY_TARBALL) \ - $(call fetch_Optional,GLM,GLM_TARBALL) \ - $(call fetch_Optional,GPGMEPP,GPGME_TARBALL) \ - $(call fetch_Optional,GRAPHITE,GRAPHITE_TARBALL) \ - $(call fetch_Optional,HARFBUZZ,HARFBUZZ_TARBALL) \ - $(call fetch_Optional,HSQLDB,HSQLDB_TARBALL) \ - $(call fetch_Optional,HUNSPELL,HUNSPELL_TARBALL) \ - $(call fetch_Optional,HYPHEN,HYPHEN_TARBALL) \ - $(if $(filter WNT,$(OS)),IACCESSIBLE2_TARBALL) \ - $(call fetch_Optional,ICU,ICU_TARBALL) \ - $(call fetch_Optional,ICU,ICU_DATA_TARBALL) \ - $(call fetch_Optional,JAVA_WEBSOCKET,JAVA_WEBSOCKET_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_FLOW_ENGINE_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_FLUTE_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBBASE_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBFONTS_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBFORMULA_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBLAYOUT_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBLOADER_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBREPOSITORY_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBSERIALIZER_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_LIBXML_TARBALL) \ - $(call fetch_Optional,JFREEREPORT,JFREEREPORT_SAC_TARBALL) \ - $(call fetch_Optional,LIBJPEG_TURBO,LIBJPEG_TURBO_TARBALL) \ - $(call fetch_Optional,LCMS2,LCMS2_TARBALL) \ - $(call fetch_Optional,LIBATOMIC_OPS,LIBATOMIC_OPS_TARBALL) \ - $(call fetch_Optional,LIBASSUAN,LIBASSUAN_TARBALL) \ - $(call fetch_Optional,LIBEOT,LIBEOT_TARBALL) \ - $(call fetch_Optional,LIBEXTTEXTCAT,LIBEXTTEXTCAT_TARBALL) \ - $(call fetch_Optional,LIBFFI,LIBFFI_TARBALL) \ - $(call fetch_Optional,LIBGPGERROR,LIBGPGERROR_TARBALL) \ - $(call fetch_Optional,LIBLANGTAG,LANGTAGREG_TARBALL) \ - $(call fetch_Optional,LIBLANGTAG,LIBLANGTAG_TARBALL) \ - $(call fetch_Optional,LIBNUMBERTEXT,LIBNUMBERTEXT_TARBALL) \ - $(call fetch_Optional,LIBPNG,LIBPNG_TARBALL) \ - $(call fetch_Optional,LIBTIFF,LIBTIFF_TARBALL) \ - $(call fetch_Optional,LIBTOMMATH,LIBTOMMATH_TARBALL) \ - $(call fetch_Optional,LIBWEBP,LIBWEBP_TARBALL) \ - $(call fetch_Optional,LIBXML2,LIBXML_TARBALL) \ - $(call fetch_Optional,XMLSEC,XMLSEC_TARBALL) \ - $(call fetch_Optional,LIBXSLT,LIBXSLT_TARBALL) \ - $(call fetch_Optional,LPSOLVE,LPSOLVE_TARBALL) \ - $(call fetch_Optional,LXML,LXML_TARBALL) \ - $(call fetch_Optional,LZMA,LZMA_TARBALL) \ - $(call fetch_Optional,MARIADB_CONNECTOR_C,MARIADB_CONNECTOR_C_TARBALL) \ - $(call fetch_Optional,MD4C,MD4C_TARBALL) \ - $(call fetch_Optional,MDDS,MDDS_TARBALL) \ - $(call fetch_Optional,ZXCVBN,ZXCVBN_C_TARBALL) \ - $(call fetch_Optional,ZXING,ZXING_TARBALL) \ - $(call fetch_Optional,MDNSRESPONDER,MDNSRESPONDER_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_CALADEA_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_CARLITO_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_DEJAVU_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_GENTIUM_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_KARLA_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_LIBERATION_NARROW_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_LIBERATION_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_LINLIBERTINEG_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_SOURCESANS_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_KUFI_ARABIC_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_ARABIC_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_NASKH_ARABIC_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_HEBREW_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_HEBREW_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_ARMENIAN_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_ARMENIAN_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_GEORGIAN_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_GEORGIAN_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_LAO_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SERIF_LAO_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_NOTO_SANS_LISU_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_CULMUS_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_LIBRE_HEBREW_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_ALEF_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_AMIRI_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_REEM_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_SCHEHERAZADE_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_OPEN_SANS_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_AGDASIMA_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_BACASIME_ANTIQUE_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_BELANOSIMA_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_CAPRASIMO_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_LUGRASIMO_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_LUMANOSIMO_TARBALL) \ - $(call fetch_Optional,MORE_FONTS,FONT_LUNASIMA_TARBALL) \ - $(call fetch_Optional,MESON,MESON_TARBALL) \ - $(call fetch_Optional,MSPUB,MSPUB_TARBALL) \ - $(call fetch_Optional,MWAW,MWAW_TARBALL) \ - $(call fetch_Optional,MYTHES,MYTHES_TARBALL) \ - $(call fetch_Optional,NSS,NSS_TARBALL) \ - $(call fetch_Optional,ODFGEN,ODFGEN_TARBALL) \ - $(call fetch_Optional,ONLINEUPDATE,ONLINEUPDATE_TARBALL) \ - $(call fetch_Optional,OPENLDAP,OPENLDAP_TARBALL) \ - $(call fetch_Optional,OPENSSL,OPENSSL_TARBALL) \ - $(call fetch_Optional,ORCUS,ORCUS_TARBALL) \ - $(call fetch_Optional,PAGEMAKER,PAGEMAKER_TARBALL) \ - $(call fetch_Optional,PDFIUM,PDFIUM_TARBALL) \ - $(call fetch_Optional,POPPLER,POPPLER_TARBALL) \ - $(call fetch_Optional,POPPLER,POPPLER_DATA_TARBALL) \ - $(call fetch_Optional,POSTGRESQL,POSTGRESQL_TARBALL) \ - $(call fetch_Optional,PYTHON,PYTHON_TARBALL) \ - $(if $(filter WNT,$(OS)),PYTHON_BOOTSTRAP_TARBALL) \ - $(call fetch_Optional,QXP,QXP_TARBALL) \ - $(call fetch_Optional,REDLAND,RAPTOR_TARBALL) \ - $(call fetch_Optional,REDLAND,RASQAL_TARBALL) \ - $(call fetch_Optional,REDLAND,REDLAND_TARBALL) \ - $(call fetch_Optional,REVENGE,REVENGE_TARBALL) \ - $(call fetch_Optional,RHINO,RHINO_TARBALL) \ - $(call fetch_Optional,SKIA,SKIA_TARBALL) \ - $(call fetch_Optional,SQLITE3,SQLITE3_TARBALL) \ - $(call fetch_Optional,STAROFFICE,STAROFFICE_TARBALL) \ - $(if $(filter WNT,$(OS)),TWAIN_DSM_TARBALL) \ - $(call fetch_Optional,VISIO,VISIO_TARBALL) \ - $(call fetch_Optional,WPD,WPD_TARBALL) \ - $(call fetch_Optional,WPG,WPG_TARBALL) \ - $(call fetch_Optional,WPS,WPS_TARBALL) \ - $(call fetch_Optional,XSLTML,XSLTML_TARBALL) \ - $(call fetch_Optional,YRS,YRS_TARBALL) \ - $(call fetch_Optional,ZLIB,ZLIB_TARBALL) \ - $(call fetch_Optional,ZSTD,ZSTD_TARBALL) \ - $(call fetch_Optional,ZMF,ZMF_TARBALL) \ - ,$(call fetch_Download_item,https://dev-www.libreoffice.org/src,$(item))) - $(foreach item, \ - $(call fetch_Optional,OPENSYMBOL,OPENSYMBOL_TTF) \ - $(call fetch_Optional,ODFVALIDATOR,ODFVALIDATOR_JAR) \ - $(call fetch_Optional,OFFICEOTRON,OFFICEOTRON_JAR) \ - ,$(call fetch_Download_item,https://dev-www.libreoffice.org/extern,$(item))) - $(foreach item, \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_MORE_TARBALL) \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SANS_JP_TARBALL) \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SANS_KR_TARBALL) \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SANS_SC_TARBALL) \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SANS_TC_TARBALL) \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SERIF_JP_TARBALL) \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SERIF_KR_TARBALL) \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SERIF_SC_TARBALL) \ - $(call fetch_Optional,NOTO_FONT,FONT_NOTO_SERIF_TC_TARBALL) \ - ,$(call fetch_Download_item,https://github.com/CollaboraOnline/online/releases/download/for-code-assets/,$(item))) - $(foreach item, \ - $(call fetch_Optional,MORE_FONTS,FONT_OPENDYSLEXIC_TARBALL) \ - ,$(call fetch_Download_item,https://github.com/antijingoist/opendyslexic/releases/download/v0.91.12,$(item))) + $(foreach item,$(fetch_SRC_TARBALLS),$(call fetch_Download_item,https://dev-www.libreoffice.org/src,$(item))) + $(foreach item,$(fetch_EXTERN_TARBALLS),$(call fetch_Download_item,https://dev-www.libreoffice.org/extern,$(item))) + $(foreach item,$(fetch_NOTO_TARBALLS),$(call fetch_Download_item,https://github.com/CollaboraOnline/online/releases/download/for-code-assets/,$(item))) + $(foreach item,$(fetch_OPENDYSLEXIC_TARBALLS),$(call fetch_Download_item,https://github.com/antijingoist/opendyslexic/releases/download/v0.91.12,$(item))) -@mkdir -p $(TARFILE_LOCATION)/cargo $(if $(call fetch_Optional,YRS,1),\ CARGO_HOME=$(TARFILE_LOCATION)/cargo cargo fetch $(if $(verbose),--verbose,) --locked --manifest-path $(SRCDIR)/external/y-crdt/Cargo.toml) diff --git a/bin/cached-fetch.sh b/bin/cached-fetch.sh new file mode 100755 index 000000000000..ecdeddba1f2c --- /dev/null +++ b/bin/cached-fetch.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# +# Cached wrapper around `make fetch` for CI source tarball generation. +# +# Instead of `rm -rf $TARFILE_LOCATION/* && make fetch` (which re-downloads +# ~500MB every time), this script uses a persistent cache directory to avoid +# redundant downloads and then prunes stale tarballs. +# +# Usage: +# bin/cached-fetch.sh [CACHE_DIR] +# +# CACHE_DIR defaults to $HOME/.cache/lo-tarballs. +# TARFILE_LOCATION is read from config_host.mk (or defaults to external/tarballs). + +set -euo pipefail + +SRCDIR=$(cd "$(dirname "$0")/.." && pwd) +CACHE_DIR="${1:-${HOME}/.cache/lo-tarballs}" + +# Get TARFILE_LOCATION from build config +if [ -f "$SRCDIR/config_host.mk" ]; then + TARFILE_LOCATION=$(sed -n 's/^export TARFILE_LOCATION=//p' "$SRCDIR/config_host.mk") +fi +TARFILE_LOCATION="${TARFILE_LOCATION:-$SRCDIR/external/tarballs}" + +echo "Cache dir: $CACHE_DIR" +echo "Tarfile location: $TARFILE_LOCATION" + +mkdir -p "$CACHE_DIR" "$TARFILE_LOCATION/tmp" + +# Ask make for the list of needed tarball filenames +NEEDED=$(make -C "$SRCDIR" fetch-names) + +# Clean target dir and pre-populate needed files from cache +rm -rf "$TARFILE_LOCATION"/* +mkdir -p "$TARFILE_LOCATION/tmp" + +cached=0 +for f in $NEEDED; do + if [ -f "$CACHE_DIR/$f" ]; then + # Hardlink from cache (fast, no extra disk). + # Fall back to copy if on a different filesystem. + ln "$CACHE_DIR/$f" "$TARFILE_LOCATION/$f" 2>/dev/null || + cp "$CACHE_DIR/$f" "$TARFILE_LOCATION/$f" + cached=$((cached + 1)) + fi +done +echo "Restored from cache: $cached" + +# Fetch anything not already present (new/updated tarballs only) +make -C "$SRCDIR" fetch + +# Update cache with any newly downloaded tarballs +new=0 +for f in $NEEDED; do + if [ -f "$TARFILE_LOCATION/$f" ] && [ ! -f "$CACHE_DIR/$f" ]; then + cp "$TARFILE_LOCATION/$f" "$CACHE_DIR/$f" + new=$((new + 1)) + fi +done +[ "$new" -gt 0 ] && echo "Added to cache: $new new tarballs" + +# Handle cargo cache for y-crdt (Rust crates) +if [ -d "$CACHE_DIR/cargo" ] && [ ! -d "$TARFILE_LOCATION/cargo/registry" ]; then + cp -a "$CACHE_DIR/cargo" "$TARFILE_LOCATION/" +fi +if [ -d "$TARFILE_LOCATION/cargo" ]; then + cp -a "$TARFILE_LOCATION/cargo" "$CACHE_DIR/" +fi + +echo "Done. $TARFILE_LOCATION is ready for source tarball packaging."
