bin/symstore.sh | 129 ++++++++++++++++++++++++++++++------ external/nss/ExternalProject_nss.mk | 6 + 2 files changed, 115 insertions(+), 20 deletions(-)
New commits: commit d612edc70458cd67def29c3934017235e70ccd42 Author: Thorsten Behrens <[email protected]> AuthorDate: Wed Aug 18 22:48:46 2021 +0200 Commit: Thorsten Behrens <[email protected]> CommitDate: Tue Nov 23 00:17:25 2021 +0100 Teach symstore more duplicated DLLs Amending commit b4dfba947768834ffecc09056992019878711c8b with adding the copied DLLs here, too (symstore complains otherwise) Change-Id: If5cc155cf2fe11b0f97cd152993609fd2c835316 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/120686 Tested-by: Thorsten Behrens <[email protected]> Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/symstore.sh b/bin/symstore.sh index c4e9870c9aa5..2734f1b22a58 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -16,6 +16,8 @@ EXCLUDE_LIST="python.exe" # # Same format as for EXCLUDE_LIST above MOREPDBS_OKLIST="libcurl.dll +libcrypto-1_1.dll +libssl-1_1.dll freebl3.dll libeay32.dll nspr4.dll commit 3a13a408113d09348039a2989f2a064c12134752 Author: Thorsten Behrens <[email protected]> AuthorDate: Tue Nov 23 00:16:49 2021 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Tue Nov 23 00:16:49 2021 +0100 replace usage of blacklist with denylist .. and a few cases of instead doing blacklist->excludelist where that made more sense. Background and motivation: https://tools.ietf.org/html/draft-knodel-terminology-02 [API CHANGE] officecfg::Office::Canvas::DeviceBlacklist -> DeviceDenylist [API CHANGE] officecfg::Office::Canvas::BlacklistCurrentDevice -> DenylistCurrentDevice [API CHANGE] officecfg::Office::Common::Misc::OpenCLBlackList -> OpenCLDenyList Change-Id: Ia35e25496bf0cc0692d5de4cb66bfc232d3a869e Reviewed-on: https://gerrit.libreoffice.org/c/core/+/98180 Tested-by: Thorsten Behrens <[email protected]> Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/symstore.sh b/bin/symstore.sh index 532efb16630f..c4e9870c9aa5 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -2,19 +2,19 @@ # Files listed here would not be store in the symbolestore-server. # The format is a string with files e.g. -# BLACKLIST="python.exe +# EXCLUDE_LIST="python.exe # file.dll # next_file.exe" # # It removes "python.exe", "file.dll", and "next_file.exe" from what's # added to the symstore. Separator is the newline -BLACK_LIST="python.exe" +EXCLUDE_LIST="python.exe" # List files here where it's ok for this script to find more than one # occurrence in the build tree. Files _not_ included here will generate # an error, if duplicates are found. # -# Same format as for BLACK_LIST above +# Same format as for EXCLUDE_LIST above MOREPDBS_OKLIST="libcurl.dll freebl3.dll libeay32.dll @@ -46,7 +46,7 @@ add_pdb() stats_morefound=0 declare -a pdball echo "Collect $extension" - ret=$(find "${INSTDIR}/" -type f -name "*.${extension}" | grep -vF "$BLACK_LIST") + ret=$(find "${INSTDIR}/" -type f -name "*.${extension}" | grep -vF "$EXCLUDE_LIST") while IFS= read -r file do ${VERBOSE} -n "Found: $file" commit e14dec6d721451e5ec43e34f1c8d0ae79e094566 Author: Juergen Funk <[email protected]> AuthorDate: Fri Mar 6 11:43:21 2020 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Tue Nov 23 00:14:36 2021 +0100 symstore.sh: add vorbose mode Change-Id: I01f9827140810080c037108021ca70f2b906b05e Reviewed-on: https://gerrit.libreoffice.org/c/core/+/90083 Tested-by: Jenkins Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/symstore.sh b/bin/symstore.sh index b1470ae9b947..532efb16630f 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -31,6 +31,10 @@ sqlite3.dll ssl3.dll ssleay32.dll" +verbose_none() +{ + do_none= +} add_pdb() { @@ -45,10 +49,15 @@ add_pdb() ret=$(find "${INSTDIR}/" -type f -name "*.${extension}" | grep -vF "$BLACK_LIST") while IFS= read -r file do + ${VERBOSE} -n "Found: $file" # store dll/exe itself (needed for minidumps) if [ $WITHEXEC == 1 ] ; then cygpath -w "$file" >> "$list" + ${VERBOSE} " insert" + else + ${VERBOSE} " " fi + # store pdb file filename=$(basename "$file" ".${extension}") pdball+=($(grep -i "/${filename}${pdbext}" <<< ${ALL_PDBS})) @@ -56,8 +65,12 @@ add_pdb() cygpath -w "${pdball[0]}" >> "$list" fi case ${#pdball[@]} in - 0) ((++stats_notfound)) ;; - 1) ((++stats_found)) ;; + 0) ((++stats_notfound)) + ${VERBOSE} " PDB not found" + ;; + 1) ((++stats_found)) + ${VERBOSE} " ${pdball[0]} insert" + ;; *) ((++stats_morefound)) if [ -z "$(echo $file | grep -F "$MOREPDBS_OKLIST")" ]; then echo "Error: found duplicate PDBs:" @@ -65,6 +78,8 @@ add_pdb() echo " $morepdbs" done exit 1 + else + ${VERBOSE} " ${pdball[0]} insert (is in more okay list)" fi ;; esac @@ -98,13 +113,15 @@ SYM_PATH=${WORKDIR}/symstore COMMENT="" COMCMD="" WITHEXEC=1 +VERBOSE=verbose_none -USAGE="Usage: $0 [-h|-k <keep_num_versions>|-p <symbol_store_path>] +USAGE="Usage: $0 [-h|-k <keep_num_versions>|-p <symbol_store_path>|-c <comment>|-n|-v] -h: this cruft -c <comment> specifies a comment for the transaction -n do not store exe/dll on the symbole server -k <int>: keep this number of old symbol versions around (default: ${MAX_KEEP}. Set to 0 for unlimited) + -v verbose mode, output detail report of files -p <path>: specify full path to symbol store tree If no path is specified, defaults to ${SYM_PATH}. " @@ -117,6 +134,7 @@ do -p|--path) SYM_PATH="$2"; shift 2;; -c|--comment) COMCMD="/c"; COMMENT="$2"; shift 2;; -n|--noexec) WITHEXEC=0; shift ;; + -v|--verbose) VERBOSE=echo; shift ;; -h|--help) echo "${USAGE}"; exit 0;; -*) echo "${USAGE}" >&2; exit 1;; *) break;; commit c10f7d27524cd45966d2bfbfafe9919c0e941113 Author: Juergen Funk <[email protected]> AuthorDate: Fri Feb 14 10:05:56 2020 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Tue Nov 23 00:14:17 2021 +0100 Optionally generate PDBs also for nss Enables pdb generation for symbol builds, for: - freebl3.dll - libeay32.dll - nspr4.dll - nss3.dll - nssckbi.dll - nssdbm3.dll - nssutil3.dll - plc4.dll - plds4.dll - smime3.dll - softokn3.dll - sqlite3.dll - ssl3.dll - ssleay32.dll Change-Id: I231fdc8e8ade7b7a8b85fc76536291e0546f2eac Reviewed-on: https://gerrit.libreoffice.org/c/core/+/88673 Tested-by: Jenkins Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/symstore.sh b/bin/symstore.sh index 9809c5554c8a..b1470ae9b947 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -15,7 +15,21 @@ BLACK_LIST="python.exe" # an error, if duplicates are found. # # Same format as for BLACK_LIST above -MOREPDBS_OKLIST="libcurl.dll" +MOREPDBS_OKLIST="libcurl.dll +freebl3.dll +libeay32.dll +nspr4.dll +nss3.dll +nssckbi.dll +nssdbm3.dll +nssutil3.dll +plc4.dll +plds4.dll +smime3.dll +softokn3.dll +sqlite3.dll +ssl3.dll +ssleay32.dll" add_pdb() diff --git a/external/nss/ExternalProject_nss.mk b/external/nss/ExternalProject_nss.mk index 722da93fa51f..7f730cb2bc87 100644 --- a/external/nss/ExternalProject_nss.mk +++ b/external/nss/ExternalProject_nss.mk @@ -19,7 +19,11 @@ ifeq ($(OS),WNT) $(call gb_ExternalProject_get_state_target,nss,build): $(call gb_ExternalExecutable_get_dependencies,python) $(call gb_ExternalProject_run,build,\ $(if $(MSVC_USE_DEBUG_RUNTIME),USE_DEBUG_RTL=1,BUILD_OPT=1) \ - OS_TARGET=WIN95 \ + $(if $(gb_Module_CURRENTMODULE_SYMBOLS_ENABLED), \ + MOZ_DEBUG_SYMBOLS=1 \ + MOZ_DEBUG_FLAGS=" " \ + OPT_CODE_SIZE=0) \ + MOZ_MSVCVERSION=9 OS_TARGET=WIN95 \ $(if $(filter X86_64,$(CPUNAME)),USE_64=1) \ LIB="$(ILIB)" \ XCFLAGS="-arch:SSE $(SOLARINC)" \ commit de5dcc0abe797c972228cd7a74e0abbde05c90c4 Author: Andrea Gelmini <[email protected]> AuthorDate: Tue Dec 17 17:20:05 2019 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Tue Nov 23 00:11:52 2021 +0100 Fix typos Change-Id: I2671a0faf78d8ece27aaa88082a60f19197e929f Reviewed-on: https://gerrit.libreoffice.org/85314 Reviewed-by: Julien Nabet <[email protected]> Tested-by: Julien Nabet <[email protected]> diff --git a/bin/symstore.sh b/bin/symstore.sh index 564739a0279f..9809c5554c8a 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -11,10 +11,10 @@ BLACK_LIST="python.exe" # List files here where it's ok for this script to find more than one -# occurence in the build tree. Files _not_ included here will generate +# occurrence in the build tree. Files _not_ included here will generate # an error, if duplicates are found. # -# Same format as for BLACK_LIST above above +# Same format as for BLACK_LIST above MOREPDBS_OKLIST="libcurl.dll" commit a20e28b8470a995589c9b16c43cccdca597c5eb7 Author: Juergen Funk <[email protected]> AuthorDate: Tue Nov 26 08:06:39 2019 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Tue Nov 23 00:11:47 2021 +0100 symstore.sh: collect PDBs for *all* artifacts - a number of external library PDBs where missing - also the soffice.bin and unopkg.bin (renaming that to *.bin.pdb) Change-Id: Idafcce87bfefadfa669807a861efab76b4122c62 Reviewed-on: https://gerrit.libreoffice.org/83726 Tested-by: Jenkins Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/symstore.sh b/bin/symstore.sh index 2f7ad2f77610..564739a0279f 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -1,22 +1,67 @@ #!/usr/bin/env bash +# Files listed here would not be store in the symbolestore-server. +# The format is a string with files e.g. +# BLACKLIST="python.exe +# file.dll +# next_file.exe" +# +# It removes "python.exe", "file.dll", and "next_file.exe" from what's +# added to the symstore. Separator is the newline +BLACK_LIST="python.exe" + +# List files here where it's ok for this script to find more than one +# occurence in the build tree. Files _not_ included here will generate +# an error, if duplicates are found. +# +# Same format as for BLACK_LIST above above +MOREPDBS_OKLIST="libcurl.dll" + + add_pdb() { extension=$1 - type=$2 + pdbext=$2 list=$3 - for file in $(find "${INSTDIR}/" -name "*.${extension}"); do + stats_notfound=0 + stats_found=0 + stats_morefound=0 + declare -a pdball + echo "Collect $extension" + ret=$(find "${INSTDIR}/" -type f -name "*.${extension}" | grep -vF "$BLACK_LIST") + while IFS= read -r file + do # store dll/exe itself (needed for minidumps) - if [ -f "$file" -a $WITHEXEC == 1 ] ; then + if [ $WITHEXEC == 1 ] ; then cygpath -w "$file" >> "$list" fi # store pdb file filename=$(basename "$file" ".${extension}") - pdb="${WORKDIR}/LinkTarget/${type}/${filename}.pdb" - if [ -f "$pdb" ]; then - cygpath -w "$pdb" >> "$list" + pdball+=($(grep -i "/${filename}${pdbext}" <<< ${ALL_PDBS})) + if [ -n "${pdball[0]}" ]; then + cygpath -w "${pdball[0]}" >> "$list" fi - done + case ${#pdball[@]} in + 0) ((++stats_notfound)) ;; + 1) ((++stats_found)) ;; + *) ((++stats_morefound)) + if [ -z "$(echo $file | grep -F "$MOREPDBS_OKLIST")" ]; then + echo "Error: found duplicate PDBs:" + for morepdbs in ${pdball[@]} ; do + echo " $morepdbs" + done + exit 1 + fi + ;; + esac + unset pdball + done <<EOF +${ret} +EOF + + echo " Found PDBs : $stats_found" + echo " Missing PDBs : $stats_notfound" + echo " Multiple PDBs : $stats_morefound" } # check preconditions @@ -73,9 +118,13 @@ fi TMPFILE=$(mktemp) || exit 1 trap '{ rm -f ${TMPFILE}; }' EXIT +# collect all PDBs +ALL_PDBS=$(find "${WORKDIR}/" -type f -name "*.pdb") + # add dlls and executables -add_pdb dll Library "${TMPFILE}" -add_pdb exe Executable "${TMPFILE}" +add_pdb dll .pdb "${TMPFILE}" +add_pdb exe .pdb "${TMPFILE}" +add_pdb bin .bin.pdb "${TMPFILE}" # stick all of it into symbol store symstore.exe add /compress /f "@$(cygpath -w "${TMPFILE}")" /s "$(cygpath -w "${SYM_PATH}")" /t "${PRODUCTNAME}" /v "${LIBO_VERSION_MAJOR}.${LIBO_VERSION_MINOR}.${LIBO_VERSION_MICRO}.${LIBO_VERSION_PATCH}${LIBO_VERSION_SUFFIX}${LIBO_VERSION_SUFFIX_SUFFIX}" "${COMCMD}" "${COMMENT}" @@ -87,6 +136,7 @@ rm -f "${TMPFILE}" if [ "${MAX_KEEP}" -gt 0 -a -d "${SYM_PATH}/000Admin" ]; then to_remove=$(ls -1 "${SYM_PATH}/000Admin" | grep -v '\.txt' | grep -v '\.deleted' | sort | head -n "-${MAX_KEEP}") for revision in $to_remove; do + echo "Remove $revision from symstore" symstore.exe del /i "${revision}" /s "$(cygpath -w "${SYM_PATH}")" done fi commit 288b8c5c60eb65231953bb43dc2629f32b39bdef Author: Juergen Funk <[email protected]> AuthorDate: Mon Nov 18 10:33:13 2019 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Tue Nov 23 00:11:43 2021 +0100 symstore.sh: use logical operators for test statements No need to spawn two subshells, test can do and/or logical ops itself. Change-Id: I2abba303383f9f0053515088d4fa32753a777a1d Reviewed-on: https://gerrit.libreoffice.org/83066 Tested-by: Jenkins Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/symstore.sh b/bin/symstore.sh index f37ea0f4e7a0..2f7ad2f77610 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -20,11 +20,11 @@ add_pdb() } # check preconditions -if [ -z "${INSTDIR}" ] || [ -z "${WORKDIR}" ]; then +if [ -z "${INSTDIR}" -o -z "${WORKDIR}" ]; then echo "INSTDIR or WORKDIR not set - script expects calling inside buildenv" exit 1 fi -if [ ! -d "${INSTDIR}" ] || [ ! -d "${WORKDIR}" ]; then +if [ ! -d "${INSTDIR}" -o ! -d "${WORKDIR}" ]; then echo "INSTDIR or WORKDIR not present - script expects calling after full build" exit 1 fi @@ -84,7 +84,7 @@ rm -f "${TMPFILE}" # Cleanup symstore, older revisions will be removed. Unless the # .dll/.exe changes, the .pdb should be shared, so with incremental # tinderbox several revisions should not be that space-demanding. -if [ "${MAX_KEEP}" -gt 0 ] && [ -d "${SYM_PATH}/000Admin" ]; then +if [ "${MAX_KEEP}" -gt 0 -a -d "${SYM_PATH}/000Admin" ]; then to_remove=$(ls -1 "${SYM_PATH}/000Admin" | grep -v '\.txt' | grep -v '\.deleted' | sort | head -n "-${MAX_KEEP}") for revision in $to_remove; do symstore.exe del /i "${revision}" /s "$(cygpath -w "${SYM_PATH}")" commit c9815d5b6e8e27698b452dcc87c95c47d8ad0c05 Author: Juergen Funk <[email protected]> AuthorDate: Thu Nov 14 12:25:35 2019 +0100 Commit: Thorsten Behrens <[email protected]> CommitDate: Tue Nov 23 00:11:33 2021 +0100 Add comment and pdb-only switch to symstore.sh Two new switches added: -c plus a comment for the transaction -n do not store the exe/dll on the symbol server Change-Id: I8c5db06909720707987970347e298be6d55ebc71 Reviewed-on: https://gerrit.libreoffice.org/82751 Tested-by: Jenkins Reviewed-by: Thorsten Behrens <[email protected]> diff --git a/bin/symstore.sh b/bin/symstore.sh index b368eb3e6715..f37ea0f4e7a0 100755 --- a/bin/symstore.sh +++ b/bin/symstore.sh @@ -7,7 +7,7 @@ add_pdb() list=$3 for file in $(find "${INSTDIR}/" -name "*.${extension}"); do # store dll/exe itself (needed for minidumps) - if [ -f "$file" ]; then + if [ -f "$file" -a $WITHEXEC == 1 ] ; then cygpath -w "$file" >> "$list" fi # store pdb file @@ -36,12 +36,17 @@ which symstore.exe > /dev/null 2>&1 || { # defaults MAX_KEEP=5 SYM_PATH=${WORKDIR}/symstore +COMMENT="" +COMCMD="" +WITHEXEC=1 USAGE="Usage: $0 [-h|-k <keep_num_versions>|-p <symbol_store_path>] - -h: this cruft - -k <int>: keep this number of old symbol versions around - (default: ${MAX_KEEP}. Set to 0 for unlimited) - -p <path>: specify full path to symbol store tree + -h: this cruft + -c <comment> specifies a comment for the transaction + -n do not store exe/dll on the symbole server + -k <int>: keep this number of old symbol versions around + (default: ${MAX_KEEP}. Set to 0 for unlimited) + -p <path>: specify full path to symbol store tree If no path is specified, defaults to ${SYM_PATH}. " @@ -51,7 +56,9 @@ do case "$1" in -k|--keep) MAX_KEEP="$2"; shift 2;; -p|--path) SYM_PATH="$2"; shift 2;; - -h|--help) echo "${USAGE}"; exit 0; shift;; + -c|--comment) COMCMD="/c"; COMMENT="$2"; shift 2;; + -n|--noexec) WITHEXEC=0; shift ;; + -h|--help) echo "${USAGE}"; exit 0;; -*) echo "${USAGE}" >&2; exit 1;; *) break;; esac @@ -71,7 +78,7 @@ add_pdb dll Library "${TMPFILE}" add_pdb exe Executable "${TMPFILE}" # stick all of it into symbol store -symstore.exe add /compress /f "@$(cygpath -w "${TMPFILE}")" /s "$(cygpath -w "${SYM_PATH}")" /t "${PRODUCTNAME}" /v "${LIBO_VERSION_MAJOR}.${LIBO_VERSION_MINOR}.${LIBO_VERSION_MICRO}.${LIBO_VERSION_PATCH}${LIBO_VERSION_SUFFIX}${LIBO_VERSION_SUFFIX_SUFFIX}" +symstore.exe add /compress /f "@$(cygpath -w "${TMPFILE}")" /s "$(cygpath -w "${SYM_PATH}")" /t "${PRODUCTNAME}" /v "${LIBO_VERSION_MAJOR}.${LIBO_VERSION_MINOR}.${LIBO_VERSION_MICRO}.${LIBO_VERSION_PATCH}${LIBO_VERSION_SUFFIX}${LIBO_VERSION_SUFFIX_SUFFIX}" "${COMCMD}" "${COMMENT}" rm -f "${TMPFILE}" # Cleanup symstore, older revisions will be removed. Unless the
