cypress_test/Makefile.am | 131 ++++++++++++++++++++----------------------- cypress_test/run_parallel.sh | 95 +++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 70 deletions(-)
New commits: commit 439b4af8ad89ab566c3ac41a955f78223644e633 Author: Tamás Zolnai <[email protected]> AuthorDate: Thu Jul 23 18:33:09 2020 +0200 Commit: Tamás Zolnai <[email protected]> CommitDate: Mon Jul 27 23:18:59 2020 +0200 cypress: better way of detecting error during parallel run. We used to grep for error messages in the output, but it can change what is written out. Better to catch if the command fails directly. Change-Id: I8c3cba212b262227551876bfb23629a243424c39 Reviewed-on: https://gerrit.libreoffice.org/c/online/+/99365 Tested-by: Jenkins Tested-by: Jenkins CollaboraOffice <[email protected]> Reviewed-by: Tamás Zolnai <[email protected]> diff --git a/cypress_test/run_parallel.sh b/cypress_test/run_parallel.sh index 38add0215..21f4e6c2a 100755 --- a/cypress_test/run_parallel.sh +++ b/cypress_test/run_parallel.sh @@ -43,6 +43,7 @@ while test $# -gt 0; do esac shift done +TEST_ERROR="${TEST_LOG}.error" TEST_FILE_PATH= if [ "${TEST_TYPE}" = "desktop" ]; then @@ -60,8 +61,6 @@ RUN_COMMAND="${CYPRESS_BINARY} run \ --env ${TEST_ENV}\ --spec=${TEST_FILE_PATH}" -ERROR_MATCHER="Error:\|Command failed:\|Timed out retrying\|The error was:" - print_error() { echo -e "\n\ CypressError: a test failed, please do one of the following:\n\n\ @@ -71,15 +70,20 @@ print_error() { \tcd cypress_test && make run-${TEST_TYPE} spec=${TEST_FILE}\n" >> ${ERROR_LOG} } +run_command() { + rm -rf ${TEST_ERROR} + echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND} || touch ${TEST_ERROR}`" > ${TEST_LOG} 2>&1 +} + mkdir -p `dirname ${TEST_LOG}` touch ${TEST_LOG} -echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND}`" > ${TEST_LOG} 2>&1 -if [ -z `grep -o -m 1 "${ERROR_MATCHER}" ${TEST_LOG}` ]; +run_command +if [ ! -f ${TEST_ERROR} ]; then cat ${TEST_LOG}; elif [ ${SECOND_CHANCE} = true ]; then echo "Second chance!" > ${TEST_LOG} && \ - echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND}`" >> ${TEST_LOG} 2>&1 && \ - if [ -z `grep -o -m 1 "${ERROR_MATCHER}" ${TEST_LOG}` ];\ + run_command && \ + if [ ! -f ${TEST_ERROR} ];\ then cat ${TEST_LOG};\ else cat ${TEST_LOG} >> ${ERROR_LOG} && \ print_error; \ commit 09abe6724f5e13181e7dfeca0ddcb2df131468f4 Author: Tamás Zolnai <[email protected]> AuthorDate: Thu Jul 23 13:23:14 2020 +0200 Commit: Tamás Zolnai <[email protected]> CommitDate: Mon Jul 27 23:18:50 2020 +0200 cypress: extract parallel test running code to a shell script file. Change-Id: Ice856737b5b4bcfc6a202b507a03fdfa16938c25 Reviewed-on: https://gerrit.libreoffice.org/c/online/+/99364 Tested-by: Jenkins CollaboraOffice <[email protected]> Tested-by: Jenkins Reviewed-by: Tamás Zolnai <[email protected]> diff --git a/cypress_test/Makefile.am b/cypress_test/Makefile.am index c0dab338b..9f6901bcf 100644 --- a/cypress_test/Makefile.am +++ b/cypress_test/Makefile.am @@ -9,6 +9,8 @@ WAIT_ON_BINARY = $(abs_builddir)/node_modules/wait-on/bin/wait-on GET_PORT_BINARY = $(abs_builddir)/node_modules/get-port-cli/cli.js NPM_INSTALLED = $(abs_builddir)/workdir/npm_installed +PARALLEL_SCRIPT = $(abs_srcdir)/run_parallel.sh + PID_FILE=$(abs_builddir)/workdir/loolwsd.pid ERROR_LOG=$(abs_builddir)/workdir/error.log @@ -202,16 +204,22 @@ define start_Xvfb endef DESKTOP_CONFIG = \ - --config integrationFolder=$(DESKTOP_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(DESKTOP_USER_AGENT) \ - --env DATA_FOLDER=$(DESKTOP_DATA_FOLDER),WORKDIR=$(DESKTOP_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)" + integrationFolder=$(DESKTOP_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(DESKTOP_USER_AGENT) + +DESKTOP_ENV = \ + DATA_FOLDER=$(DESKTOP_DATA_FOLDER),WORKDIR=$(DESKTOP_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)" MOBILE_CONFIG = \ - --config integrationFolder=$(MOBILE_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(MOBILE_USER_AGENT) \ - --env DATA_FOLDER=$(MOBILE_DATA_FOLDER),WORKDIR=$(MOBILE_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)" + integrationFolder=$(MOBILE_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(MOBILE_USER_AGENT) + +MOBILE_ENV = \ + DATA_FOLDER=$(MOBILE_DATA_FOLDER),WORKDIR=$(MOBILE_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)" MULTIUSER_CONFIG = \ - --config integrationFolder=$(MULTIUSER_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(DESKTOP_USER_AGENT),defaultCommandTimeout=30000 \ - --env DATA_FOLDER=$(MULTIUSER_DATA_FOLDER),WORKDIR=$(MULTIUSER_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)" + integrationFolder=$(MULTIUSER_TEST_FOLDER),supportFile=$(SUPPORT_FILE),userAgent=$(DESKTOP_USER_AGENT),defaultCommandTimeout=30000 + +MULTIUSER_ENV = \ + DATA_FOLDER=$(MULTIUSER_DATA_FOLDER),WORKDIR=$(MULTIUSER_WORKDIR),WSD_VERSION_HASH=$(WSD_VERSION_HASH),SERVER_PORT=$(FREE_PORT),LO_CORE_VERSION="$(CORE_VERSION)" define run_interactive_test $(if $(2),\ @@ -219,98 +227,81 @@ define run_interactive_test --browser $(CHROME) \ --headed --no-exit \ $(if $(findstring DESKTOP,$(1)),\ - $(DESKTOP_CONFIG) \ - --spec=$(abs_dir)/integration_tests/desktop/$(2),\ - $(MOBILE_CONFIG) \ + --config $(DESKTOP_CONFIG) \ + --env $(DESKTOP_ENV) \ + --spec=$(abs_dir)/integration_tests/desktop/$(2)\ + ,\ + --config $(MOBILE_CONFIG) \ + --env $(MOBILE_ENV) \ --spec=$(abs_dir)/integration_tests/mobile/$(2)),\ $(CYPRESS_BINARY) open \ - $(if $(findstring DESKTOP,$(1)),$(DESKTOP_CONFIG),$(MOBILE_CONFIG))) + $(if $(findstring DESKTOP,$(1)),\ + --config $(DESKTOP_CONFIG) \ + --env $(DESKTOP_ENV) \ + ,\ + --config $(MOBILE_CONFIG) \ + --env $(MOBILE_ENV) \)) endef define run_desktop_tests @echo $(if $(1),"Running cypress desktop test: $(1)","Running cypress desktop tests...") @echo - $(eval RUN_COMMAND = \ + $(if $(PARALLEL_BUILD),\ + @$(PARALLEL_SCRIPT) \ + --browser $(CHROME) \ + --config $(DESKTOP_CONFIG) \ + --env $(DESKTOP_ENV)$(if $(3),$(COMMA)$(3)) \ + --spec $(1) \ + --type desktop \ + --log-file $(2) \ + --second-chance \ + ,\ $(CYPRESS_BINARY) run \ --browser $(CHROME) \ --headless \ - $(DESKTOP_CONFIG)$(if $(3),$(COMMA)$(3)) \ + --config $(DESKTOP_CONFIG) \ + --env $(DESKTOP_ENV)$(if $(3),$(COMMA)$(3)) \ $(if $(1), --spec=$(abs_dir)/integration_tests/desktop/$(1)) \ - $(if $(PARALLEL_BUILD),,|| ($(KILL_COMMAND) && false))) - $(if $(PARALLEL_BUILD),\ - $(call execute_run_parallel,$(RUN_COMMAND),$(2),desktop,$(1))\ - ,\ - $(RUN_COMMAND)\ + || ($(KILL_COMMAND) && false) \ ) endef define run_mobile_tests @echo $(if $(1),"Running cypress mobile test: $(1)","Running cypress mobile tests...") @echo - $(eval RUN_COMMAND = \ + $(if $(PARALLEL_BUILD),\ + @$(PARALLEL_SCRIPT) \ + --browser $(CHROME) \ + --config $(MOBILE_CONFIG) \ + --env $(MOBILE_ENV)$(if $(3),$(COMMA)$(3)) \ + --spec $(1) \ + --type mobile \ + --log-file $(2) \ + --second-chance \ + ,\ $(CYPRESS_BINARY) run \ --browser $(CHROME) \ --headless \ - $(MOBILE_CONFIG)$(if $(3),$(COMMA)$(3)) \ + --config $(MOBILE_CONFIG) \ + --env $(MOBILE_ENV)$(if $(3),$(COMMA)$(3)) \ $(if $(1), --spec=$(abs_dir)/integration_tests/mobile/$(1)) \ - $(if $(PARALLEL_BUILD),,|| ($(KILL_COMMAND) && false))) - $(if $(PARALLEL_BUILD),\ - $(call execute_run_parallel,$(RUN_COMMAND),$(2),mobile,$(1))\ - ,\ - $(RUN_COMMAND)\ + || ($(KILL_COMMAND) && false) \ ) endef define run_multiuser_tests @echo "Running cypress multi-user test: $(1)" @echo - $(eval RUN_COMMAND = \ - $(CYPRESS_BINARY) run \ + $(if $(PARALLEL_BUILD),\ + @$(PARALLEL_SCRIPT) \ --browser $(CHROME) \ - --headless \ - $(MULTIUSER_CONFIG)$(if $(3),$(COMMA)$(3)) \ - $(if $(1), --spec=$(abs_dir)/integration_tests/multiuser/$(1)) \ - ) - $(call execute_run_parallel_no_second_chance,$(RUN_COMMAND),$(2),desktop,$(1)) -endef - -ERROR_MATCHER = "Error:\|Command failed:\|Timed out retrying\|The error was:" - -define execute_run_parallel - @mkdir -p $(dir $(2)) && touch $(2) && \ - echo "`echo $(1) && $(1)`" > $(2) 2>&1 && \ - if [ -z `grep -o -m 1 $(ERROR_MATCHER) $(2)` ];\ - then cat $(2);\ - elif [ -z $(CYPRESS_NO_SECOND_CHANCE) ]; \ - then echo "Second chance!" > $(2) && \ - echo "`echo $(1) && $(1)`" >> $(2) 2>&1 && \ - if [ -z `grep -o -m 1 $(ERROR_MATCHER) $(2)` ];\ - then cat $(2);\ - else cat $(2) >> $(ERROR_LOG) && \ - echo -e $(call error_output_string,$(3),$(4)) >> $(ERROR_LOG); \ - fi;\ - else cat $(2) >> $(ERROR_LOG) && \ - echo -e $(call error_output_string,$(3),$(4)) >> $(ERROR_LOG); \ - fi; -endef - -define execute_run_parallel_no_second_chance - @mkdir -p $(dir $(2)) && touch $(2) && \ - echo "`echo $(1) && $(1)`" > $(2) 2>&1 && \ - if [ -z `grep -o -m 1 $(ERROR_MATCHER) $(2)` ];\ - then cat $(2);\ - else cat $(2) >> $(ERROR_LOG) && \ - echo -e $(call error_output_string,$(3),$(4)) >> $(ERROR_LOG); \ - fi; -endef - -define error_output_string -"\n\ -CypressError: a test failed, please do one of the following:\n\n\ -Run the failing test in headless mode:\n\ -\tcd cypress_test && make check-$(1) spec=$(2)\n\n\ -Open the failing test in the interactive test runner:\n\ -\tcd cypress_test && make run-$(1) spec=$(2)\n" + --config $(MULTIUSER_CONFIG) \ + --env $(MULTIUSER_ENV)$(if $(3),$(COMMA)$(3)) \ + --spec $(1) \ + --type multi-user \ + --log-file $(2) \ + ,\ + ) endef NODE_BINS = \ diff --git a/cypress_test/run_parallel.sh b/cypress_test/run_parallel.sh new file mode 100755 index 000000000..38add0215 --- /dev/null +++ b/cypress_test/run_parallel.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +CYPRESS_BINARY="${DIR}/node_modules/cypress/bin/cypress" +DESKTOP_TEST_FOLDER="${DIR}/integration_tests/desktop/" +MOBILE_TEST_FOLDER="${DIR}/integration_tests/mobile/" +MULTIUSER_TEST_FOLDER="${DIR}/integration_tests/multiuser/" +ERROR_LOG="${DIR}/workdir/error.log" + +print_help () +{ + echo "Usage: run_parallel.sh --spec <name_spec.js> OPTIONS" + echo "Runs a specified cypress test" + echo "" + echo " --spec <file> The test file we need to run" + echo " --log-file <file> Log output to this test" + echo " --config <string> Configure options passed to cypress" + echo " --env <string> Cypress own environment variables" + echo " --type <string> Type of the test (e.g. mobile, desktop)" + echo " --browser <file> Path to the browser binary" + echo " --second-chance Enable second chance" + exit 1 +} + +TEST_FILE= +TEST_LOG= +TEST_CONFIG= +TEST_ENV= +TEST_TYPE= +BROWSER= +SECOND_CHANCE=false +while test $# -gt 0; do + case $1 in + --spec) TEST_FILE=$2; shift;; + --log-file) TEST_LOG=$2; shift;; + --config) TEST_CONFIG=$2; shift;; + --env) TEST_ENV=$2; shift;; + --type) TEST_TYPE=$2; shift;; + --browser) BROWSER=$2; shift;; + --second-chance) SECOND_CHANCE=true; shift;; + --help) print_help ;; + -*) ;; # ignore + esac + shift +done + +TEST_FILE_PATH= +if [ "${TEST_TYPE}" = "desktop" ]; then + TEST_FILE_PATH=${DESKTOP_TEST_FOLDER}${TEST_FILE}; +elif [ "${TEST_TYPE}" = "multi-user" ]; then + TEST_FILE_PATH=${MULTIUSER_TEST_FOLDER}${TEST_FILE}; +else + TEST_FILE_PATH=${MOBILE_TEST_FOLDER}${TEST_FILE}; +fi + +RUN_COMMAND="${CYPRESS_BINARY} run \ + --browser ${BROWSER} \ + --headless \ + --config ${TEST_CONFIG}\ + --env ${TEST_ENV}\ + --spec=${TEST_FILE_PATH}" + +ERROR_MATCHER="Error:\|Command failed:\|Timed out retrying\|The error was:" + +print_error() { + echo -e "\n\ + CypressError: a test failed, please do one of the following:\n\n\ + Run the failing test in headless mode:\n\ + \tcd cypress_test && make check-${TEST_TYPE} spec=${TEST_FILE}\n\n\ + Open the failing test in the interactive test runner:\n\ + \tcd cypress_test && make run-${TEST_TYPE} spec=${TEST_FILE}\n" >> ${ERROR_LOG} +} + +mkdir -p `dirname ${TEST_LOG}` +touch ${TEST_LOG} +echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND}`" > ${TEST_LOG} 2>&1 +if [ -z `grep -o -m 1 "${ERROR_MATCHER}" ${TEST_LOG}` ]; + then cat ${TEST_LOG}; + elif [ ${SECOND_CHANCE} = true ]; + then echo "Second chance!" > ${TEST_LOG} && \ + echo "`echo ${RUN_COMMAND} && ${RUN_COMMAND}`" >> ${TEST_LOG} 2>&1 && \ + if [ -z `grep -o -m 1 "${ERROR_MATCHER}" ${TEST_LOG}` ];\ + then cat ${TEST_LOG};\ + else cat ${TEST_LOG} >> ${ERROR_LOG} && \ + print_error; \ + fi; + else cat ${TEST_LOG} >> ${ERROR_LOG} && \ + print_error; +fi; + +# vim:set shiftwidth=4 expandtab: _______________________________________________ Libreoffice-commits mailing list [email protected] https://lists.freedesktop.org/mailman/listinfo/libreoffice-commits
