This is an automated email from the ASF dual-hosted git repository. adelapena pushed a commit to branch cassandra-4.1 in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 1cdb747b4ca11b4d3d8ea76633bdf5de64175fc3 Merge: 1e67ef02f5 5e705a8492 Author: Andrés de la Peña <[email protected]> AuthorDate: Fri Nov 4 16:26:51 2022 +0000 Merge branch 'cassandra-4.0' into cassandra-4.1 .circleci/config-2_1.yml | 6 ++-- .circleci/config.yml | 40 +++++++++++------------ .circleci/config.yml.HIGHRES | 40 +++++++++++------------ .circleci/config.yml.LOWRES | 40 +++++++++++------------ .circleci/config.yml.MIDRES | 40 +++++++++++------------ build.xml | 78 ++++++++++++++++++++++++++------------------ 6 files changed, 130 insertions(+), 114 deletions(-) diff --cc .circleci/config-2_1.yml index 7c348937de,ff20ec4a16..0ca3d6a193 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@@ -2007,33 -1979,6 +2007,33 @@@ commands no_output_timeout: 15m + run_simulator_tests: + parameters: + no_output_timeout: + type: string + default: 30m + steps: + - run: + name: Run Simulator Tests + command: | + set -x + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant test-simulator-dtest ++ ant test-simulator-dtest -Dno-build-test=true + no_output_timeout: <<parameters.no_output_timeout>> + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + run_junit_tests: parameters: target: @@@ -2108,7 -2050,7 +2108,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant <<parameters.target>> <<parameters.arguments>> -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>> - ant <<parameters.target>> -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>> -Dno-build-test=true ++ ant <<parameters.target>> <<parameters.arguments>> -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=<<parameters.classlistprefix>> -Dno-build-test=true no_output_timeout: <<parameters.no_output_timeout>> - store_test_results: path: /tmp/cassandra/build/test/output/ diff --cc .circleci/config.yml index 3e06f9b7f2,e4feeed771..238c4f12bf --- a/.circleci/config.yml +++ b/.circleci/config.yml @@@ -83,7 -83,7 +83,7 @@@ jobs if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -445,7 -640,7 +445,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -720,98 -992,103 +720,98 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=test-compression - testtag="" - if [[ $target == "test-cdc" ]]; then - testtag="cdc" - elif [[ $target == "test-compression" ]]; then - testtag="compression" - elif [[ $target == "test-system-keyspace-directory" ]]; then - testtag="system_keyspace_directory" + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "fqltool-test" || \ - $target == "long-test" || \ - $target == "stress-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) ++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_utests_compression_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...] - store_test_results: path: /tmp/results/repeated_utests/output - store_artifacts: @@@ -930,7 -1204,7 +930,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -1836,7 -2287,7 +1836,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2025,7 -2470,7 +2025,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2142,7 -2584,7 +2142,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2256,8 -2652,8 +2256,8 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant test-simulator-dtest - ant long-test -Dno-build-test=true - no_output_timeout: 15m ++ ant test-simulator-dtest -Dno-build-test=true + no_output_timeout: 30m - store_test_results: path: /tmp/cassandra/build/test/output/ - store_artifacts: @@@ -2369,19 -2745,113 +2369,19 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=testsome - testtag="" - if [[ $target == "test-cdc" ]]; then - testtag="cdc" - elif [[ $target == "test-compression" ]]; then - testtag="compression" - elif [[ $target == "test-system-keyspace-directory" ]]; then - testtag="system_keyspace_directory" + test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "fqltool-test" || \ - $target == "long-test" || \ - $target == "stress-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant testsome $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) ++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true + no_output_timeout: 15m - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout + path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/results/repeated_utests/output + path: /tmp/cassandra/build/test/output destination: junitxml - store_artifacts: - path: /tmp/results/repeated_utests/logs + path: /tmp/cassandra/build/test/logs destination: logs environment: - ANT_HOME: /usr/share/ant @@@ -2436,114 -2903,56 +2436,114 @@@ - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies + name: Run Unit Tests (long-test) command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...] - - run: - name: Run dtests (j8_without_vnodes) + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant long-test ++ ant long-test -Dno-build-test=true no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_unit_tests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information command: | - echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then - export CQLSH_PYTHON=/usr/bin/python2.7 - fi - + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...] - store_test_results: - path: /tmp/results + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes + path: /tmp/results/repeated_utests/stdout + destination: stdout - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@@ -3278,7 -3889,7 +3278,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -4458,7 -5467,7 +4458,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -4575,7 -5581,7 +4575,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -5581,7 -6668,7 +5581,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -5830,7 -6911,7 +5830,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ diff --cc .circleci/config.yml.HIGHRES index 7c8c2ddc61,d2c0721ed1..d855f4dd94 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@@ -83,7 -83,7 +83,7 @@@ jobs if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -445,7 -640,7 +445,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -720,98 -992,103 +720,98 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=test-compression - testtag="" - if [[ $target == "test-cdc" ]]; then - testtag="cdc" - elif [[ $target == "test-compression" ]]; then - testtag="compression" - elif [[ $target == "test-system-keyspace-directory" ]]; then - testtag="system_keyspace_directory" + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "fqltool-test" || \ - $target == "long-test" || \ - $target == "stress-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) ++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_utests_compression_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...] - store_test_results: path: /tmp/results/repeated_utests/output - store_artifacts: @@@ -930,7 -1204,7 +930,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -1836,7 -2287,7 +1836,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2025,7 -2470,7 +2025,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2142,7 -2584,7 +2142,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2256,8 -2652,8 +2256,8 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant test-simulator-dtest - ant long-test -Dno-build-test=true - no_output_timeout: 15m ++ ant test-simulator-dtest -Dno-build-test=true + no_output_timeout: 30m - store_test_results: path: /tmp/cassandra/build/test/output/ - store_artifacts: @@@ -2369,19 -2745,113 +2369,19 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=testsome - testtag="" - if [[ $target == "test-cdc" ]]; then - testtag="cdc" - elif [[ $target == "test-compression" ]]; then - testtag="compression" - elif [[ $target == "test-system-keyspace-directory" ]]; then - testtag="system_keyspace_directory" + test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "fqltool-test" || \ - $target == "long-test" || \ - $target == "stress-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant testsome $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) ++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true + no_output_timeout: 15m - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout + path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/results/repeated_utests/output + path: /tmp/cassandra/build/test/output destination: junitxml - store_artifacts: - path: /tmp/results/repeated_utests/logs + path: /tmp/cassandra/build/test/logs destination: logs environment: - ANT_HOME: /usr/share/ant @@@ -2436,114 -2903,56 +2436,114 @@@ - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies + name: Run Unit Tests (long-test) command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...] - - run: - name: Run dtests (j8_without_vnodes) + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant long-test ++ ant long-test -Dno-build-test=true no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_unit_tests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information command: | - echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then - export CQLSH_PYTHON=/usr/bin/python2.7 - fi - + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...] - store_test_results: - path: /tmp/results + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes + path: /tmp/results/repeated_utests/stdout + destination: stdout - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@@ -3278,7 -3889,7 +3278,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -4458,7 -5467,7 +4458,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -4575,7 -5581,7 +4575,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -5581,7 -6668,7 +5581,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -5830,7 -6911,7 +5830,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ diff --cc .circleci/config.yml.LOWRES index 1455e61615,657c8a8624..1e0d88ab51 --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@@ -83,7 -83,7 +83,7 @@@ jobs if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -445,7 -640,7 +445,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -720,98 -992,103 +720,98 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=test-compression - testtag="" - if [[ $target == "test-cdc" ]]; then - testtag="cdc" - elif [[ $target == "test-compression" ]]; then - testtag="compression" - elif [[ $target == "test-system-keyspace-directory" ]]; then - testtag="system_keyspace_directory" + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "fqltool-test" || \ - $target == "long-test" || \ - $target == "stress-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) ++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_utests_compression_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...] - store_test_results: path: /tmp/results/repeated_utests/output - store_artifacts: @@@ -930,7 -1204,7 +930,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -1836,7 -2287,7 +1836,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2025,7 -2470,7 +2025,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2142,7 -2584,7 +2142,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2256,8 -2652,8 +2256,8 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant test-simulator-dtest - ant long-test -Dno-build-test=true - no_output_timeout: 15m ++ ant test-simulator-dtest -Dno-build-test=true + no_output_timeout: 30m - store_test_results: path: /tmp/cassandra/build/test/output/ - store_artifacts: @@@ -2369,19 -2745,113 +2369,19 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=testsome - testtag="" - if [[ $target == "test-cdc" ]]; then - testtag="cdc" - elif [[ $target == "test-compression" ]]; then - testtag="compression" - elif [[ $target == "test-system-keyspace-directory" ]]; then - testtag="system_keyspace_directory" + test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "fqltool-test" || \ - $target == "long-test" || \ - $target == "stress-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant testsome $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) ++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true + no_output_timeout: 15m - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout + path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/results/repeated_utests/output + path: /tmp/cassandra/build/test/output destination: junitxml - store_artifacts: - path: /tmp/results/repeated_utests/logs + path: /tmp/cassandra/build/test/logs destination: logs environment: - ANT_HOME: /usr/share/ant @@@ -2436,114 -2903,56 +2436,114 @@@ - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies + name: Run Unit Tests (long-test) command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...] - - run: - name: Run dtests (j8_without_vnodes) + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant long-test ++ ant long-test -Dno-build-test=true no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_unit_tests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information command: | - echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then - export CQLSH_PYTHON=/usr/bin/python2.7 - fi - + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...] - store_test_results: - path: /tmp/results + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes + path: /tmp/results/repeated_utests/stdout + destination: stdout - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@@ -3278,7 -3889,7 +3278,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -4458,7 -5467,7 +4458,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -4575,7 -5581,7 +4575,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -5581,7 -6668,7 +5581,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -5830,7 -6911,7 +5830,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ diff --cc .circleci/config.yml.MIDRES index 9b309ecdd0,8937f0f703..d903332c3b --- a/.circleci/config.yml.MIDRES +++ b/.circleci/config.yml.MIDRES @@@ -83,7 -83,7 +83,7 @@@ jobs if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -445,7 -640,7 +445,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -720,98 -992,103 +720,98 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=test-compression - testtag="" - if [[ $target == "test-cdc" ]]; then - testtag="cdc" - elif [[ $target == "test-compression" ]]; then - testtag="compression" - elif [[ $target == "test-system-keyspace-directory" ]]; then - testtag="system_keyspace_directory" + test_timeout=$(grep 'name="test.distributed.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "fqltool-test" || \ - $target == "long-test" || \ - $target == "stress-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant test-compression $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) ++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_utests_compression_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...] - store_test_results: path: /tmp/results/repeated_utests/output - store_artifacts: @@@ -930,7 -1204,7 +930,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -1836,7 -2287,7 +1836,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2025,7 -2470,7 +2025,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-system-keyspace-directory -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2142,7 -2584,7 +2142,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.jvm.args='-Dcassandra.dtest.num_tokens=16' -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -2256,8 -2652,8 +2256,8 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - ant test-simulator-dtest - ant long-test -Dno-build-test=true - no_output_timeout: 15m ++ ant test-simulator-dtest -Dno-build-test=true + no_output_timeout: 30m - store_test_results: path: /tmp/cassandra/build/test/output/ - store_artifacts: @@@ -2369,19 -2745,113 +2369,19 @@@ if [ -d ~/dtest_jars ]; then cp ~/dtest_jars/dtest* /tmp/cassandra/build/ fi - - # Calculate the number of test iterations to be run by the current parallel runner. - count=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL)) - if (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then - count=$((count+1)) - fi - - # Put manually specified tests and automatically detected tests together, removing duplicates - tests=$(echo ${REPEATED_UTESTS} | sed -e "s/<nil>//" | sed -e "s/ //" | tr "," "\n" | tr " " "\n" | sort -n | uniq -u) - echo "Tests to be repeated: ${tests}" - - # Prepare the testtag for the target, used by the test macro in build.xml to group the output files - target=testsome - testtag="" - if [[ $target == "test-cdc" ]]; then - testtag="cdc" - elif [[ $target == "test-compression" ]]; then - testtag="compression" - elif [[ $target == "test-system-keyspace-directory" ]]; then - testtag="system_keyspace_directory" + test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk -F'"' '{print $4}' || true) + if [ -z "$test_timeout" ]; then + test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - - # Run each test class as many times as requested. - exit_code="$?" - for test in $tests; do - - # Split class and method names from the test name - if [[ $test =~ "#" ]]; then - class=${test%"#"*} - method=${test#*"#"} - else - class=$test - method="" - fi - - # Prepare the -Dtest.name argument. - # It can be the fully qualified class name or the short class name, depending on the target. - if [[ $target == "test" || \ - $target == "test-cdc" || \ - $target == "test-compression" || \ - $target == "test-system-keyspace-directory" || \ - $target == "fqltool-test" || \ - $target == "long-test" || \ - $target == "stress-test" ]]; then - name_arg="-Dtest.name=${class##*.}" - else - name_arg="-Dtest.name=$class" - fi - - # Prepare the -Dtest.methods argument, which is optional - if [[ $method == "" ]]; then - methods_arg="" - else - methods_arg="-Dtest.methods=$method" - fi - - for i in $(seq -w 1 $count); do - echo "Running test $test, iteration $i of $count" - - # run the test - status="passes" - if !( set -o pipefail && \ - ant testsome $name_arg $methods_arg -Dno-build-test=true | \ - tee stdout.txt \ - ); then - status="fails" - exit_code=1 - fi - - # move the stdout output file - dest=/tmp/results/repeated_utests/stdout/${status}/${i} - mkdir -p $dest - mv stdout.txt $dest/${test}.txt - - # move the XML output files - source=build/test/output/${testtag} - dest=/tmp/results/repeated_utests/output/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # move the log files - source=build/test/logs/${testtag} - dest=/tmp/results/repeated_utests/logs/${status}/${i} - mkdir -p $dest - if [[ -d $source && -n "$(ls $source)" ]]; then - mv $source/* $dest/ - fi - - # maybe stop iterations on test failure - if [[ ${REPEATED_TESTS_STOP_ON_FAILURE} = true ]] && (( $exit_code > 0 )); then - break - fi - done - done - (exit ${exit_code}) ++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true + no_output_timeout: 15m - store_test_results: - path: /tmp/results/repeated_utests/output - - store_artifacts: - path: /tmp/results/repeated_utests/stdout - destination: stdout + path: /tmp/cassandra/build/test/output/ - store_artifacts: - path: /tmp/results/repeated_utests/output + path: /tmp/cassandra/build/test/output destination: junitxml - store_artifacts: - path: /tmp/results/repeated_utests/logs + path: /tmp/cassandra/build/test/logs destination: logs environment: - ANT_HOME: /usr/share/ant @@@ -2436,114 -2903,56 +2436,114 @@@ - attach_workspace: at: /home/cassandra - run: - name: Clone Cassandra dtest Repository (via git) - command: | - git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest - - run: - name: Configure virtualenv and python Dependencies + name: Run Unit Tests (long-test) command: | - # note, this should be super quick as all dependencies should be pre-installed in the docker image - # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated - # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and - # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) - source ~/env3.6/bin/activate export PATH=$JAVA_HOME/bin:$PATH - pip3 install --exists-action w --upgrade -r ~/cassandra-dtest/requirements.txt - pip3 uninstall -y cqlsh - pip3 freeze - - run: - name: Determine Tests to Run (j8_without_vnodes) - no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env3.6/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-i [...] - - run: - name: Run dtests (j8_without_vnodes) + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant long-test ++ ant long-test -Dno-build-test=true no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: https://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: trunk + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - REPEATED_TESTS_STOP_ON_FAILURE: false + - REPEATED_UTESTS: null + - REPEATED_UTESTS_COUNT: 500 + - REPEATED_UTESTS_FQLTOOL: null + - REPEATED_UTESTS_FQLTOOL_COUNT: 500 + - REPEATED_UTESTS_LONG: null + - REPEATED_UTESTS_LONG_COUNT: 100 + - REPEATED_UTESTS_STRESS: null + - REPEATED_UTESTS_STRESS_COUNT: 500 + - REPEATED_SIMULATOR_DTESTS: null + - REPEATED_SIMULATOR_DTESTS_COUNT: 500 + - REPEATED_JVM_DTESTS: null + - REPEATED_JVM_DTESTS_COUNT: 500 + - REPEATED_JVM_UPGRADE_DTESTS: null + - REPEATED_JVM_UPGRADE_DTESTS_COUNT: 500 + - REPEATED_DTESTS: null + - REPEATED_DTESTS_COUNT: 500 + - REPEATED_UPGRADE_DTESTS: null + - REPEATED_UPGRADE_DTESTS_COUNT: 25 + - REPEATED_ANT_TEST_TARGET: testsome + - REPEATED_ANT_TEST_CLASS: null + - REPEATED_ANT_TEST_METHODS: null + - REPEATED_ANT_TEST_VNODES: false + - REPEATED_ANT_TEST_COUNT: 500 + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j8_unit_tests_repeat: + docker: + - image: apache/cassandra-testing-ubuntu2004-java11-w-dependencies:latest + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 25 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information command: | - echo "cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt" - cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt - - source ~/env3.6/bin/activate - export PATH=$JAVA_HOME/bin:$PATH - if [ -n 'CQLSH_PYTHON=/usr/bin/python2.7' ]; then - export CQLSH_PYTHON=/usr/bin/python2.7 - fi - + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java java -version - cd ~/cassandra-dtest - mkdir -p /tmp/dtest - - echo "env: $(env)" - echo "** done env" - mkdir -p /tmp/results/dtests - # we need the "set -o pipefail" here so that the exit code that circleci will actually use is from pytest and not the exit code from tee - export SPLIT_TESTS=`cat /tmp/split_dtest_tests_j8_without_vnodes_final.txt` - set -o pipefail && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-cli-level=DEBUG --junit-xml=/tmp/results/dtests/pytest_result_j8_without_vnodes.xml -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1 | tee /tmp/dtest/stdout.txt + - run: + name: Repeatedly run new or modifed JUnit tests + no_output_timeout: 15m + command: "set -x\nexport PATH=$JAVA_HOME/bin:$PATH\ntime mv ~/cassandra /tmp\ncd /tmp/cassandra\nif [ -d ~/dtest_jars ]; then\n cp ~/dtest_jars/dtest* /tmp/cassandra/build/\nfi\n\n# Calculate the number of test iterations to be run by the current parallel runner.\ncount=$((${REPEATED_UTESTS_COUNT} / CIRCLE_NODE_TOTAL))\nif (($CIRCLE_NODE_INDEX < (${REPEATED_UTESTS_COUNT} % CIRCLE_NODE_TOTAL))); then\n count=$((count+1))\nfi\n\n# Put manually specified tests and automatically d [...] - store_test_results: - path: /tmp/results + path: /tmp/results/repeated_utests/output - store_artifacts: - path: /tmp/dtest - destination: dtest_j8_without_vnodes + path: /tmp/results/repeated_utests/stdout + destination: stdout - store_artifacts: - path: ~/cassandra-dtest/logs - destination: dtest_j8_without_vnodes_logs + path: /tmp/results/repeated_utests/output + destination: junitxml + - store_artifacts: + path: /tmp/results/repeated_utests/logs + destination: logs environment: - ANT_HOME: /usr/share/ant - JAVA11_HOME: /usr/lib/jvm/java-11-openjdk-amd64 @@@ -3278,7 -3889,7 +3278,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-compression -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -4458,7 -5467,7 +4458,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -4575,7 -5581,7 +4575,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -5581,7 -6668,7 +5581,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit - ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true ++ ant testclasslist-cdc -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -5830,7 -6911,7 +5830,7 @@@ if [ -z "$test_timeout" ]; then test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"' '{print $4}') fi - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed - ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true ++ ant testclasslist -Dtest.timeout="$test_timeout" -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=distributed -Dno-build-test=true no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ diff --cc build.xml index 00a19abcdc,e11f5f6909..3659911d33 --- a/build.xml +++ b/build.xml @@@ -1040,60 -1007,6 +1040,60 @@@ </testmacro> </target> + <!-- + simulator asm build file + --> + <property name="simulator-asm.build.src" value="${test.simulator-asm.src}" /> + <property name="simulator-asm.build.classes" value="${build.classes}/simulator-asm" /> + <property name="simulator-asm.manifest" value="${simulator-asm.build.classes}/MANIFEST.MF" /> + + <property name="simulator-bootstrap.build.src" value="${test.simulator-bootstrap.src}" /> + <property name="simulator-bootstrap.build.classes" value="${build.classes}/simulator-bootstrap" /> + <property name="simulator-bootstrap.manifest" value="${simulator-bootstrap.build.classes}/MANIFEST.MF" /> + + <target name="simulator-asm-build" depends="_build_unsafe" description="build simulator-asm"> + <antcall target="_simulator-asm_build"/> + </target> + + <target name="simulator-bootstrap-build" depends="_build_unsafe" description="build simulator-bootstrap"> + <antcall target="_simulator-bootstrap_build"/> + </target> + + <target name="_simulator-asm_build"> + <mkdir dir="${simulator-asm.build.classes}" /> + <javac compiler="modern" debug="true" debuglevel="${debuglevel}" + source="${source.version}" target="${target.version}" + encoding="utf-8" destdir="${simulator-asm.build.classes}" includeantruntime="true"> + <src path="${simulator-asm.build.src}" /> + <classpath> + <fileset dir="${test.lib}"> - <include name="**/asm-*${asm.version}.jar" /> ++ <include name="**/asm-*${asm.version}.jar" /> + </fileset> + <fileset dir="${build.lib}"> + <include name="**/asm-*${asm.version}.jar" /> + </fileset> + </classpath> + </javac> + </target> + + <target name="_simulator-bootstrap_build"> + <mkdir dir="${simulator-bootstrap.build.classes}" /> + <javac compiler="modern" debug="true" debuglevel="${debuglevel}" + source="${source.version}" target="${target.version}" + encoding="utf-8" destdir="${simulator-bootstrap.build.classes}" includeantruntime="true"> + <src path="${simulator-bootstrap.build.src}" /> + <classpath> + <fileset dir="${test.lib}"> + <include name="**/asm-*${asm.version}.jar" /> + </fileset> + <fileset dir="${build.lib}"> + <include name="**/asm-*${asm.version}.jar" /> + </fileset> + </classpath> + <compilerarg value="-XDignore.symbol.file"/> + </javac> + </target> + <target name="_write-poms" depends="maven-declare-dependencies"> <artifact:writepom pomRefId="parent-pom" file="${build.dir}/${final.name}-parent.pom"/> <artifact:writepom pomRefId="all-pom" file="${build.dir}/${final.name}.pom"/> @@@ -1390,7 -1256,23 +1390,23 @@@ </jar> </target> - <target name="build-test" depends="_main-jar,stress-build,fqltool-build,resolver-dist-lib,simulator-jars,checkstyle-test" unless="no-build-test" + <!-- Wrapper of build-test without dependencies, so both that target and its dependencies are skipped if the property + no-build-test is true. This is meant to be used to run tests without actually building them, provided that they have + been built before. All test targets depend on this, so one can run them using the no-build-test property. + For example: + + ant test -Dno-build-test=true + ant test -Dtest.name=StorageServiceServerTest -Dno-build-test=true + ant testsome -Dtest.name=org.apache.cassandra.service.StorageServiceServerTest -Dno-build-test=true + + The property is false by default, so if it's not specified the tests would be built with all their dependencies. + See CASSANDRA-16625 and CASSANDRA-18000 for further details. + --> + <target name="maybe-build-test" unless="no-build-test"> + <antcall target="build-test"/> + </target> + - <target name="build-test" depends="_main-jar,stress-build,fqltool-build,resolver-dist-lib" ++ <target name="build-test" depends="_main-jar,stress-build,fqltool-build,resolver-dist-lib,simulator-jars,checkstyle-test" description="Compile test classes"> <antcall target="_build-test"/> </target> @@@ -1653,10 -1540,9 +1669,10 @@@ </target> <!-- Use this with an FQDN for test class, and an optional csv list of methods like this: + ant testsome -Dtest.name=org.apache.cassandra.service.StorageServiceServerTest ant testsome -Dtest.name=org.apache.cassandra.service.StorageServiceServerTest -Dtest.methods=testRegularMode,testGetAllRangesEmpty --> - <target name="testsome" depends="build-test" description="Execute specific unit tests" > + <target name="testsome" depends="maybe-build-test" description="Execute specific unit tests" > <testmacro inputdir="${test.unit.src}" timeout="${test.timeout}"> <test unless:blank="${test.methods}" name="${test.name}" methods="${test.methods}" outfile="build/test/output/TEST-${test.name}-${test.methods}"/> <test if:blank="${test.methods}" name="${test.name}" outfile="build/test/output/TEST-${test.name}"/> @@@ -1808,10 -1694,10 +1824,10 @@@ </target> <!-- Use this with an simple class name for test class, and an optional csv list of methods like this: - ant cql-test-some -Dtest.name=ListsTest - ant cql-test-some -Dtest.name=ListsTest -Dtest.methods=testPrecisionTime_getNext_simple - --> + ant cql-test-some -Dtest.name=ListsTest + ant cql-test-some -Dtest.name=ListsTest -Dtest.methods=testPrecisionTime_getNext_simple + --> - <target name="cql-test-some" depends="build-test" description="Execute specific CQL tests" > + <target name="cql-test-some" depends="maybe-build-test" description="Execute specific CQL tests" > <sequential> <echo message="running ${test.methods} tests from ${test.name}"/> <mkdir dir="${build.test.dir}/cassandra"/> @@@ -1997,8 -1882,8 +2013,8 @@@ </jar> </target> - <target name="test-jvm-dtest" depends="build-test" description="Execute in-jvm dtests"> + <target name="test-jvm-dtest" depends="maybe-build-test" description="Execute in-jvm dtests"> - <testmacro inputdir="${test.distributed.src}" timeout="${test.distributed.timeout}" forkmode="once" showoutput="true" filter="**/test/*Test.java"> + <testmacro inputdir="${test.distributed.src}" timeout="${test.distributed.timeout}" forkmode="once" showoutput="true" filter="**/test/${test.name}.java"> <jvmarg value="-Dlogback.configurationFile=test/conf/logback-dtest.xml"/> <jvmarg value="-Dcassandra.ring_delay_ms=10000"/> <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/> @@@ -2006,28 -1891,7 +2022,28 @@@ </testmacro> </target> - <target name="test-simulator-dtest" depends="build-test" description="Execute simulator dtests"> ++ <target name="test-simulator-dtest" depends="maybe-build-test" description="Execute simulator dtests"> + <testmacro inputdir="${test.simulator-test.src}" timeout="${test.simulation.timeout}" forkmode="perTest" showoutput="true" filter="**/test/${test.name}.java"> + <jvmarg value="-Dlogback.configurationFile=test/conf/logback-simulator.xml"/> + <jvmarg value="-Dcassandra.ring_delay_ms=10000"/> + <jvmarg value="-Dcassandra.tolerate_sstable_size=true"/> + <jvmarg value="-Dcassandra.skip_sync=true" /> + <jvmarg value="-Dcassandra.debugrefcount=false"/> + <jvmarg value="-Dcassandra.test.simulator.determinismcheck=strict"/> + <!-- Support Simulator Tests --> + <jvmarg line="-javaagent:${test.lib}/jars/simulator-asm.jar"/> + <jvmarg line="-Xbootclasspath/a:${test.lib}/jars/simulator-bootstrap.jar"/> + <jvmarg line="-XX:ActiveProcessorCount=4"/> + <jvmarg line="-XX:-TieredCompilation"/> + <jvmarg line="-XX:-BackgroundCompilation"/> + <jvmarg line="-XX:CICompilerCount=1"/> + <jvmarg line="-XX:Tier4CompileThreshold=1000"/> + <jvmarg line="-XX:ReservedCodeCacheSize=256M"/> + <jvmarg line="-Xmx8G"/> + </testmacro> + </target> + - <target name="test-jvm-upgrade-dtest" depends="build-test" description="Execute in-jvm dtests"> + <target name="test-jvm-upgrade-dtest" depends="maybe-build-test" description="Execute in-jvm dtests"> <testmacro inputdir="${test.distributed.src}" timeout="${test.distributed.timeout}" forkmode="once" showoutput="true" filter="**/upgrade/*Test.java"> <jvmarg value="-Dlogback.configurationFile=test/conf/logback-dtest.xml"/> <jvmarg value="-Dcassandra.ring_delay_ms=10000"/> @@@ -2037,10 -1901,9 +2053,10 @@@ </target> <!-- Use this with an FQDN for test class, and an optional csv list of methods like this: + ant test-jvm-dtest-some -Dtest.name=org.apache.cassandra.distributed.test.ResourceLeakTest ant test-jvm-dtest-some -Dtest.name=org.apache.cassandra.distributed.test.ResourceLeakTest -Dtest.methods=looperTest --> - <target name="test-jvm-dtest-some" depends="build-test" description="Execute some in-jvm dtests"> + <target name="test-jvm-dtest-some" depends="maybe-build-test" description="Execute some in-jvm dtests"> <testmacro inputdir="${test.distributed.src}" timeout="${test.distributed.timeout}" forkmode="once" showoutput="true"> <test unless:blank="${test.methods}" name="${test.name}" methods="${test.methods}" outfile="build/test/output/TEST-${test.name}-${test.methods}"/> <test if:blank="${test.methods}" name="${test.name}" outfile="build/test/output/TEST-${test.name}"/> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
