This is an automated email from the ASF dual-hosted git repository. marcuse pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 8da277e573bcfcbeee7b190a23a63fc13ce2fc72 Merge: 9d2ffad 3bb388b Author: Marcus Eriksson <[email protected]> AuthorDate: Thu Jan 16 11:03:33 2020 +0100 Merge branch 'cassandra-3.11' into trunk .circleci/config-2_1.yml | 74 ++++++ .circleci/config.yml | 263 +++++++++++---------- .circleci/config.yml.HIGHRES | 253 +++++++++++--------- .circleci/config.yml.LOWRES | 253 +++++++++++--------- CHANGES.txt | 1 + build.xml | 36 +++ .../cassandra/distributed/test/TestLocator.java | 8 +- 7 files changed, 547 insertions(+), 341 deletions(-) diff --cc .circleci/config-2_1.yml index d0bd003,b55b1fb..ee0b914 --- a/.circleci/config-2_1.yml +++ b/.circleci/config-2_1.yml @@@ -28,132 -29,77 +28,142 @@@ j8_seq_executor: &j8_seq_executo #exec_resource_class: xlarge parallelism: 1 # sequential, single container tests: no parallelism benefits -with_dtests_jobs: &with_dtest_jobs - jobs: - - build - # Java 8 unit tests will be run automatically - - j8_unit_tests: - requires: - - build - - j8_jvm_dtests: - requires: - - build - # specialized unit tests (all run on request using Java 8) - - start_utests_long: - type: approval - requires: - - build - - utests_long: - requires: - - start_utests_long - - start_utests_compression: - type: approval - requires: - - build - - utests_compression: - requires: - - start_utests_compression - - start_utests_stress: - type: approval - requires: - - build - - utests_stress: - requires: - - start_utests_stress - - start_jvm_upgrade_dtest: - type: approval - requires: - - build - - dtest_jars_build: - requires: - - start_jvm_upgrade_dtest - - j8_jvm_upgrade_dtests: - requires: - - dtest_jars_build - # Java 8 dtests (on request) - - start_j8_dtests: - type: approval - requires: - - build - - j8_dtests-with-vnodes: - requires: - - start_j8_dtests - - j8_dtests-no-vnodes: - requires: - - start_j8_dtests - # Java 8 upgrade tests - - start_upgrade_tests: - type: approval - requires: - - build - - j8_upgradetests-no-vnodes: - requires: - - start_upgrade_tests +j11_par_executor: &j11_par_executor + executor: + name: java11-executor + #exec_resource_class: xlarge + parallelism: 4 -with_dtest_jobs_only: &with_dtest_jobs_only +j8_with_dtests_jobs: &j8_with_dtests_jobs + jobs: + - j8_build + # Java 8 unit tests will be run automatically + - j8_unit_tests: + requires: + - j8_build + - j8_jvm_dtests: + requires: + - j8_build + # Java 11 unit tests (on request, currently not working) + - start_j11_unit_tests: + type: approval + requires: + - j8_build + - j11_unit_tests: + requires: + - start_j11_unit_tests + # specialized unit tests (all run on request using Java 8) + - start_utests_long: + type: approval + requires: + - j8_build + - utests_long: + requires: + - start_utests_long + - start_utests_compression: + type: approval + requires: + - j8_build + - utests_compression: + requires: + - start_utests_compression + - start_utests_stress: + type: approval + requires: + - j8_build + - utests_stress: + requires: + - start_utests_stress + - start_utests_fqltool: + type: approval + requires: + - j8_build + - utests_fqltool: + requires: + - start_utests_fqltool ++ - start_jvm_upgrade_dtest: ++ type: approval ++ requires: ++ - j8_build ++ - dtest_jars_build: ++ requires: ++ - start_jvm_upgrade_dtest ++ - j8_jvm_upgrade_dtests: ++ requires: ++ - dtest_jars_build + # Java 8 dtests (on request) + - start_j8_dtests: + type: approval + requires: + - j8_build + - j8_dtests-with-vnodes: + requires: + - start_j8_dtests + - j8_dtests-no-vnodes: + requires: + - start_j8_dtests + # Java 11 dtests (on request) + - start_j11_dtests: + type: approval + requires: + - j8_build + - j11_dtests-with-vnodes: + requires: + - start_j11_dtests + - j11_dtests-no-vnodes: + requires: + - start_j11_dtests + # Java 8 upgrade tests + - start_upgrade_tests: + type: approval + requires: + - j8_build + - j8_upgradetests-no-vnodes: + requires: + - start_upgrade_tests + +j11_with_dtests_jobs: &j11_with_dtests_jobs + jobs: + - j11_build + # Java 11 unit tests (on request, currently not working) + - start_j11_unit_tests: + type: approval + requires: + - j11_build + - j11_unit_tests: + requires: + - start_j11_unit_tests + # Java 11 dtests (on request) + - start_j11_dtests: + type: approval + requires: + - j11_build + - j11_dtests-with-vnodes: + requires: + - start_j11_dtests + - j11_dtests-no-vnodes: + requires: + - start_j11_dtests + +j8_with_dtest_jobs_only: &j8_with_dtest_jobs_only jobs: - - build + - j8_build - j8_dtests-with-vnodes: requires: - - build + - j8_build - j8_dtests-no-vnodes: requires: - - build + - j8_build + +j11_with_dtest_jobs_only: &j11_with_dtest_jobs_only + jobs: + - build + - j11-with-vnodes: + requires: + - j11_build + - j11_dtests-no-vnodes: + requires: + - j11_build workflows: version: 2 @@@ -178,44 -122,33 +188,56 @@@ executors JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + java11-executor: + parameters: + exec_resource_class: + type: string + default: medium + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: << parameters.exec_resource_class >> + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + environment: + <<: *default_env_vars + JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + CASSANDRA_USE_JDK11: true + +build_common: &build_common + parallelism: 1 # This job doesn't benefit from parallelism + steps: + - log_environment + - clone_cassandra + - build_cassandra + - run_eclipse_warnings + - persist_to_workspace: + root: /home/cassandra + paths: + - cassandra + - .m2 + jobs: - build: + j8_build: executor: java8-executor - parallelism: 1 # This job doesn't benefit from parallelism - steps: - - log_environment - - clone_cassandra - - build_cassandra - - run_eclipse_warnings - - persist_to_workspace: - root: /home/cassandra - paths: - - cassandra - - .m2 + <<: *build_common + + j11_build: + executor: java11-executor + <<: *build_common + dtest_jars_build: + executor: java8-executor + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - build_cassandra_dtest_jars + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + j8_unit_tests: <<: *j8_par_executor steps: @@@ -233,15 -166,14 +255,23 @@@ - run_junit_tests: target: test-jvm-dtest-forking + j11_unit_tests: + <<: *j11_par_executor + steps: + - attach_workspace: + at: /home/cassandra + - create_junit_containers + - log_environment + - run_parallel_junit_tests + + j8_jvm_upgrade_dtests: + <<: *j8_seq_executor + steps: + - attach_workspace: + at: /home/cassandra + - run_junit_tests: + target: test-jvm-upgrade-dtest-forking + utests_long: <<: *j8_seq_executor steps: @@@ -459,7 -396,10 +530,10 @@@ commands export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean <<parameters.target>> + ant <<parameters.target>> no_output_timeout: <<parameters.no_output_timeout>> - store_test_results: path: /tmp/cassandra/build/test/output/ diff --cc .circleci/config.yml index 53d18d5,e0c65f4..0a02e3a --- a/.circleci/config.yml +++ b/.circleci/config.yml @@@ -1,8 -1,54 +1,53 @@@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-upgrade-dtest-forking ++ ant test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - build: + j11_unit_tests: docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 resource_class: medium working_directory: ~/ shell: /bin/bash -eo pipefail -l @@@ -52,127 -77,41 +97,102 @@@ which java java -version - run: - name: Clone Cassandra Repository (via git) - command: | - git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra - - run: - name: Build Cassandra + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant clean jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_dtests-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra - run: - name: Run eclipse-warnings + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - ant eclipse-warnings - - persist_to_workspace: - root: /home/cassandra - paths: - - cassandra - - .m2 + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" ++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-re [...] + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" ++ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant - LANG: en_US.UTF-8 - KEEP_TEST_DIR: true @@@ -388,7 -300,10 +380,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean stress-test + ant stress-test no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -497,89 -413,7 +491,88 @@@ - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra Repository (via git) + command: | + git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra + - run: + name: Build Cassandra + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean realclean jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + no_output_timeout: 15m + - run: + name: Run eclipse-warnings + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + ant eclipse-warnings + - persist_to_workspace: + root: /home/cassandra + paths: + - cassandra + - .m2 + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true j8_dtests-with-vnodes: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@@ -679,7 -488,10 +646,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-dtest-forking + ant test-jvm-dtest-forking no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -690,87 -502,7 +660,86 @@@ path: /tmp/cassandra/build/test/logs destination: logs environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j8_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra Repository (via git) + command: | + git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra + - run: + name: Build Cassandra + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean realclean jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + no_output_timeout: 15m + - run: + name: Run eclipse-warnings + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + ant eclipse-warnings + - persist_to_workspace: + root: /home/cassandra + paths: + - cassandra + - .m2 + environment: - ANT_HOME: /usr/share/ant - LANG: en_US.UTF-8 - KEEP_TEST_DIR: true @@@ -803,50 -534,10 +771,55 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean long-test + ant long-test + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_fqltool: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (fqltool-test) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi + ant fqltool-test no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -871,114 -563,7 +844,87 @@@ - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_dtests-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes\ - \ > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" ++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-int [...] + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ - \ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset -o pipefail\ - \ && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" ++ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true utests_compression: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@@ -1061,7 -647,75 +1007,74 @@@ - CCM_MAX_HEAP_SIZE: 1024M - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 @@@ -1102,13 -749,16 +1115,23 @@@ - utests_stress: requires: - start_utests_stress + - start_utests_fqltool: + type: approval + requires: + - j8_build + - utests_fqltool: + requires: + - start_utests_fqltool + - start_jvm_upgrade_dtest: + type: approval + requires: - - build ++ - j8_build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - start_j8_dtests: type: approval requires: diff --cc .circleci/config.yml.HIGHRES index 8ac50c9,c6688d9..b96a718 --- a/.circleci/config.yml.HIGHRES +++ b/.circleci/config.yml.HIGHRES @@@ -1,34 -1,59 +1,79 @@@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-upgrade-dtest-forking ++ ant test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - build: + j11_unit_tests: docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 - resource_class: medium + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: xlarge working_directory: ~/ shell: /bin/bash -eo pipefail -l - parallelism: 1 + parallelism: 100 steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Determine Unit Tests to Run + command: | + # reminder: this code (along with all the steps) is independently executed on every circle container + # so the goal here is to get the circleci script to return the tests *this* container will run + # which we do via the `circleci` cli tool. + + rm -fr ~/cassandra-dtest/upgrade_tests + echo "***java tests***" + + # get all of our unit test filenames + set -eo pipefail && circleci tests glob "$HOME/cassandra/test/unit/**/*.java" > /tmp/all_java_unit_tests.txt + + # split up the unit tests into groups based on the number of containers we have + set -eo pipefail && circleci tests split --split-by=timings --timings-type=filename --index=${CIRCLE_NODE_INDEX} --total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt > /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt + set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt | cut -c 37-1000000 | grep "Test\.java$" > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt" + cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt + no_output_timeout: 15m - run: name: Log Environment Information command: | @@@ -52,127 -77,41 +97,102 @@@ which java java -version - run: - name: Clone Cassandra Repository (via git) - command: | - git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra - - run: - name: Build Cassandra + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant clean jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_dtests-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra - run: - name: Run eclipse-warnings + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra dtest Repository (via git) command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - ant eclipse-warnings - - persist_to_workspace: - root: /home/cassandra - paths: - - cassandra - - .m2 + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" ++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-re [...] + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" ++ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant - LANG: en_US.UTF-8 - KEEP_TEST_DIR: true @@@ -386,7 -300,10 +380,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean stress-test + ant stress-test no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -674,7 -488,10 +646,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-dtest-forking + ant test-jvm-dtest-forking no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -796,7 -534,10 +771,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean long-test + ant long-test no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -822,154 -564,6 +800,131 @@@ - CCM_HEAP_NEWSIZE: 512M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_fqltool: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (fqltool-test) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi + ant fqltool-test + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_dtests-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: xlarge + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 100 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes\ - \ > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" ++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-int [...] + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ - \ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset -o pipefail\ - \ && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" ++ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true utests_compression: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@@ -1053,24 -648,86 +1008,92 @@@ - CCM_HEAP_NEWSIZE: 512M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 2048M + - CCM_HEAP_NEWSIZE: 512M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 - build_and_run_tests: + java8_build_and_run_tests: jobs: - - build + - j8_build - j8_unit_tests: requires: - - build + - j8_build - j8_jvm_dtests: requires: - - build + - j8_build + - start_j11_unit_tests: + type: approval + requires: + - j8_build + - j11_unit_tests: + requires: + - start_j11_unit_tests - start_utests_long: type: approval requires: @@@ -1092,13 -749,16 +1115,23 @@@ - utests_stress: requires: - start_utests_stress + - start_utests_fqltool: + type: approval + requires: + - j8_build + - utests_fqltool: + requires: + - start_utests_fqltool + - start_jvm_upgrade_dtest: + type: approval + requires: - - build ++ - j8_build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - start_j8_dtests: type: approval requires: diff --cc .circleci/config.yml.LOWRES index 27fda51,e0c65f4..0a02e3a --- a/.circleci/config.yml.LOWRES +++ b/.circleci/config.yml.LOWRES @@@ -1,8 -1,54 +1,53 @@@ version: 2 jobs: + j8_jvm_upgrade_dtests: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (test-jvm-upgrade-dtest-forking) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-upgrade-dtest-forking ++ ant test-jvm-upgrade-dtest-forking + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - build: + j11_unit_tests: docker: - - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 resource_class: medium working_directory: ~/ shell: /bin/bash -eo pipefail -l @@@ -52,127 -77,41 +97,102 @@@ which java java -version - run: - name: Clone Cassandra Repository (via git) - command: | - git clone --single-branch --depth 1 --branch $CIRCLE_BRANCH git://github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME.git ~/cassandra - - run: - name: Build Cassandra + name: Run Unit Tests (testclasslist) command: | export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. - for x in $(seq 1 3); do - ${ANT_HOME}/bin/ant clean jar - RETURN="$?" - if [ "${RETURN}" -eq "0" ]; then - break - fi - done - # Exit, if we didn't build successfully - if [ "${RETURN}" -ne "0" ]; then - echo "Build failed with exit code: ${RETURN}" - exit ${RETURN} - fi + time mv ~/cassandra /tmp + cd /tmp/cassandra + ant testclasslist -Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt -Dtest.classlistprefix=unit no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true + j11_dtests-with-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra - run: - name: Run eclipse-warnings + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate export PATH=$JAVA_HOME/bin:$PATH - cd ~/cassandra - ant eclipse-warnings - - persist_to_workspace: - root: /home/cassandra - paths: - - cassandra - - .m2 + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_with_vnodes) + no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py\ - \ --use-vnodes --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j11_with_vnodes_raw --cassandra-dir=../cassandra\n\ - if [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_with_vnodes_raw /tmp/all_dtest_tests_j11_with_vnodes\n\ - else\n grep -e '' /tmp/all_dtest_tests_j11_with_vnodes_raw > /tmp/all_dtest_tests_j11_with_vnodes\ - \ || { echo \"Filter did not match any tests! Exiting build.\"; exit 0;\ - \ }\nfi\nset -eo pipefail && circleci tests split --split-by=timings --timings-type=classname\ - \ /tmp/all_dtest_tests_j11_with_vnodes > /tmp/split_dtest_tests_j11_with_vnodes.txt\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes.txt | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n" ++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_with_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --use-vnodes --skip-re [...] + - run: + name: Run dtests (j11_with_vnodes) + no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\n\ - cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j11_with_vnodes_final.txt`\nset -o pipefail &&\ - \ cd ~/cassandra-dtest && pytest --use-vnodes --num-tokens=32 --skip-resource-intensive-tests\ - \ --log-level=\"INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_with_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" ++ command: "echo \"cat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_with_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not the ex [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_with_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_with_vnodes_logs environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - ANT_HOME: /usr/share/ant - LANG: en_US.UTF-8 - KEEP_TEST_DIR: true @@@ -386,7 -300,10 +380,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean stress-test + ant stress-test no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -674,7 -488,10 +646,10 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean test-jvm-dtest-forking + ant test-jvm-dtest-forking no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -796,49 -534,10 +771,55 @@@ export PATH=$JAVA_HOME/bin:$PATH time mv ~/cassandra /tmp cd /tmp/cassandra + if [ -d ~/dtest_jars ]; then + cp ~/dtest_jars/dtest* /tmp/cassandra/build/ + fi - ant clean long-test + ant long-test + no_output_timeout: 15m + - store_test_results: + path: /tmp/cassandra/build/test/output/ + - store_artifacts: + path: /tmp/cassandra/build/test/output + destination: junitxml + - store_artifacts: + path: /tmp/cassandra/build/test/logs + destination: logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + utests_fqltool: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Run Unit Tests (fqltool-test) + command: | + export PATH=$JAVA_HOME/bin:$PATH + time mv ~/cassandra /tmp + cd /tmp/cassandra ++ if [ -d ~/dtest_jars ]; then ++ cp ~/dtest_jars/dtest* /tmp/cassandra/build/ ++ fi + ant fqltool-test no_output_timeout: 15m - store_test_results: path: /tmp/cassandra/build/test/output/ @@@ -864,112 -564,6 +845,86 @@@ - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + j11_dtests-no-vnodes: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11:20181210 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 4 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Log Environment Information + command: | + echo '*** id ***' + id + echo '*** cat /proc/cpuinfo ***' + cat /proc/cpuinfo + echo '*** free -m ***' + free -m + echo '*** df -m ***' + df -m + echo '*** ifconfig -a ***' + ifconfig -a + echo '*** uname -a ***' + uname -a + echo '*** mount ***' + mount + echo '*** env ***' + env + echo '*** java ***' + which java + java -version + - run: + name: Clone Cassandra dtest Repository (via git) + command: | + git clone --single-branch --branch $DTEST_BRANCH --depth 1 $DTEST_REPO ~/cassandra-dtest + - run: + name: Configure virtualenv and python Dependencies + command: | + # note, this should be super quick as all dependencies should be pre-installed in the docker image + # if additional dependencies were added to requirmeents.txt and the docker image hasn't been updated + # we'd have to install it here at runtime -- which will make things slow, so do yourself a favor and + # rebuild the docker image! (it automatically pulls the latest requirements.txt on build) + source ~/env/bin/activate + export PATH=$JAVA_HOME/bin:$PATH + pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt + pip3 freeze + - run: + name: Determine Tests to Run (j11_without_vnodes) + no_output_timeout: 5m - command: "# reminder: this code (along with all the steps) is independently\ - \ executed on every circle container\n# so the goal here is to get the circleci\ - \ script to return the tests *this* container will run\n# which we do via\ - \ the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\n\ - echo \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail &&\ - \ ./run_dtests.py --skip-resource-intensive-tests --dtest-print-tests-only\ - \ --dtest-print-tests-output=/tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ --cassandra-dir=../cassandra\nif [ -z '' ]; then\n mv /tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ /tmp/all_dtest_tests_j11_without_vnodes\nelse\n grep -e '' /tmp/all_dtest_tests_j11_without_vnodes_raw\ - \ > /tmp/all_dtest_tests_j11_without_vnodes || { echo \"Filter did not match\ - \ any tests! Exiting build.\"; exit 0; }\nfi\nset -eo pipefail && circleci\ - \ tests split --split-by=timings --timings-type=classname /tmp/all_dtest_tests_j11_without_vnodes\ - \ > /tmp/split_dtest_tests_j11_without_vnodes.txt\ncat /tmp/split_dtest_tests_j11_without_vnodes.txt\ - \ | tr '\\n' ' ' > /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\ - cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n" ++ command: "# reminder: this code (along with all the steps) is independently executed on every circle container\n# so the goal here is to get the circleci script to return the tests *this* container will run\n# which we do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n export \nfi\n\necho \"***Collected DTests (j11_without_vnodes)***\"\nset -eo pipefail && ./run_dtests.py --skip-resource-int [...] + - run: + name: Run dtests (j11_without_vnodes) + no_output_timeout: 15m - command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ - \ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env/bin/activate\n\ - export PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\n\ - java -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env:\ - \ $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need\ - \ the \"set -o pipefail\" here so that the exit code that circleci will\ - \ actually use is from pytest and not the exit code from tee\nexport SPLIT_TESTS=`cat\ - \ /tmp/split_dtest_tests_j11_without_vnodes_final.txt`\nset -o pipefail\ - \ && cd ~/cassandra-dtest && pytest --skip-resource-intensive-tests --log-level=\"\ - INFO\" --junit-xml=/tmp/results/dtests/pytest_result_j11_without_vnodes.xml\ - \ -s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS\ - \ 2>&1 | tee /tmp/dtest/stdout.txt\n" ++ command: "echo \"cat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\"\ncat /tmp/split_dtest_tests_j11_without_vnodes_final.txt\n\nsource ~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho \"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need the \"set -o pipefail\" here so that the exit code that circleci will actually use is from pytest and not [...] + - store_test_results: + path: /tmp/results + - store_artifacts: + path: /tmp/dtest + destination: dtest_j11_without_vnodes + - store_artifacts: + path: ~/cassandra-dtest/logs + destination: dtest_j11_without_vnodes_logs + environment: + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-11-openjdk-amd64 + - CASSANDRA_USE_JDK11: true utests_compression: docker: - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 @@@ -1053,24 -648,86 +1008,92 @@@ - CCM_HEAP_NEWSIZE: 256M - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + dtest_jars_build: + docker: + - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306 + resource_class: medium + working_directory: ~/ + shell: /bin/bash -eo pipefail -l + parallelism: 1 + steps: + - attach_workspace: + at: /home/cassandra + - run: + name: Build Cassandra DTest jars + command: | + export PATH=$JAVA_HOME/bin:$PATH + cd ~/cassandra + git remote add apache git://github.com/apache/cassandra.git + for branch in cassandra-2.2 cassandra-3.0 cassandra-3.11 trunk; do + # check out the correct cassandra version: + git remote set-branches --add apache '$branch' + git fetch --depth 1 apache $branch + git checkout $branch + # Loop to prevent failure due to maven-ant-tasks not downloading a jar.. + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + # Exit, if we didn't build successfully + if [ "${RETURN}" -ne "0" ]; then + echo "Build failed with exit code: ${RETURN}" + exit ${RETURN} + fi + done + # and build the dtest-jar for the branch under test + git checkout origin/$CIRCLE_BRANCH + for x in $(seq 1 3); do + ${ANT_HOME}/bin/ant clean jar dtest-jar + RETURN="$?" + if [ "${RETURN}" -eq "0" ]; then + break + fi + done + mkdir ~/dtest_jars + cp build/dtest*.jar ~/dtest_jars + ls -l ~/dtest_jars + no_output_timeout: 15m + - persist_to_workspace: + root: /home/cassandra + paths: + - dtest_jars + environment: - - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - ANT_HOME: /usr/share/ant + - LANG: en_US.UTF-8 + - KEEP_TEST_DIR: true + - DEFAULT_DIR: /home/cassandra/cassandra-dtest + - PYTHONIOENCODING: utf-8 + - PYTHONUNBUFFERED: true + - CASS_DRIVER_NO_EXTENSIONS: true + - CASS_DRIVER_NO_CYTHON: true + - CASSANDRA_SKIP_SYNC: true + - DTEST_REPO: git://github.com/apache/cassandra-dtest.git + - DTEST_BRANCH: master + - CCM_MAX_HEAP_SIZE: 1024M + - CCM_HEAP_NEWSIZE: 256M + - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64 + - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64 workflows: version: 2 - build_and_run_tests: + java8_build_and_run_tests: jobs: - - build + - j8_build - j8_unit_tests: requires: - - build + - j8_build - j8_jvm_dtests: requires: - - build + - j8_build + - start_j11_unit_tests: + type: approval + requires: + - j8_build + - j11_unit_tests: + requires: + - start_j11_unit_tests - start_utests_long: type: approval requires: @@@ -1092,13 -749,16 +1115,23 @@@ - utests_stress: requires: - start_utests_stress + - start_utests_fqltool: + type: approval + requires: + - j8_build + - utests_fqltool: + requires: + - start_utests_fqltool + - start_jvm_upgrade_dtest: + type: approval + requires: - - build ++ - j8_build + - dtest_jars_build: + requires: + - start_jvm_upgrade_dtest + - j8_jvm_upgrade_dtests: + requires: + - dtest_jars_build - start_j8_dtests: type: approval requires: diff --cc CHANGES.txt index 522edf8,54f09a1..f7ecf79 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,29 -1,10 +1,30 @@@ -3.11.6 +4.0-alpha3 + * Improve error when JVM 11 can't access required modules (CASSANDRA-15468) + * Better handling of file deletion failures by DiskFailurePolicy (CASSANDRA-15143) + * Prevent read repair mutations from increasing read timeout (CASSANDRA-15442) + * Document 4.0 system keyspace changes, bump generations (CASSANDRA-15454) + * Make it possible to disable STCS-in-L0 during runtime (CASSANDRA-15445) + * Removed obsolete OldNetworkTopologyStrategy (CASSANDRA-13990) + * Align record header of FQL and audit binary log (CASSANDRA-15076) + * Shuffle forwarding replica for messages to non-local DC (CASSANDRA-15318) + * Optimise native protocol ASCII string encoding (CASSANDRA-15410) + * Make sure all exceptions are propagated in DebuggableThreadPoolExecutor (CASSANDRA-15332) + * Make it possible to resize concurrent read / write thread pools at runtime (CASSANDRA-15277) + * Close channels on error (CASSANDRA-15407) + * Add documentation for Java 11 support in Cassandra (CASSANDRA-15428) + * Integrate SJK into nodetool (CASSANDRA-12197) + * Ensure that empty clusterings with kind==CLUSTERING are Clustering.EMPTY (CASSANDRA-15498) + * The flag 'cross_node_timeout' has been set as true by default. This change + is done under the assumption that users have setup NTP on their clusters or + otherwise synchronize their clocks, and that clocks are mostly in sync, since + this is a requirement for general correctness of last write wins. (CASSANDRA-15216) +Merged from 3.11: * Fix nodetool compactionstats showing extra pending task for TWCS - patch implemented (CASSANDRA-15409) * Fix SELECT JSON formatting for the "duration" type (CASSANDRA-15075) - * Fix LegacyLayout to have same behavior as 2.x when handling unknown column names (CASSANDRA-15081) * Update nodetool help stop output (CASSANDRA-15401) Merged from 3.0: + * Fix race condition when setting bootstrap flags (CASSANDRA-14878) + * Run in-jvm upgrade dtests in circleci (CASSANDRA-15506) * Include updates to static column in mutation size calculations (CASSANDRA-15293) * Fix point-in-time recoevery ignoring timestamp of updates to static columns (CASSANDRA-15292) * GC logs are also put under $CASSANDRA_LOG_DIR (CASSANDRA-14306) diff --cc build.xml index a458023,eaedc7a..51ba2f5 --- a/build.xml +++ b/build.xml @@@ -62,7 -66,9 +62,10 @@@ <property name="test.burn.src" value="${test.dir}/burn"/> <property name="test.microbench.src" value="${test.dir}/microbench"/> <property name="test.distributed.src" value="${test.dir}/distributed"/> + <property name="test.compression_algo" value="LZ4"/> + <property name="test.distributed.listfile" value="ant-jvm-dtest-list"/> + <property name="test.distributed.upgrade.listfile" value="ant-jvm-dtest-upgrade-list"/> + <property name="test.distributed.upgrade.package" value="org.apache.cassandra.distributed.upgrade"/> <property name="dist.dir" value="${build.dir}/dist"/> <property name="tmp.dir" value="${java.io.tmpdir}"/> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
