This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch cassandra-3.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit f139c6a6c171ec4a991f21da1dc97376bbf1d264
Merge: d7f0361 5d26980
Author: Mick Semb Wever <m...@apache.org>
AuthorDate: Tue Oct 27 10:47:05 2020 +0100
Merge branch 'cassandra-2.2' into cassandra-3.0
.circleci/config-2_1.yml | 2 +-
.circleci/config.yml | 20 ++++++++++----------
.circleci/config.yml.HIGHRES | 20 ++++++++++----------
.circleci/config.yml.LOWRES | 20 ++++++++++----------
.jenkins/Jenkinsfile | 2 +-
5 files changed, 32 insertions(+), 32 deletions(-)
diff --cc .circleci/config.yml
index f8526be,0997d8a..b8aa6ea
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@@ -1,97 -1,5 +1,97 @@@
version: 2
jobs:
+ j8_jvm_upgrade_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently
executed on every circle container
+ # so the goal here is to get the circleci script to return the
tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob
"$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of
containers we have
+ set -eo pipefail && circleci tests split --split-by=timings
--timings-type=filename --index=${CIRCLE_NODE_INDEX}
--total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt >
/tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" |
grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml |
awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -173,163 -75,6 +173,163 @@@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --skip-resource-inte [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat
/tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest and not th [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently
executed on every circle container
+ # so the goal here is to get the circleci script to return the
tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests
--dtest-print-tests-only
--dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw
--cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests'
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw >
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not
match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings
--timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes >
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr
'\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that
circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest
--execute-upgrade-tests --log-level="INFO"
--junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml
-s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1
| tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -382,87 -127,26 +382,87 @@@
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk
-F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
command: |
- set -x
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk
-F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
- fi
- ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=unit
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --use-vnodes --skip-res [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
+ command: "echo \"cat
/tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest and not the exit [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
diff --cc .circleci/config.yml.HIGHRES
index 2e40513,64e2e74..c21ad30
--- a/.circleci/config.yml.HIGHRES
+++ b/.circleci/config.yml.HIGHRES
@@@ -1,97 -1,5 +1,97 @@@
version: 2
jobs:
+ j8_jvm_upgrade_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently
executed on every circle container
+ # so the goal here is to get the circleci script to return the
tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob
"$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of
containers we have
+ set -eo pipefail && circleci tests split --split-by=timings
--timings-type=filename --index=${CIRCLE_NODE_INDEX}
--total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt >
/tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" |
grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml |
awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -173,163 -75,6 +173,163 @@@
- CCM_HEAP_NEWSIZE: 512M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --skip-resource-inte [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat
/tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest and not th [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently
executed on every circle container
+ # so the goal here is to get the circleci script to return the
tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests
--dtest-print-tests-only
--dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw
--cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests'
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw >
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not
match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings
--timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes >
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr
'\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that
circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest
--execute-upgrade-tests --log-level="INFO"
--junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml
-s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1
| tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -382,87 -127,26 +382,87 @@@
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk
-F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 2048M
+ - CCM_HEAP_NEWSIZE: 512M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: xlarge
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 100
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
command: |
- set -x
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk
-F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
- fi
- ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=unit
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --use-vnodes --skip-res [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
+ command: "echo \"cat
/tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest and not the exit [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
diff --cc .circleci/config.yml.LOWRES
index f8526be,0997d8a..b8aa6ea
--- a/.circleci/config.yml.LOWRES
+++ b/.circleci/config.yml.LOWRES
@@@ -1,97 -1,5 +1,97 @@@
version: 2
jobs:
+ j8_jvm_upgrade_dtests:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 1
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Determine distributed Tests to Run
+ command: |
+ # reminder: this code (along with all the steps) is independently
executed on every circle container
+ # so the goal here is to get the circleci script to return the
tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ rm -fr ~/cassandra-dtest/upgrade_tests
+ echo "***java tests***"
+
+ # get all of our unit test filenames
+ set -eo pipefail && circleci tests glob
"$HOME/cassandra/test/distributed/**/*.java" > /tmp/all_java_unit_tests.txt
+
+ # split up the unit tests into groups based on the number of
containers we have
+ set -eo pipefail && circleci tests split --split-by=timings
--timings-type=filename --index=${CIRCLE_NODE_INDEX}
--total=${CIRCLE_NODE_TOTAL} /tmp/all_java_unit_tests.txt >
/tmp/java_tests_${CIRCLE_NODE_INDEX}.txt
+ set -eo pipefail && cat /tmp/java_tests_${CIRCLE_NODE_INDEX}.txt |
sed "s;^/home/cassandra/cassandra/test/distributed/;;g" | grep "Test\.java$" |
grep upgrade > /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ echo "** /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt"
+ cat /tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
+ no_output_timeout: 15m
+ - run:
+ name: Log Environment Information
+ command: |
+ echo '*** id ***'
+ id
+ echo '*** cat /proc/cpuinfo ***'
+ cat /proc/cpuinfo
+ echo '*** free -m ***'
+ free -m
+ echo '*** df -m ***'
+ df -m
+ echo '*** ifconfig -a ***'
+ ifconfig -a
+ echo '*** uname -a ***'
+ uname -a
+ echo '*** mount ***'
+ mount
+ echo '*** env ***'
+ env
+ echo '*** java ***'
+ which java
+ java -version
+ - run:
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.distributed.timeout"' build.xml |
awk -F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=distributed
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
build:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -173,163 -75,6 +173,163 @@@
- CCM_HEAP_NEWSIZE: 256M
- JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_without_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_without_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --skip-resource-inte [...]
+ - run:
+ name: Run dtests (j8_without_vnodes)
+ no_output_timeout: 15m
+ command: "echo \"cat
/tmp/split_dtest_tests_j8_without_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_without_vnodes_final.txt\n\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest and not th [...]
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_upgradetests-no-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
+ command: |
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_upgradetests_without_vnodes)
+ no_output_timeout: 5m
+ command: |
+ # reminder: this code (along with all the steps) is independently
executed on every circle container
+ # so the goal here is to get the circleci script to return the
tests *this* container will run
+ # which we do via the `circleci` cli tool.
+
+ cd cassandra-dtest
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ echo "***Collected DTests (j8_upgradetests_without_vnodes)***"
+ set -eo pipefail && ./run_dtests.py --execute-upgrade-tests
--dtest-print-tests-only
--dtest-print-tests-output=/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw
--cassandra-dir=../cassandra
+ if [ -z '^upgrade_tests' ]; then
+ mv /tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes
+ else
+ grep -e '^upgrade_tests'
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes_raw >
/tmp/all_dtest_tests_j8_upgradetests_without_vnodes || { echo "Filter did not
match any tests! Exiting build."; exit 0; }
+ fi
+ set -eo pipefail && circleci tests split --split-by=timings
--timings-type=classname /tmp/all_dtest_tests_j8_upgradetests_without_vnodes >
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes.txt | tr
'\n' ' ' > /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+ - run:
+ name: Run dtests (j8_upgradetests_without_vnodes)
+ no_output_timeout: 15m
+ command: |
+ echo "cat
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt"
+ cat /tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt
+
+ source ~/env/bin/activate
+ export PATH=$JAVA_HOME/bin:$PATH
+ if [ -n 'RUN_STATIC_UPGRADE_MATRIX=true' ]; then
+ export RUN_STATIC_UPGRADE_MATRIX=true
+ fi
+
+ java -version
+ cd ~/cassandra-dtest
+ mkdir -p /tmp/dtest
+
+ echo "env: $(env)"
+ echo "** done env"
+ mkdir -p /tmp/results/dtests
+ # we need the "set -o pipefail" here so that the exit code that
circleci will actually use is from pytest and not the exit code from tee
+ export SPLIT_TESTS=`cat
/tmp/split_dtest_tests_j8_upgradetests_without_vnodes_final.txt`
+ set -o pipefail && cd ~/cassandra-dtest && pytest
--execute-upgrade-tests --log-level="INFO"
--junit-xml=/tmp/results/dtests/pytest_result_j8_upgradetests_without_vnodes.xml
-s --cassandra-dir=/home/cassandra/cassandra --keep-test-dir $SPLIT_TESTS 2>&1
| tee /tmp/dtest/stdout.txt
+ - store_test_results:
+ path: /tmp/results
+ - store_artifacts:
+ path: /tmp/dtest
+ destination: dtest_j8_upgradetests_without_vnodes
+ - store_artifacts:
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_upgradetests_without_vnodes_logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
j8_unit_tests:
docker:
- image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
@@@ -382,87 -127,26 +382,87 @@@
which java
java -version
- run:
- name: Run Unit Tests (testclasslist)
+ name: Run Unit Tests (testclasslist)
+ command: |
+ set -x
+ export PATH=$JAVA_HOME/bin:$PATH
+ time mv ~/cassandra /tmp
+ cd /tmp/cassandra
+ if [ -d ~/dtest_jars ]; then
+ cp ~/dtest_jars/dtest* /tmp/cassandra/build/
+ fi
+ test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk
-F'"' '{print $4}' || true)
+ if [ -z "$test_timeout" ]; then
+ test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
+ fi
+ ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=unit
+ no_output_timeout: 15m
+ - store_test_results:
+ path: /tmp/cassandra/build/test/output/
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/output
+ destination: junitxml
+ - store_artifacts:
+ path: /tmp/cassandra/build/test/logs
+ destination: logs
+ environment:
+ - JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - ANT_HOME: /usr/share/ant
+ - LANG: en_US.UTF-8
+ - KEEP_TEST_DIR: true
+ - DEFAULT_DIR: /home/cassandra/cassandra-dtest
+ - PYTHONIOENCODING: utf-8
+ - PYTHONUNBUFFERED: true
+ - CASS_DRIVER_NO_EXTENSIONS: true
+ - CASS_DRIVER_NO_CYTHON: true
+ - CASSANDRA_SKIP_SYNC: true
+ - DTEST_REPO: git://github.com/apache/cassandra-dtest.git
- - DTEST_BRANCH: master
++ - DTEST_BRANCH: trunk
+ - CCM_MAX_HEAP_SIZE: 1024M
+ - CCM_HEAP_NEWSIZE: 256M
+ - JAVA_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ - JDK_HOME: /usr/lib/jvm/java-8-openjdk-amd64
+ j8_dtests-with-vnodes:
+ docker:
+ - image: spod/cassandra-testing-ubuntu1810-java11-w-dependencies:20190306
+ resource_class: medium
+ working_directory: ~/
+ shell: /bin/bash -eo pipefail -l
+ parallelism: 4
+ steps:
+ - attach_workspace:
+ at: /home/cassandra
+ - run:
+ name: Clone Cassandra dtest Repository (via git)
command: |
- set -x
+ git clone --single-branch --branch $DTEST_BRANCH --depth 1
$DTEST_REPO ~/cassandra-dtest
+ - run:
+ name: Configure virtualenv and python Dependencies
+ command: |
+ # note, this should be super quick as all dependencies should be
pre-installed in the docker image
+ # if additional dependencies were added to requirmeents.txt and the
docker image hasn't been updated
+ # we'd have to install it here at runtime -- which will make things
slow, so do yourself a favor and
+ # rebuild the docker image! (it automatically pulls the latest
requirements.txt on build)
+ source ~/env/bin/activate
export PATH=$JAVA_HOME/bin:$PATH
- time mv ~/cassandra /tmp
- cd /tmp/cassandra
- test_timeout=$(grep 'name="test.unit.timeout"' build.xml | awk
-F'"' '{print $4}' || true)
- if [ -z "$test_timeout" ]; then
- test_timeout=$(grep 'name="test.timeout"' build.xml | awk -F'"'
'{print $4}')
- fi
- ant testclasslist -Dtest.timeout="$test_timeout"
-Dtest.classlistfile=/tmp/java_tests_${CIRCLE_NODE_INDEX}_final.txt
-Dtest.classlistprefix=unit
+ pip3 install --upgrade -r ~/cassandra-dtest/requirements.txt
+ pip3 freeze
+ - run:
+ name: Determine Tests to Run (j8_with_vnodes)
+ no_output_timeout: 5m
+ command: "# reminder: this code (along with all the steps) is
independently executed on every circle container\n# so the goal here is to get
the circleci script to return the tests *this* container will run\n# which we
do via the `circleci` cli tool.\n\ncd cassandra-dtest\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\n\nif [ -n '' ]; then\n
export \nfi\n\necho \"***Collected DTests (j8_with_vnodes)***\"\nset -eo
pipefail && ./run_dtests.py --use-vnodes --skip-res [...]
+ - run:
+ name: Run dtests (j8_with_vnodes)
no_output_timeout: 15m
+ command: "echo \"cat
/tmp/split_dtest_tests_j8_with_vnodes_final.txt\"\ncat
/tmp/split_dtest_tests_j8_with_vnodes_final.txt\n\nsource
~/env/bin/activate\nexport PATH=$JAVA_HOME/bin:$PATH\nif [ -n '' ]; then\n
export \nfi\n\njava -version\ncd ~/cassandra-dtest\nmkdir -p /tmp/dtest\n\necho
\"env: $(env)\"\necho \"** done env\"\nmkdir -p /tmp/results/dtests\n# we need
the \"set -o pipefail\" here so that the exit code that circleci will actually
use is from pytest and not the exit [...]
- store_test_results:
- path: /tmp/cassandra/build/test/output/
+ path: /tmp/results
- store_artifacts:
- path: /tmp/cassandra/build/test/output
- destination: junitxml
+ path: /tmp/dtest
+ destination: dtest_j8_with_vnodes
- store_artifacts:
- path: /tmp/cassandra/build/test/logs
- destination: logs
+ path: ~/cassandra-dtest/logs
+ destination: dtest_j8_with_vnodes_logs
environment:
- JAVA8_HOME: /usr/lib/jvm/java-8-openjdk-amd64
- ANT_HOME: /usr/share/ant
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
