Repository: cassandra-builds Updated Branches: refs/heads/master 27bc20645 -> 2c1842cef
Move apache dtests to running in docker use dumb-init to get signal handling to work (https://engineeringblog.yelp.com/2016/01/dumb-init-an-init-for-docker.html) Patch by marcuse; reviewed by Michael Shuler for CASSANDRA-14241 Project: http://git-wip-us.apache.org/repos/asf/cassandra-builds/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra-builds/commit/2c1842ce Tree: http://git-wip-us.apache.org/repos/asf/cassandra-builds/tree/2c1842ce Diff: http://git-wip-us.apache.org/repos/asf/cassandra-builds/diff/2c1842ce Branch: refs/heads/master Commit: 2c1842cef7ce08327c95114df635fab1f81b8bc8 Parents: 27bc206 Author: Marcus Eriksson <[email protected]> Authored: Fri Feb 23 09:48:08 2018 -0800 Committer: Marcus Eriksson <[email protected]> Committed: Mon Feb 26 13:47:54 2018 -0800 ---------------------------------------------------------------------- build-scripts/cassandra-dtest-pytest.sh | 8 ++-- docker/jenkins/dtest.sh | 4 +- docker/jenkins/jenkinscommand.sh | 15 ++++--- jenkins-dsl/cassandra_job_dsl_seed.groovy | 62 ++------------------------ 4 files changed, 20 insertions(+), 69 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra-builds/blob/2c1842ce/build-scripts/cassandra-dtest-pytest.sh ---------------------------------------------------------------------- diff --git a/build-scripts/cassandra-dtest-pytest.sh b/build-scripts/cassandra-dtest-pytest.sh index 993bcb3..7eae11e 100755 --- a/build-scripts/cassandra-dtest-pytest.sh +++ b/build-scripts/cassandra-dtest-pytest.sh @@ -52,13 +52,13 @@ cd cassandra-dtest/ rm -r upgrade_tests/ # TEMP: remove upgrade_tests - we have no dual JDK installation set +e # disable immediate exit from this point if [ "${DTEST_TARGET}" = "dtest" ]; then - pytest --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt + pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt elif [ "${DTEST_TARGET}" = "dtest-novnode" ]; then - pytest --log-level="INFO" --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt + pytest -vv --log-level="INFO" --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt elif [ "${DTEST_TARGET}" = "dtest-offheap" ]; then - pytest --log-level="INFO" --use-vnodes --num-tokens=32 --use-off-heap-memtables --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt + pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --use-off-heap-memtables --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --skip-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt elif [ "${DTEST_TARGET}" = "dtest-large" ]; then - pytest --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --force-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt + pytest -vv --log-level="INFO" --use-vnodes --num-tokens=32 --junit-xml=nosetests.xml -s --cassandra-dir=$CASSANDRA_DIR --force-resource-intensive-tests 2>&1 | tee -a ${WORKSPACE}/test_stdout.txt else echo "Unknown dtest target: ${DTEST_TARGET}" exit 1 http://git-wip-us.apache.org/repos/asf/cassandra-builds/blob/2c1842ce/docker/jenkins/dtest.sh ---------------------------------------------------------------------- diff --git a/docker/jenkins/dtest.sh b/docker/jenkins/dtest.sh index 87dd77f..0d8cdb9 100644 --- a/docker/jenkins/dtest.sh +++ b/docker/jenkins/dtest.sh @@ -3,7 +3,9 @@ export WORKSPACE=/home/cassandra/cassandra export LANG=en_US.UTF-8 export PYTHONIOENCODING=utf-8 export PYTHONUNBUFFERED=true +echo "dtest.sh: running: git clone --depth=1 --branch=$BRANCH https://github.com/$REPO/cassandra.git"; git clone --depth=1 --branch=$BRANCH https://github.com/$REPO/cassandra.git cd cassandra +echo git clone --branch=$DTEST_BRANCH $DTEST_REPO git clone --branch=$DTEST_BRANCH $DTEST_REPO -../cassandra-builds/build-scripts/cassandra-dtest-pytest.sh +../cassandra-builds/build-scripts/cassandra-dtest-pytest.sh $1 http://git-wip-us.apache.org/repos/asf/cassandra-builds/blob/2c1842ce/docker/jenkins/jenkinscommand.sh ---------------------------------------------------------------------- diff --git a/docker/jenkins/jenkinscommand.sh b/docker/jenkins/jenkinscommand.sh index f218ff9..5e4741d 100644 --- a/docker/jenkins/jenkinscommand.sh +++ b/docker/jenkins/jenkinscommand.sh @@ -1,13 +1,18 @@ #!/bin/sh +DOCKER_IMAGE="kjellman/cassandra-test:0.4.4" +BUILDSREPO=$5 +BUILDSBRANCH=$6 cat > env.list <<EOF -REPO=$REPO -BRANCH=$BRANCH -DTEST_REPO=$DTEST_REPO -DTEST_BRANCH=$DTEST_BRANCH +REPO=$1 +BRANCH=$2 +DTEST_REPO=$3 +DTEST_BRANCH=$4 EOF -ID=$(docker run --env-file env.list -dt kjellman/cassandra-test:0.4.3 bash -ilc "git clone --depth=1 --branch dock https://github.com/krummas/cassandra-builds.git; sh ./cassandra-builds/docker/jenkins/dtest.sh") +echo "jenkinscommand.sh: running: git clone --depth=1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/docker/jenkins/dtest.sh $7" +ID=$(docker run --env-file env.list -dt $DOCKER_IMAGE dumb-init bash -ilc "git clone --depth=1 --branch $BUILDSBRANCH $BUILDSREPO; sh ./cassandra-builds/docker/jenkins/dtest.sh $7") # use docker attach instead of docker wait to get output docker attach --no-stdin $ID echo "$ID done, copying files" docker cp $ID:/home/cassandra/cassandra/cassandra-dtest/nosetests.xml . docker cp $ID:/home/cassandra/cassandra/test_stdout.txt . +docker rm $ID http://git-wip-us.apache.org/repos/asf/cassandra-builds/blob/2c1842ce/jenkins-dsl/cassandra_job_dsl_seed.groovy ---------------------------------------------------------------------- diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy b/jenkins-dsl/cassandra_job_dsl_seed.groovy index 9ab039c..1d4d0da 100644 --- a/jenkins-dsl/cassandra_job_dsl_seed.groovy +++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy @@ -344,7 +344,7 @@ cassandraBranches.each { scm(triggerInterval) } steps { - shell("./cassandra-builds/build-scripts/cassandra-dtest-pytest.sh ${targetName}") + shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh apache ${branchName} https://github.com/apache/cassandra-dtest.git master ${buildsRepo} ${buildsBranch} ${targetName}") } } } @@ -423,7 +423,7 @@ job('Cassandra-devbranch-testall') { } /** - * Parameterized Dev Branch dtest + * Parameterized Dev Branch dtest in docker */ job('Cassandra-devbranch-dtest') { description(jobDescription) @@ -446,7 +446,6 @@ job('Cassandra-devbranch-dtest') { stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL') stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout') - stringParam('DTEST_SCRIPT', './cassandra-builds/build-scripts/cassandra-dtest-pytest.sh', 'A temporary means of specifying an alternate script to run the dtests.') } scm { git { @@ -462,62 +461,7 @@ job('Cassandra-devbranch-dtest') { steps { buildDescription('', buildDescStr) shell("git clean -xdff ; git clone -b ${buildsBranch} ${buildsRepo}") - shell('git clone -b ${DTEST_BRANCH} ${DTEST_REPO}') - shell('${DTEST_SCRIPT}') - } - publishers { - archiveArtifacts('test_stdout.txt') - archiveJunit('cassandra-dtest/nosetests.xml') { - testDataPublishers { - publishTestStabilityData() - } - } - postBuildTask { - task('.', 'echo "Finding job process orphans.."; if pgrep -af ${JOB_BASE_NAME}; then pkill -9 -f ${JOB_BASE_NAME}; fi') - } - } -} - -/** - * Parameterized Dev Branch dtest in docker - */ -job('Cassandra-devbranch-dtest-docker') { - description(jobDescription) - concurrentBuild() - jdk(jdkLabel) - label(slaveLabel) - logRotator { - numToKeep(50) - } - wrappers { - timeout { - noActivity(2400) - } - } - throttleConcurrentBuilds { - categories(['Cassandra']) - } - parameters { - stringParam('REPO', 'apache', 'The github user/org to clone cassandra repo from') - stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout') - stringParam('DTEST_REPO', "${dtestRepo}", 'The cassandra-dtest repo URL') - stringParam('DTEST_BRANCH', 'master', 'The branch of cassandra-dtest to checkout') - } - scm { - git { - remote { - url('https://github.com/${REPO}/cassandra.git') - } - branch('${BRANCH}') - extensions { - cleanAfterCheckout() - } - } - } - steps { - buildDescription('', buildDescStr) - shell("git clean -xdff ; git clone -b dock https://github.com/krummas/cassandra-builds.git";) - shell('sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh') + shell("sh ./cassandra-builds/docker/jenkins/jenkinscommand.sh \$REPO \$BRANCH \$DTEST_REPO \$DTEST_BRANCH ${buildsRepo} ${buildsBranch}") } publishers { archiveArtifacts('test_stdout.txt') --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
