This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra-builds.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4431728 Dockerise cqlshlib tests
4431728 is described below
commit 4431728f6483aed303362b4893c7d27118a4c841
Author: Mick Semb Wever <[email protected]>
AuthorDate: Wed May 5 12:34:45 2021 +0200
Dockerise cqlshlib tests
patch by Mick Semb Wever; reviewed by Brandon Williams for CASSANDRA-16809
---
build-scripts/cassandra-test-docker.sh | 5 +++--
build-scripts/cassandra-test.sh | 6 ++++++
docker/testing/ubuntu2004_j11.docker | 15 ++++++++++++++-
jenkins-dsl/cassandra_job_dsl_seed.groovy | 28 +++++++++++++++++++---------
4 files changed, 42 insertions(+), 12 deletions(-)
diff --git a/build-scripts/cassandra-test-docker.sh
b/build-scripts/cassandra-test-docker.sh
index a48a39a..254c1a9 100755
--- a/build-scripts/cassandra-test-docker.sh
+++ b/build-scripts/cassandra-test-docker.sh
@@ -15,7 +15,7 @@ if [ "$#" -lt 3 ]; then
export LC_CTYPE=en_US.UTF-8
export PYTHONIOENCODING=utf-8
export PYTHONUNBUFFERED=true
- if [ "${JAVA_VERSION}" -ge 11 ]; then
+ if [ "${JAVA_VERSION}" -ge 11 ] ; then
sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg
--print-architecture)
export CASSANDRA_USE_JDK11=true
export JAVA_HOME=$(sudo update-java-alternatives -l | grep
"java-1.11.0-openjdk" | awk '{print $3}')
@@ -76,7 +76,7 @@ EOF
# for relevant test targets calculate how many docker containers we should
split the test list over
case $TARGET in
# test-burn doesn't have enough tests in it to split beyond 8, and burn
and long we want a bit more resources anyway
- "stress-test" | "fqltool-test" | "microbench" | "test-burn" |
"long-test")
+ "stress-test" | "fqltool-test" | "microbench" | "test-burn" |
"long-test" | "cqlsh-test")
docker_runs=1
;;
"test"| "test-cdc" | "test-compression" | "jvm-dtest" |
"jvm-dtest-upgrade")
@@ -160,6 +160,7 @@ EOF
docker cp
"$docker_id:/home/cassandra/cassandra/${TARGET}-${inner_split}-${INNER_SPLITS}-cassandra.head"
.
docker cp $docker_id:/home/cassandra/cassandra/build/test/output/.
build/test/output
docker cp $docker_id:/home/cassandra/cassandra/build/test/logs/.
build/test/logs
+ docker cp
$docker_id:/home/cassandra/cassandra/pylib/cqlshlib/cqlshlib.xml cqlshlib.xml
fi
docker rm $docker_id
((i++))
diff --git a/build-scripts/cassandra-test.sh b/build-scripts/cassandra-test.sh
index 2e6750a..d783d86 100755
--- a/build-scripts/cassandra-test.sh
+++ b/build-scripts/cassandra-test.sh
@@ -89,6 +89,9 @@ _main() {
regx_version="(2.2|3.0).([0-9]+)$"
! [[ $version =~ $regx_version ]] || { echo "Skipping ${target}. It does
not exist in ${version}"; exit 0; }
;;
+ "cqlsh-test")
+ [[ -f "./pylib/cassandra-cqlsh-tests.sh" ]] || { echo "Skipping
${target}. It does not exist in ${version}"; exit 0; }
+ ;;
*)
;;
esac
@@ -145,6 +148,9 @@ _main() {
fi
ant testclasslist -Dtest.classlistprefix=distributed
-Dtest.timeout=$(_timeout_for "test.distributed.timeout")
-Dtest.classlistfile=<(echo "${testlist}") -Dtmp.dir="${TMP_DIR}" || echo
"failed $target"
;;
+ "cqlsh-test")
+ ./pylib/cassandra-cqlsh-tests.sh .
+ ;;
*)
echo "unregconised \"$target\""
exit 1
diff --git a/docker/testing/ubuntu2004_j11.docker
b/docker/testing/ubuntu2004_j11.docker
index 5540715..e432bb4 100644
--- a/docker/testing/ubuntu2004_j11.docker
+++ b/docker/testing/ubuntu2004_j11.docker
@@ -26,15 +26,22 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
RUN export DEBIAN_FRONTEND=noninteractive && \
add-apt-repository -y ppa:deadsnakes/ppa && \
apt-get update && \
- apt-get install -y git-core python2.7 python-dev python3-pip \
+ apt-get install -y curl git-core python2.7 python-dev python3-pip \
python3.6 python3.6-venv python3.6-dev \
python3.7 python3.7-venv python3.7-dev \
python3.8 python3.8-venv python3.8-dev \
net-tools libev4 libev-dev wget gcc libssl-dev libxml2 libxslt1-dev
+# install python2-pip
+RUN curl https://bootstrap.pypa.io/pip/2.7/get-pip.py -o get-pip.py
+RUN python2.7 get-pip.py
+RUN if ! [ -f /usr/bin/python2 ] ; then ln -s /usr/bin/python2.7
/usr/bin/python2 ; fi
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python2.7 1
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 3
RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 4
+RUN python2.7 -m pip install --upgrade pip
RUN python3.6 -m pip install --upgrade pip
RUN python3.7 -m pip install --upgrade pip
RUN python3.8 -m pip install --upgrade pip
@@ -61,6 +68,8 @@ ADD
https://raw.githubusercontent.com/apache/cassandra-dtest/trunk/requirements.
RUN chmod 0644 /opt/requirements.txt
# now setup python via virtualenv with all of the python dependencies we need
according to requirements.txt
+RUN pip2 install virtualenv
+RUN pip2 install --upgrade wheel
RUN pip3 install virtualenv
RUN pip3 install --upgrade wheel
@@ -104,6 +113,10 @@ ENV ANT_HOME=/usr/share/ant
# default in test jobs using the CASS_DRIVER_X env vars below. However, if the
extensions are
# included in the base image, the compiled objects are not updated by pip at
run time, which can
# cause errors if the tests rely on new driver functionality or bug fixes.
+RUN virtualenv --python=python2.7 env2.7
+RUN chmod +x env2.7/bin/activate
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1
&& source ~/env2.7/bin/activate && pip2 install --upgrade pip && pip2 install
-r /opt/requirements.txt && pip2 freeze --user"
+
RUN virtualenv --python=python3.6 env3.6
RUN chmod +x env3.6/bin/activate
RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1
&& source ~/env3.6/bin/activate && pip3 install --upgrade pip && pip3 install
-r /opt/requirements.txt && pip3 freeze --user"
diff --git a/jenkins-dsl/cassandra_job_dsl_seed.groovy
b/jenkins-dsl/cassandra_job_dsl_seed.groovy
index 36809f3..0ca8842 100644
--- a/jenkins-dsl/cassandra_job_dsl_seed.groovy
+++ b/jenkins-dsl/cassandra_job_dsl_seed.groovy
@@ -310,9 +310,6 @@ matrixJob('Cassandra-template-cqlsh-tests') {
}
timestamps()
}
- throttleConcurrentBuilds {
- categories(['Cassandra'])
- }
axes {
text('cython', 'yes', 'no')
jdk(jdkLabel)
@@ -353,7 +350,7 @@ matrixJob('Cassandra-template-cqlsh-tests') {
buildDescription('', buildDescStr)
shell("""
git clean -xdff ;
- ./pylib/cassandra-cqlsh-tests.sh \$WORKSPACE ;
+ ./cassandra-builds/build-scripts/cassandra-test-docker.sh apache
${branchName} ${buildsRepo} ${buildsBranch} ${testDockerImage} cqlsh-test ;
echo "\${BUILD_TAG}) cassandra: `git log -1 --pretty=format:'%h
%an %ad %s'`" > \${BUILD_TAG}.head ;
wget --retry-connrefused --waitretry=1
"\${BUILD_URL}/timestamps/?time=HH:mm:ss&timeZone=UTC&appendLog" -qO - >
console.log.xz || echo wget failed
""")
@@ -416,6 +413,8 @@ cassandraBranches.each {
// if the agent is busy, just prune everything that is older
than maxJobHours
task('.', """
echo "Cleaning project…"; git clean -xdff ;
+ echo "Cleaning processes…" ;
+ if ! pgrep -af "cassandra-builds/build-scripts" ; then
pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ;
echo "Pruning docker…" ;
if pgrep -af "cassandra-builds/build-scripts" ; then
docker system prune --all --force --filter "until=${maxJobHours}h" || true ;
else docker system prune --all --force --volumes || true ; fi;
echo "Reporting disk usage…"; df -h ;
@@ -507,6 +506,8 @@ cassandraBranches.each {
// if the agent is busy, just prune everything that is
older than maxJobHours
task('.', """
echo "Cleaning project…"; git clean -xdff -e
build/test/jmh-result.json ;
+ echo "Cleaning processes…" ;
+ if ! pgrep -af "cassandra-builds/build-scripts" ;
then pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ;
echo "Pruning docker…" ;
if pgrep -af "cassandra-builds/build-scripts" ;
then docker system prune --all --force --filter "until=${maxJobHours}h" || true
; else docker system prune --all --force --volumes || true ; fi;
echo "Reporting disk usage…"; du -xm / 2>/dev/null
| sort -rn | head -n 30 ; df -h ;
@@ -595,6 +596,8 @@ cassandraBranches.each {
// if the agent is busy, just prune everything
that is older than maxJobHours
task('.', """
echo "Cleaning project…"; git clean -xdff ;
+ echo "Cleaning processes…" ;
+ if ! pgrep -af
"cassandra-builds/build-scripts" ; then pkill -9 -f org.apache.cassandra. ||
echo "already clean" ; fi ;
echo "Pruning docker…" ;
if pgrep -af "cassandra-builds/build-scripts"
; then docker system prune --all --force --filter "until=${maxJobHours}h" ||
true ; else docker system prune --all --force --volumes || true ; fi;
echo "Reporting disk usage…"; df -h ;
@@ -648,6 +651,8 @@ cassandraBranches.each {
// if the agent is busy, just prune everything that is
older than maxJobHours
task('.', """
echo "Cleaning project…"; git clean -xdff ;
+ echo "Cleaning processes…" ;
+ if ! pgrep -af "cassandra-builds/build-scripts" ; then
pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ;
echo "Pruning docker…" ;
if pgrep -af "cassandra-builds/build-scripts" ; then
docker system prune --all --force --filter "until=${maxJobHours}h" || true ;
else docker system prune --all --force --volumes || true ; fi;
echo "Reporting disk usage…"; df -h ;
@@ -793,6 +798,8 @@ matrixJob('Cassandra-devbranch-artifacts') {
// if the agent is busy, just prune everything that is older than
maxJobHours
task('.', """
echo "Cleaning project…"; git clean -xdff ;
+ echo "Cleaning processes…" ;
+ if ! pgrep -af "cassandra-builds/build-scripts" ; then pkill
-9 -f org.apache.cassandra. || echo "already clean" ; fi ;
echo "Pruning docker…" ;
if pgrep -af "cassandra-builds/build-scripts" ; then docker
system prune --all --force --filter "until=${maxJobHours}h" || true ; else
docker system prune --all --force --volumes || true ; fi;
echo "Reporting disk usage…"; df -h ;
@@ -909,6 +916,8 @@ testTargets.each {
// if the agent is busy, just prune everything that is older
than maxJobHours
task('.', """
echo "Cleaning project…"; git clean -xdff ${targetName ==
'microbench' ? '-e build/test/jmh-result.json' : ''};
+ echo "Cleaning processes…" ;
+ if ! pgrep -af "cassandra-builds/build-scripts" ; then
pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ;
echo "Pruning docker…" ;
if pgrep -af "cassandra-builds/build-scripts" ; then
docker system prune --all --force --filter "until=${maxJobHours}h" || true ;
else docker system prune --all --force --volumes || true ; fi;
echo "Reporting disk usage…"; du -xm / 2>/dev/null | sort
-rn | head -n 30 ; df -h ;
@@ -1040,6 +1049,8 @@ archs.each {
// if the agent is busy, just prune everything that is
older than maxJobHours
task('.', """
echo "Cleaning project…" ; git clean -xdff ;
+ echo "Cleaning processes…" ;
+ if ! pgrep -af "cassandra-builds/build-scripts" ; then
pkill -9 -f org.apache.cassandra. || echo "already clean" ; fi ;
echo "Pruning docker…" ;
if pgrep -af "cassandra-builds/build-scripts" ; then
docker system prune --all --force --filter "until=${maxJobHours}h" || true ;
else docker system prune --all --force --volumes || true ; fi;
echo "Reporting disk usage…"; df -h ;
@@ -1073,9 +1084,6 @@ matrixJob('Cassandra-devbranch-cqlsh-tests') {
}
timestamps()
}
- throttleConcurrentBuilds {
- categories(['Cassandra'])
- }
parameters {
stringParam('REPO', 'apache', 'The github user/org to clone cassandra
repo from')
stringParam('BRANCH', 'trunk', 'The branch of cassandra to checkout')
@@ -1122,8 +1130,8 @@ matrixJob('Cassandra-devbranch-cqlsh-tests') {
buildDescription('', buildDescStr)
shell("""
git clean -xdff ;
- echo "Cassandra-devbranch-cqlsh-tests) cassandra: `git log -1
--pretty=format:'%h %an %ad %s'`" > Cassandra-devbranch-cqlsh-tests.head ;
- ./pylib/cassandra-cqlsh-tests.sh \$WORKSPACE ;
+ echo "\${BUILD_TAG}) cassandra: `git log -1
--pretty=format:'%h %an %ad %s'`" > \${BUILD_TAG}.head ;
+ ./cassandra-builds/build-scripts/cassandra-test-docker.sh
\${REPO} \${BRANCH} ${buildsRepo} ${buildsBranch} ${testDockerImage} cqlsh-test
;
wget --retry-connrefused --waitretry=1
"\${BUILD_URL}/timestamps/?time=HH:mm:ss&timeZone=UTC&appendLog" -qO - >
console.log.xz || echo wget failed
""")
}
@@ -1149,6 +1157,8 @@ matrixJob('Cassandra-devbranch-cqlsh-tests') {
// if the agent is busy, just prune everything that is older than
maxJobHours
task('.', """
echo "Cleaning project…"; git clean -xdff ;
+ echo "Cleaning processes…" ;
+ if ! pgrep -af "cassandra-builds/build-scripts" ; then pkill
-9 -f org.apache.cassandra. || echo "already clean" ; fi ;
echo "Pruning docker…" ;
if pgrep -af "cassandra-builds/build-scripts" ; then docker
system prune --all --force --filter "until=${maxJobHours}h" || true ; else
docker system prune --all --force --volumes || true ; fi;
echo "Reporting disk usage…"; df -h ;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
