This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-5.0 by this push:
new 987d03c142 Fixes for in-tree scripts: offline mode, maybe-build for
fqltool-test, jvm-dtest-upgrade
987d03c142 is described below
commit 987d03c142f086f05857f9c6a2eafef7e157eb15
Author: Mick Semb Wever <[email protected]>
AuthorDate: Sat Sep 9 21:14:10 2023 +0200
Fixes for in-tree scripts: offline mode, maybe-build for fqltool-test,
jvm-dtest-upgrade
- jvm-dtest-upgrade script running in ci-cassandra.a.o (needs default bash)
- add offline mode (to avoid `git fetch` before building dtest-jars on
other branches),
- avoid rebuilding in fqltool-test target, as is standard for test targets.
- python detection and override
- FileTest cleanup (tmp directory was left unreadable and undeletable)
- reuse ~/.ccm (like normal)
- remove support for python 3.6
- remove docker volumes in image (they are added at run time)
- remove superfluousjava version debug
– extra mount for build/tmp as hack to prevent buggy hardlinks on overlay
patch by Mick Semb Wever; reviewed by Brandon Williams, Ekaterina
Dimitrova, Josh McKenzie, Maxim Muzafarov for CASSANDRA-18665
---
.build/docker/_set_java.sh | 6 +--
.build/docker/almalinux-build.docker | 3 --
.build/docker/bullseye-build.docker | 3 --
.build/docker/centos7-build.docker | 3 --
.build/docker/run-tests.sh | 10 ++++-
.build/docker/ubuntu2004_test.docker | 44 +++++++++++-----------
.build/run-python-dtests.sh | 13 ++++---
.build/run-tests.sh | 11 ++++--
build.xml | 2 +-
test/unit/org/apache/cassandra/db/ImportTest.java | 2 +
.../org/apache/cassandra/io/util/FileTest.java | 7 +++-
11 files changed, 54 insertions(+), 50 deletions(-)
diff --git a/.build/docker/_set_java.sh b/.build/docker/_set_java.sh
index 6770803c6c..e324f3350b 100755
--- a/.build/docker/_set_java.sh
+++ b/.build/docker/_set_java.sh
@@ -62,8 +62,4 @@ else
sudo alternatives --set javac $(alternatives --display javac | grep
"family java-${java_version}-openjdk" | cut -d' ' -f1)
fi
export JAVA_HOME=$(readlink -f /usr/bin/javac | sed "s:/bin/javac::")
-echo "Cassandra will be built with Java ${java_version}"
-
-# print debug information on versions
-java -version 2>&1
-javac -version 2>&1
+echo "Using Java ${java_version}"
\ No newline at end of file
diff --git a/.build/docker/almalinux-build.docker
b/.build/docker/almalinux-build.docker
index c648eb581c..c850942620 100644
--- a/.build/docker/almalinux-build.docker
+++ b/.build/docker/almalinux-build.docker
@@ -28,9 +28,6 @@ ARG GID_ARG=1000
LABEL org.cassandra.buildenv=almalinux
-VOLUME ${DIST_DIR}
-VOLUME ${CASSANDRA_DIR}
-
RUN echo "Building with arguments:" \
&& echo " - DIST_DIR=${DIST_DIR}" \
&& echo " - BUILD_HOME=${BUILD_HOME}" \
diff --git a/.build/docker/bullseye-build.docker
b/.build/docker/bullseye-build.docker
index 7612a00290..7eb928bf5a 100644
--- a/.build/docker/bullseye-build.docker
+++ b/.build/docker/bullseye-build.docker
@@ -25,9 +25,6 @@ ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
LABEL org.cassandra.buildenv=bullseye
-VOLUME ${DIST_DIR}
-VOLUME ${CASSANDRA_DIR}
-
RUN echo "Building with arguments:" \
&& echo " - DIST_DIR=${DIST_DIR}" \
&& echo " - BUILD_HOME=${BUILD_HOME}" \
diff --git a/.build/docker/centos7-build.docker
b/.build/docker/centos7-build.docker
index 2f8aef98f2..9f9782ba80 100644
--- a/.build/docker/centos7-build.docker
+++ b/.build/docker/centos7-build.docker
@@ -29,9 +29,6 @@ ARG GID_ARG=1000
LABEL org.cassandra.buildenv=centos
-VOLUME ${DIST_DIR}
-VOLUME ${CASSANDRA_DIR}
-
RUN echo "Building with arguments:" \
&& echo " - DIST_DIR=${DIST_DIR}" \
&& echo " - BUILD_HOME=${BUILD_HOME}" \
diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh
index e18af803e9..ede5569834 100755
--- a/.build/docker/run-tests.sh
+++ b/.build/docker/run-tests.sh
@@ -69,8 +69,11 @@ if [[ ! "${java_version}" =~ $regx_java_version ]]; then
exit 1
fi
-python_version="3.6"
-command -v python >/dev/null 2>&1 && python_version="$(python -V | awk '{print
$2}' | awk -F'.' '{print $1"."$2}')"
+# allow python version override, otherwise default to current python version
or 3.7
+if [ "x" == "x${python_version}" ] ; then
+ command -v python >/dev/null 2>&1 && python_version="$(python -V 2>&1 |
awk '{print $2}' | awk -F'.' '{print $1"."$2}')"
+ python_version="${python_version:-3.7}"
+fi
# print debug information on versions
docker --version
@@ -82,6 +85,8 @@ dockerfile="ubuntu2004_test.docker"
image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)"
image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v
"${build_dir}":/home/cassandra/cassandra/build -v
${HOME}/.m2/repository:/home/cassandra/.m2/repository"
+# HACK hardlinks in overlay are buggy, the following mount prevents hardlinks
from being used. ref $TMP_DIR in .build/run-tests.sh
+docker_mounts="${docker_mounts} -v
"${build_dir}/tmp":/home/cassandra/cassandra/build/tmp"
# Look for existing docker image, otherwise build
if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
@@ -150,6 +155,7 @@ elif [[ "${target}" =~ dtest* ]] ; then
else
docker_flags="--cpus=${docker_cpus} -m 5g --memory-swap 5g"
fi
+
docker_flags="${docker_flags} --env-file build/env.list -d --rm"
# make sure build_dir is good
diff --git a/.build/docker/ubuntu2004_test.docker
b/.build/docker/ubuntu2004_test.docker
index 7035c5228f..1d21ae623c 100644
--- a/.build/docker/ubuntu2004_test.docker
+++ b/.build/docker/ubuntu2004_test.docker
@@ -25,9 +25,6 @@ ENV PYTHONUNBUFFERED=true
LABEL org.cassandra.buildenv=ubuntu_2004
-VOLUME ${CASSANDRA_DIR}
-VOLUME ${DIST_DIR}
-
RUN echo "Building with arguments:" \
&& echo " - DIST_DIR=${DIST_DIR}" \
&& echo " - BUILD_HOME=${BUILD_HOME}" \
@@ -41,7 +38,7 @@ RUN echo 'Acquire::http::Timeout "60";' >
/etc/apt/apt.conf.d/80proxy.conf
RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf
# install our python dependencies and some other stuff we need
-# libev4 libev-dev are for the python driver / libssl-dev is for python3.6
+# libev4 libev-dev are for the python driver
RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get update && \
@@ -51,20 +48,17 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
add-apt-repository -y ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install -y curl git-core python3-pip \
- python3.6 python3.6-venv python3.6-dev \
python3.7 python3.7-venv python3.7-dev \
python3.8 python3.8-venv python3.8-dev \
python3.11 python3.11-venv python3.11-dev \
- virtualenv net-tools libev4 libev-dev wget gcc libssl-dev libxml2
libxslt1-dev \
+ virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \
vim lsof sudo libjemalloc2 dumb-init locales \
openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.6 2
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 3
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 4
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 5
-RUN python3.6 -m pip install --upgrade pip
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 2
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 3
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 4
RUN python3.7 -m pip install --upgrade pip
RUN python3.8 -m pip install --upgrade pip
@@ -109,15 +103,6 @@ ENV ANT_HOME=/usr/share/ant
# included in the base image, the compiled objects are not updated by pip at
run time, which can
# cause errors if the tests rely on new driver functionality or bug fixes.
-RUN virtualenv --python=python3.6 ${BUILD_HOME}/env3.6
-RUN chmod +x ${BUILD_HOME}/env3.6/bin/activate
-
-RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
- && source ${BUILD_HOME}/env3.6/bin/activate \
- && pip3 install --upgrade pip \
- && pip3 install -r /opt/requirements.txt \
- && pip3 freeze --user"
-
RUN virtualenv --python=python3.7 ${BUILD_HOME}/env3.7
RUN chmod +x ${BUILD_HOME}/env3.7/bin/activate
@@ -146,11 +131,26 @@ RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1
CASS_DRIVER_NO_EXTENSIONS=1 \
&& pip3 freeze --user"
# Initialize the CCM git repo as well as this also can fail to clone
-RUN /bin/bash -c "source ${BUILD_HOME}/env3.6/bin/activate && \
- ccm create -n 1 -v git:trunk test && ccm remove test && \
+RUN /bin/bash -c "source ${BUILD_HOME}/env3.7/bin/activate && \
ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \
ccm create -n 1 -v git:cassandra-4.0 test && ccm remove test"
+# Initialize ccm versions. right side of each sequence needs to be updated
with new releases.
+# this can be checked with:
+# `curl -s https://downloads.apache.org/cassandra/ | grep -oP
'(?<=href=\")[0-9]+\.[0-9]+\.[0-9]+(?=)' | sort -V | uniq -w 3`
+RUN bash -c 'source ~/env3.7/bin/activate && \
+ for i in {1..11} ; do echo $i ; ccm create --quiet -n 1 -v binary:4.0.$i
test && ccm remove test ; done && \
+ for i in {1..3} ; do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i
test && ccm remove test ; done'
+
+# 5+ requires java11
+RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg
--print-architecture)
+
+# Initialize the CCM git repo as well as this also can fail to clone
+RUN /bin/bash -c 'source ~/env3.7/bin/activate && \
+ ccm create -n 1 -v git:trunk test && ccm remove test && \
+ ccm create -n 1 -v git:cassandra-5.0 test && ccm remove test && \
+ ccm create --quiet -n 1 -v binary:5.0-alpha1 test && ccm remove test '
+
# the .git subdirectories to pip installed cassandra-driver breaks
virtualenv-clone, so just remove them
RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git
# fix permissions, runtime user has different uid/gid
diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh
index 91b597be1c..4b791e267d 100755
--- a/.build/run-python-dtests.sh
+++ b/.build/run-python-dtests.sh
@@ -51,11 +51,10 @@ export CASS_DRIVER_NO_EXTENSIONS=true
export CASS_DRIVER_NO_CYTHON=true
export CCM_MAX_HEAP_SIZE="1024M"
export CCM_HEAP_NEWSIZE="512M"
-export CCM_CONFIG_DIR=${DIST_DIR}/.ccm
export NUM_TOKENS="16"
#Have Cassandra skip all fsyncs to improve test performance and reliability
export CASSANDRA_SKIP_SYNC=true
-export TMPDIR="$(mktemp -d /tmp/run-python-dtest.XXXXXX)"
+export TMPDIR="$(mktemp -d ${DIST_DIR}/run-python-dtest.XXXXXX)"
unset CASSANDRA_HOME
# pre-conditions
@@ -73,7 +72,9 @@ if [ "${java_version}" -eq 17 ] && [[ "${target}" ==
"dtest-upgrade" ]] ; then
exit 1
fi
-python_version=$(python -V | awk '{print $2}' | awk -F'.' '{print $1"."$2}')
+python_version=$(python -V 2>&1 | awk '{print $2}' | awk -F'.' '{print
$1"."$2}')
+python_regx_supported_versions="^(3.7|3.8|3.11)$"
+[[ $python_version =~ $python_regx_supported_versions ]] || { echo "Python
${python_version} not supported."; exit 1; }
# check project is already built. no cleaning is done, so jenkins unstash
works, beware.
[[ -f "${DIST_DIR}/apache-cassandra-${version}.jar" ]] || [[ -f
"${DIST_DIR}/apache-cassandra-${version}-SNAPSHOT.jar" ]] || { echo "Project
must be built first. Use \`ant jar\`. Build directory is ${DIST_DIR} with: $(ls
${DIST_DIR})"; exit 1; }
@@ -163,8 +164,8 @@ fi
# merge all unit xml files into one, and print summary test numbers
pushd ${CASSANDRA_DIR}/ >/dev/null
# remove <testsuites> wrapping elements. `ant generate-unified-test-report`
doesn't like it`
-sed -r "s/<[\/]?testsuites>//g" ${DIST_DIR}/test/output/nosetests.xml >
/tmp/nosetests.xml
-cat /tmp/nosetests.xml > ${DIST_DIR}/test/output/nosetests.xml
+sed -r "s/<[\/]?testsuites>//g" ${DIST_DIR}/test/output/nosetests.xml >
${TMPDIR}/nosetests.xml
+cat ${TMPDIR}/nosetests.xml > ${DIST_DIR}/test/output/nosetests.xml
ant -quiet -silent generate-unified-test-report
popd >/dev/null
@@ -174,7 +175,7 @@ popd >/dev/null
#
################################
-rm -rf "/tmp/run-python-dtest.${TMPDIR/\/tmp\/run-python-dtest./}"
+[[ "${TMPDIR}" == *"${DIST_DIR}/run-python-dtest."* ]] && rm -rf "${TMPDIR}"
unset TMPDIR
deactivate
diff --git a/.build/run-tests.sh b/.build/run-tests.sh
index 4a0cecfa33..9e1291cf74 100755
--- a/.build/run-tests.sh
+++ b/.build/run-tests.sh
@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -49,6 +49,9 @@ git --version
java -version 2>&1
javac -version 2>&1
+# set the OFFLINE env var (to anything) to allow running jvm-dtest-upgrade
offline
+[ "x" != "x${OFFLINE}" ] && echo "WARNING: running in offline mode.
jvm-dtest-upgrade results may be stale."
+
# lists all tests for the specific test type
_list_tests() {
local -r classlistprefix="$1"
@@ -84,7 +87,9 @@ _build_all_dtest_jars() {
fi
if [ -d ${TMP_DIR}/cassandra-dtest-jars ] && [
"https://github.com/apache/cassandra.git"; == "$(git -C
${TMP_DIR}/cassandra-dtest-jars remote get-url origin)" ] ; then
- until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet origin ; do
echo "git pull failed… trying again… " ; done
+ if [ "x" == "x${OFFLINE}" ] ; then
+ until git -C ${TMP_DIR}/cassandra-dtest-jars fetch --quiet origin ; do
echo "git fetch failed… trying again… " ; done
+ fi
else
rm -fR ${TMP_DIR}/cassandra-dtest-jars
pushd $TMP_DIR >/dev/null
@@ -171,7 +176,7 @@ _main() {
# ant test setup
export TMP_DIR="${DIST_DIR}/tmp"
- mkdir -p "${TMP_DIR}" || true
+ [ -d ${TMP_DIR} ] || mkdir -p "${TMP_DIR}"
export ANT_TEST_OPTS="-Dno-build-test=true -Dtmp.dir=${TMP_DIR}"
# fresh virtualenv and test logs results everytime
diff --git a/build.xml b/build.xml
index b61e5a240f..de370b1154 100644
--- a/build.xml
+++ b/build.xml
@@ -624,7 +624,7 @@
</javac>
</target>
- <target name="fqltool-test" depends="fqltool-build-test, build-test"
description="Runs fqltool tests">
+ <target name="fqltool-test" depends="fqltool-build-test, maybe-build-test"
description="Runs fqltool tests">
<testmacro inputdir="${fqltool.test.src}"
timeout="${test.timeout}">
</testmacro>
diff --git a/test/unit/org/apache/cassandra/db/ImportTest.java
b/test/unit/org/apache/cassandra/db/ImportTest.java
index c8624b96d4..0af3f612ed 100644
--- a/test/unit/org/apache/cassandra/db/ImportTest.java
+++ b/test/unit/org/apache/cassandra/db/ImportTest.java
@@ -43,6 +43,7 @@ import org.apache.cassandra.dht.BootStrapper;
import org.apache.cassandra.io.sstable.format.SSTableFormat.Components;
import org.apache.cassandra.io.sstable.format.SSTableReader;
import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.PathUtils;
import org.apache.cassandra.locator.InetAddressAndPort;
import org.apache.cassandra.locator.TokenMetadata;
import org.apache.cassandra.service.CacheService;
@@ -245,6 +246,7 @@ public class ImportTest extends CQLTester
}
}
}
+ PathUtils.deleteRecursiveOnExit(temp);
return new File(backupdir);
}
diff --git a/test/unit/org/apache/cassandra/io/util/FileTest.java
b/test/unit/org/apache/cassandra/io/util/FileTest.java
index 0a7c478daf..718f321042 100644
--- a/test/unit/org/apache/cassandra/io/util/FileTest.java
+++ b/test/unit/org/apache/cassandra/io/util/FileTest.java
@@ -37,7 +37,6 @@ import org.apache.commons.lang3.RandomUtils;
import org.junit.Assert;
import org.junit.Test;
-import org.apache.cassandra.config.CassandraRelevantProperties;
import org.apache.cassandra.config.DatabaseDescriptor;
import org.assertj.core.api.Assertions;
import org.psjava.util.Triple;
@@ -45,13 +44,14 @@ import org.psjava.util.Triple;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static java.util.concurrent.TimeUnit.SECONDS;
import static
org.apache.cassandra.config.CassandraRelevantProperties.JAVA_IO_TMPDIR;
+import static
org.apache.cassandra.config.CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE;
public class FileTest
{
private static final java.io.File dir;
static
{
- CassandraRelevantProperties.USE_NIX_RECURSIVE_DELETE.setBoolean(false);
+ USE_NIX_RECURSIVE_DELETE.setBoolean(false);
java.io.File parent = new java.io.File(JAVA_IO_TMPDIR.getString());
//checkstyle: permit this instantiation
String dirName =
Long.toHexString(ThreadLocalRandom.current().nextLong());
while (new java.io.File(parent, dirName).exists()) //checkstyle:
permit this instantiation
@@ -115,6 +115,9 @@ public class FileTest
regularLink.delete();
regular.delete();
emptySubdir.delete();
+ nonEmptySubdir.delete();
+ nonEmptySubdirLink.delete();
+ dir.setReadable(true);
}
private static String nonAbsolute(java.io.File file)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
