This is an automated email from the ASF dual-hosted git repository. mck pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit cd707ba67d182c5a8cfba087d0ae48da87532469 Merge: 0fa26aa75d b3ee9713f8 Author: Mick Semb Wever <[email protected]> AuthorDate: Tue Jan 27 20:14:48 2026 +0100 Merge branch 'cassandra-5.0' into trunk * cassandra-5.0: Implement microbench test target type .build/build-artifacts.sh | 2 +- .build/build-bench.xml | 22 ++++++- .build/build-jars.sh | 2 +- .build/check-code.sh | 2 +- .build/ci/generate-ci-summary.sh | 2 +- .build/ci/generate-test-report.sh | 2 +- .build/docker/_build-debian.sh | 2 +- .build/docker/_build-redhat.sh | 2 +- .build/docker/_docker_run.sh | 2 +- .build/docker/build-artifacts.sh | 2 +- .build/docker/build-debian.sh | 2 +- .build/docker/build-jars.sh | 2 +- .build/docker/build-redhat.sh | 2 +- .build/docker/check-code.sh | 2 +- .build/docker/run-tests.sh | 66 +++++++++++---------- .build/run-ci | 2 +- .build/run-python-dtests.sh | 20 ++++++- .build/run-tests.sh | 42 ++++++++------ .jenkins/Jenkinsfile | 67 +++++++++++++++------- .jenkins/k8s/jenkins-deployment.yaml | 3 +- build.xml | 8 --- .../test/microbench/BatchStatementBench.java | 3 +- .../cassandra/test/microbench/CompactionBench.java | 2 +- .../test/microbench/MetadataCollectorBench.java | 2 + 24 files changed, 161 insertions(+), 102 deletions(-) diff --cc .build/build-bench.xml index 65854c6875,b10fc64425..4ea0ff571e --- a/.build/build-bench.xml +++ b/.build/build-bench.xml @@@ -97,8 -112,9 +112,9 @@@ <extra-args/> <!-- TODO https://issues.apache.org/jira/browse/CASSANDRA-18873 --> + <!-- The classes listed below are broken, and so currently ignored, see CASSANDRA-18873 --> <arg value="-e" if:blank="${benchmark.name}"/> - <arg value="ZeroCopyStreamingBench|MutationBench|FastThreadLocalBench" if:blank="${benchmark.name}"/> - <arg value="AtomicBTreePartitionUpdateBench|BTreeSearchIteratorBench|BTreeTransformBench|BTreeUpdateBench|FastThreadLocalBench|KeyLookupBench|MutationBench|(instance.*Bench)|ReadWriteBench|ZeroCopyStreamingBench" if:blank="${benchmark.name}"/> ++ <arg value="AtomicBTreePartitionUpdateBench|BTreeSearchIteratorBench|BTreeTransformBench|BTreeUpdateBench|CompactionWideRowBench|FastThreadLocalBench|KeyLookupBench|(instance.*Bench)|LatencyTrackingBench|MessageOutBench|MutationBench|ReadWriteBench|SSTablePipeCursorBench|SSTableReadingCursorBench|SSTableReadingFileBench|SSTableReadingFileCursorBench|ZeroCopyStreamingBench" if:blank="${benchmark.name}"/> <arg value=".*microbench.*${benchmark.name}"/> </java> diff --cc .build/docker/build-redhat.sh index 4163ce9153,37cd2b8e4c..1d11b25986 --- a/.build/docker/build-redhat.sh +++ b/.build/docker/build-redhat.sh @@@ -40,5 -36,5 +40,5 @@@ ech # # Creates the redhat package - $(dirname "$0")/_docker_run.sh almalinux-build.docker docker/_build-redhat.sh "${java_version}" -$(dirname -- "$0")/_docker_run.sh almalinux-build.docker docker/_build-redhat.sh "${java_version}" ${rpm_dist} ++$(dirname -- "$0")/_docker_run.sh almalinux-build.docker docker/_build-redhat.sh "${java_version}" exit $? diff --cc .build/run-tests.sh index 9270922898,344f3d9e18..ee80aadd69 --- a/.build/run-tests.sh +++ b/.build/run-tests.sh @@@ -27,9 -27,12 +27,12 @@@ set -o pipefai [ $DEBUG ] && set -x # variables, with defaults - [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname "$0")/..)" + [ "x${CASSANDRA_DIR}" != "x" ] || CASSANDRA_DIR="$(readlink -f $(dirname -- "$0")/..)" [ "x${DIST_DIR}" != "x" ] || DIST_DIR="${CASSANDRA_DIR}/build" + # target types -TARGET_TYPES="build_dtest_jars stress-test fqltool-test microbench microbench-test test-burn long-test cqlsh-test simulator-dtest test test-cdc test-compression test-oa test-system-keyspace-directory test-latest jvm-dtest jvm-dtest-upgrade jvm-dtest-novnode jvm-dtest-upgrade-novnode" ++TARGET_TYPES="build_dtest_jars stress-test fqltool-test sstableloader-test microbench microbench-test test-burn long-test cqlsh-test simulator-dtest test test-cdc test-compression test-oa test-system-keyspace-directory test-latest jvm-dtest jvm-dtest-upgrade jvm-dtest-novnode jvm-dtest-upgrade-novnode" + # pre-conditions command -v ant >/dev/null 2>&1 || { error 1 "ant needs to be installed"; } command -v git >/dev/null 2>&1 || { error 1 "git needs to be installed"; } @@@ -344,13 -346,9 +347,14 @@@ _main() ant fqltool-build-test ${ANT_TEST_OPTS} ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}" ;; + "sstableloader-test") + # hard fail on test compilation, but dont fail the test run so unstable test reports are processed + ant sstableloader-build-test ${ANT_TEST_OPTS} + ant $target ${ANT_TEST_OPTS} || echo "failed ${target} ${split_chunk}" + ;; - "microbench") - ant $target ${ANT_TEST_OPTS} -Dmaven.test.failure.ignore=true + "microbench" | "microbench-test") + [[ "x${test_name_regexp}" != "x" ]] && test_name_regexp="-Dbenchmark.name=${test_name_regexp}" + ant $target ${ANT_TEST_OPTS} ${test_name_regexp} -Dmaven.test.failure.ignore=true ;; "test") _run_testlist "unit" "testclasslist" "${test_name_regexp}" "${split_chunk}" "$(_timeout_for 'test.timeout')" "${repeat_count}" diff --cc .jenkins/Jenkinsfile index 877cdc2850,4aa1dcd44a..0e5980bb19 --- a/.jenkins/Jenkinsfile +++ b/.jenkins/Jenkinsfile @@@ -140,9 -141,10 +141,10 @@@ def pipelineProfiles() return [ 'packaging': ['artifacts', 'lint', 'debian', 'redhat'], 'skinny': ['lint', 'cqlsh-test', 'test', 'jvm-dtest', 'simulator-dtest', 'dtest'], - 'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest'], - 'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade'], - 'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-no [...] - 'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest', 'microbench-test'], - 'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade', 'microbench-test'], - 'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-large-novnode', 'microbe [...] ++ 'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest', 'microbench-test'], ++ 'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade', 'microbench-test'], ++ 'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'sstableloader-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-la [...] + 'performance': ['microbench'], 'custom': [] ] } @@@ -201,10 -202,13 +203,13 @@@ def tasks() 'dtest-large': [splits: 6, size: 'large'], 'dtest-large-novnode': [splits: 6, size: 'large'], 'dtest-large-latest': [splits: 6, size: 'large'], - 'dtest-upgrade': [splits: 128, size: 'large'], - 'dtest-upgrade-novnode': [splits: 128, size: 'large'], - 'dtest-upgrade-large': [splits: 32, size: 'large'], - 'dtest-upgrade-large-novnode': [splits: 32, size: 'large'], - 'microbench-test': [splits: 1, size: 'large'], + 'dtest-upgrade': [splits: 160, size: 'large'], + 'dtest-upgrade-novnode': [splits: 160, size: 'large'], + 'dtest-upgrade-large': [splits: 40, size: 'large'], - 'dtest-upgrade-novnode-large': [splits: 40, size: 'large'], ++ 'dtest-upgrade-large-novnode': [splits: 40, size: 'large'], ++ 'microbench-test': [splits: 1, size: 'large', timeout_hours: 2], // TODO implement splits + // performance tests need 'cassandra-*large-dedicated' nodes + 'microbench': [splits: 1, size: 'large', timeout_hours: 6, benchmark: true], ] testSteps.each() { it.value.put('type', 'test') diff --cc test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java index 87eb237f92,16dbbda72a..f2f88409ad --- a/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/BatchStatementBench.java @@@ -45,6 -29,6 +45,7 @@@ import org.openjdk.jmh.runner.options.O import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.cql3.Attributes; import org.apache.cassandra.cql3.BatchQueryOptions; ++import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.cql3.QueryHandler; import org.apache.cassandra.cql3.QueryOptions; import org.apache.cassandra.cql3.QueryProcessor; @@@ -75,7 -78,7 +76,7 @@@ public class BatchStatementBenc static { -- DatabaseDescriptor.toolInitialization(); ++ CQLTester.setUpClass(); // Partitioner is not set in client mode. if (DatabaseDescriptor.getPartitioner() == null) DatabaseDescriptor.setPartitionerUnsafe(Murmur3Partitioner.instance); diff --cc test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java index 393e088e51,6ca814f564..8462d30090 --- a/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/CompactionBench.java @@@ -80,15 -52,7 +80,15 @@@ public class CompactionBench extends CQ @Setup(Level.Trial) public void setup() throws Throwable { - CQLTester.prepareServer(); + CQLTester.setUpClass(); + DatabaseDescriptor.setCursorCompactionEnabled(isCursor); + DatabaseDescriptor.setCompactionThroughputMebibytesPerSec(10*1024); // no rate limiting + createSStables(); + takeSnapshot(); + } + + protected void createSStables() + { keyspace = createKeyspace("CREATE KEYSPACE %s with replication = { 'class' : 'SimpleStrategy', 'replication_factor' : 1 } and durable_writes = false"); table = createTable(keyspace, "CREATE TABLE %s ( userid bigint, picid bigint, commentid bigint, PRIMARY KEY(userid, picid))"); execute("use "+keyspace+";"); diff --cc test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java index 170627cdbd,c03623efdb..2adb0c2047 --- a/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java +++ b/test/microbench/org/apache/cassandra/test/microbench/MetadataCollectorBench.java @@@ -23,19 -23,6 +23,20 @@@ import java.util.Arrays import java.util.concurrent.ThreadLocalRandom; import java.util.concurrent.TimeUnit; +import org.jctools.util.Pow2; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; + ++import org.apache.cassandra.cql3.CQLTester; import org.apache.cassandra.db.Clustering; import org.apache.cassandra.db.ClusteringBound; import org.apache.cassandra.db.ClusteringBoundary; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
