This is an automated email from the ASF dual-hosted git repository.
mck pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 2108f9dcaf525fcedf3b9e4d703aa8f230fd4cd1
Merge: a1b0a310ed f7c11bdcd4
Author: Mick Semb Wever <m...@apache.org>
AuthorDate: Thu Apr 4 11:57:23 2024 +0200
Merge branch 'cassandra-5.0' into trunk
* cassandra-5.0:
Standalone Jenkinsfile
.build/README.md | 122 +-
.build/check-code.sh | 2 +-
.build/ci/ci_parser.py | 307 +++++
.build/ci/generate-ci-summary.sh | 60 +
.../{check-code.sh => ci/generate-test-report.sh} | 17 +-
.build/ci/junit_helpers.py | 320 +++++
.build/ci/logging.sh | 124 ++
.build/ci/logging_helper.py | 136 +++
.build/{check-code.sh => ci/precommit_check.sh} | 41 +-
.build/{check-code.sh => ci/requirements.txt} | 15 +-
.build/docker/_docker_run.sh | 8 +-
.build/{check-code.sh => docker/build-jars.sh} | 14 +-
.build/docker/bullseye-build.docker | 3 +
.build/docker/run-tests.sh | 79 +-
.build/run-python-dtests.sh | 21 +-
.build/run-tests.sh | 12 +-
.jenkins/Jenkinsfile | 1230 ++++++++------------
CHANGES.txt | 1 +
build.xml | 73 +-
pylib/cassandra-cqlsh-tests.sh | 3 +-
.../org/apache/cassandra/utils/FBUtilities.java | 2 +-
.../org/apache/cassandra/utils/SystemInfo.java | 15 +-
.../CassandraXMLJUnitResultFormatter.java | 23 +-
.../apache/cassandra/utils/FBUtilitiesTest.java | 2 +-
24 files changed, 1707 insertions(+), 923 deletions(-)
diff --cc CHANGES.txt
index c770ef1b7c,5cd68dd179..f42a740790
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,33 -1,5 +1,34 @@@
-5.0-beta2
+5.1
+ * Expose all dropwizard metrics in virtual tables (CASSANDRA-14572)
+ * Ensured that PropertyFileSnitchTest do not overwrite
cassandra-toploogy.properties (CASSANDRA-19502)
+ * Add option for MutualTlsAuthenticator to restrict the certificate validity
period (CASSANDRA-18951)
+ * Fix StorageService::constructRangeToEndpointMap for non-distributed
keyspaces (CASSANDRA-19255)
+ * Group nodetool cms commands into single command group (CASSANDRA-19393)
+ * Register the measurements of the bootstrap process as Dropwizard metrics
(CASSANDRA-19447)
+ * Add LIST SUPERUSERS CQL statement (CASSANDRA-19417)
+ * Modernize CQLSH datetime conversions (CASSANDRA-18879)
+ * Harry model and in-JVM tests for partition-restricted 2i queries
(CASSANDRA-18275)
+ * Refactor cqlshmain global constants (CASSANDRA-19201)
+ * Remove native_transport_port_ssl (CASSANDRA-19397)
+ * Make nodetool reconfigurecms sync by default and add --cancel to be able
to cancel ongoing reconfigurations (CASSANDRA-19216)
+ * Expose auth mode in system_views.clients, nodetool clientstats, metrics
(CASSANDRA-19366)
+ * Remove sealed_periods and last_sealed_period tables (CASSANDRA-19189)
+ * Improve setup and initialisation of LocalLog/LogSpec (CASSANDRA-19271)
+ * Refactor structure of caching metrics and expose auth cache metrics via
JMX (CASSANDRA-17062)
+ * Allow CQL client certificate authentication to work without sending an
AUTHENTICATE request (CASSANDRA-18857)
+ * Extend nodetool tpstats and system_views.thread_pools with detailed pool
parameters (CASSANDRA-19289)
+ * Remove dependency on Sigar in favor of OSHI (CASSANDRA-16565)
+ * Simplify the bind marker and Term logic (CASSANDRA-18813)
+ * Limit cassandra startup to supported JDKs, allow higher JDKs by setting
CASSANDRA_JDK_UNSUPPORTED (CASSANDRA-18688)
+ * Standardize nodetool tablestats formatting of data units (CASSANDRA-19104)
+ * Make nodetool tablestats use number of significant digits for time and
average values consistently (CASSANDRA-19015)
+ * Upgrade jackson to 2.15.3 and snakeyaml to 2.1 (CASSANDRA-18875)
+ * Transactional Cluster Metadata [CEP-21] (CASSANDRA-18330)
+ * Add ELAPSED command to cqlsh (CASSANDRA-18861)
+ * Add the ability to disable bulk loading of SSTables (CASSANDRA-18781)
+ * Clean up obsolete functions and simplify cql_version handling in cqlsh
(CASSANDRA-18787)
+Merged from 5.0:
+ * Fix FBUtilities' parsing of gcp cos_containerd kernel versions
(CASSANDRA-18594)
* Clean up KeyRangeIterator classes (CASSANDRA-19428)
* Warn clients about possible consistency violations for filtering queries
against multiple mutable columns (CASSANDRA-19489)
* Align buffer with commitlog segment size (CASSANDRA-19471)
diff --cc src/java/org/apache/cassandra/utils/FBUtilities.java
index 414811aff3,eeefab136f..ff1aacf445
--- a/src/java/org/apache/cassandra/utils/FBUtilities.java
+++ b/src/java/org/apache/cassandra/utils/FBUtilities.java
@@@ -65,6 -66,8 +65,7 @@@ import com.google.common.base.Joiner
import com.google.common.base.Preconditions;
import com.google.common.base.Suppliers;
import com.google.common.collect.ImmutableList;
+ import com.vdurmont.semver4j.Semver;
-import com.vdurmont.semver4j.SemverException;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
diff --cc src/java/org/apache/cassandra/utils/SystemInfo.java
index ec793a692b,0000000000..7d8b974833
mode 100644,000000..100644
--- a/src/java/org/apache/cassandra/utils/SystemInfo.java
+++ b/src/java/org/apache/cassandra/utils/SystemInfo.java
@@@ -1,228 -1,0 +1,235 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.cassandra.utils;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.function.Supplier;
+import java.util.regex.Pattern;
+
+import com.vdurmont.semver4j.Semver;
++import org.apache.commons.lang3.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
++import oshi.PlatformEnum;
+
+import org.apache.cassandra.io.util.File;
+import org.apache.cassandra.io.util.FileUtils;
- import oshi.PlatformEnum;
+
+import static java.lang.String.format;
+import static java.util.Optional.empty;
+import static java.util.Optional.of;
+
+/**
+ * An abstraction of System information, this class provides access to system
information without specifying how
+ * it is retrieved.
+ */
+public class SystemInfo
+{
+ // TODO: Determine memlock limits if possible
+ // TODO: Determine if file system is remote or local
+ // TODO: Determine if disk latency is within acceptable limits
-
++
+ private static final Logger logger =
LoggerFactory.getLogger(SystemInfo.class);
+
+ private static final long INFINITY = -1L;
+ static final long EXPECTED_MIN_NUMBER_OF_OPENED_FILES = 10000L; // number
of files that can be opened
+ static final long EXPECTED_MIN_NUMBER_OF_PROCESSES = 32768L; // number of
processes
+ static final long EXPECTED_ADDRESS_SPACE = 0x7FFFFFFFL; // address space
+
+ static final String OPEN_FILES_VIOLATION_MESSAGE = format("Minimum value
for max open files should be >= %s. ", EXPECTED_MIN_NUMBER_OF_OPENED_FILES);
+ static final String NUMBER_OF_PROCESSES_VIOLATION_MESSAGE =
format("Number of processes should be >= %s. ",
EXPECTED_MIN_NUMBER_OF_PROCESSES);
+ static final String ADDRESS_SPACE_VIOLATION_MESSAGE = format("Amount of
available address space should be >= %s. ", EXPECTED_ADDRESS_SPACE);
+ static final String SWAP_VIOLATION_MESSAGE = "Swap should be disabled. ";
+
+ /**
+ * The default number of processes that are reported if the actual value
can not be retrieved.
+ */
+ private static final long DEFAULT_MAX_PROCESSES = 1024;
+
+ private static final Pattern SPACES_PATTERN = Pattern.compile("\\s+");
+
+ /**
+ * The oshi.SystemInfo has the following note:
+ * Platform-specific Hardware and Software objects are retrieved via
memoized suppliers. To conserve memory at the
+ * cost of additional processing time, create a new version of
SystemInfo() for subsequent calls. To conserve
+ * processing time at the cost of additional memory usage, re-use the
same {@link SystemInfo} object for future
+ * queries.
+ * <p>
+ * We are opting for minimal memory footprint.
+ */
+ private final oshi.SystemInfo si;
+
+ public SystemInfo()
+ {
+ si = new oshi.SystemInfo();
+ }
+
+ /**
+ * @return The PlatformEnum for the current platform. (e.g. Linux,
Windows, AIX, etc.)
+ */
+ public PlatformEnum platform()
+ {
+ return oshi.SystemInfo.getCurrentPlatform();
+ }
+
+ /**
+ * Gets the maximum number of processes the user can create.
+ * Note: if not on a Linux system this always return the
+ *
+ * @return The maximum number of processes.
+ * @see #DEFAULT_MAX_PROCESSES
+ */
+ public long getMaxProcess()
+ {
+ // this check only works on Linux systems. Errors fall through to
return default.
+ if (platform() == PlatformEnum.LINUX)
+ {
+ String path = format("/proc/%s/limits", getPid());
+ try
+ {
+ List<String> lines = FileUtils.readLines(new File(path));
+ for (String line : lines)
+ {
+ if (line.startsWith("Max processes"))
+ {
+ String[] parts = SPACES_PATTERN.split(line);
+
+ if (parts.length < 3)
+ continue;
+
+ String limit = parts[2];
+ return "unlimited".equals(limit) ? INFINITY :
Long.parseLong(limit);
+ }
+ }
+ logger.error("'Max processes' not found in {}", path);
+ }
+ catch (Exception t)
+ {
+ logger.error(format("Unable to read %s", path), t);
+ }
+ }
+
+ /* return the default value for non-Linux systems or parsing error.
+ * Can not return 0 as we know there is at least 1 process (this one)
and
+ * -1 historically represents infinity.
+ */
+ return DEFAULT_MAX_PROCESSES;
+ }
+
+ /**
+ * @return The maximum number of open files allowd to the current
process/user.
+ */
+ public long getMaxOpenFiles()
+ {
+ // ulimit -H -n
+ return
si.getOperatingSystem().getCurrentProcess().getHardOpenFileLimit();
+ }
+
+ /**
+ * Gets the Virtual Memory Size (VSZ). Includes all memory that the
process can access,
+ * including memory that is swapped out and memory that is from shared
libraries.
+ *
+ * @return The amount of virtual memory allowed to be allocatedby the
current process/user.
+ */
+ public long getVirtualMemoryMax()
+ {
+ return si.getOperatingSystem().getCurrentProcess().getVirtualSize();
+ }
+
+ /**
+ * @return The amount of swap space allocated on the system.
+ */
+ public long getSwapSize()
+ {
+ return si.getHardware().getMemory().getVirtualMemory().getSwapTotal();
+ }
+
+ /**
+ * @return the PID of the current system.
+ */
+ public long getPid()
+ {
+ return si.getOperatingSystem().getProcessId();
+ }
+
+ /**
+ * @return the Semver for the kernel version of the OS.
+ */
+ public Semver getKernelVersion()
- {
- return new
Semver(si.getOperatingSystem().getVersionInfo().getBuildNumber(),
Semver.SemverType.LOOSE);
++ {
++ String version =
si.getOperatingSystem().getVersionInfo().getBuildNumber();
++
++ // gcp's cos_containerd has a trailing +
++ if (version.endsWith("+"))
++ version = StringUtils.chop(version);
++
++ return new Semver(version, Semver.SemverType.LOOSE);
+ }
+
+ /**
+ * Tests if the system is running in degraded mode.
+ *
+ * @return non-empty optional with degradation messages if the system is
in degraded mode, empty optional otherwise.
+ */
+ public Optional<String> isDegraded()
+ {
+ Supplier<String> expectedNumProc = () -> {
+ // only check proc on nproc linux
+ if (platform() == PlatformEnum.LINUX)
+ return invalid(getMaxProcess(),
EXPECTED_MIN_NUMBER_OF_PROCESSES) ? NUMBER_OF_PROCESSES_VIOLATION_MESSAGE
+
: null;
+ else
+ return format("System is running %s, Linux OS is recommended.
", platform());
+ };
+
+ Supplier<String> swapShouldBeDisabled = () -> (getSwapSize() > 0) ?
SWAP_VIOLATION_MESSAGE : null;
+
+ Supplier<String> expectedAddressSpace = () ->
invalid(getVirtualMemoryMax(), EXPECTED_ADDRESS_SPACE)
+ ?
ADDRESS_SPACE_VIOLATION_MESSAGE
+ : null;
+
+ Supplier<String> expectedMinNoFile = () -> invalid(getMaxOpenFiles(),
EXPECTED_MIN_NUMBER_OF_OPENED_FILES)
+ ?
OPEN_FILES_VIOLATION_MESSAGE
+ : null;
+
+ StringBuilder sb = new StringBuilder();
+
+ for (Supplier<String> check : List.of(expectedNumProc,
swapShouldBeDisabled, expectedAddressSpace, expectedMinNoFile))
+ Optional.ofNullable(check.get()).map(sb::append);
+
+ String message = sb.toString();
+ return message.isEmpty() ? empty() : of(message);
+ }
+
+ /**
+ * Checks if a value is invalid.
+ * <p>
+ * Value is invalid if it is smaller than {@code min} and it is not
{@code INFINITY},
+ * here represented as a value of -1;
+ *
+ * @param value the value to check
+ * @param min the minimum value
+ * @return true if value is valid
+ */
+ private boolean invalid(long value, long min)
+ {
+ return value < min && value != INFINITY;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org
For additional commands, e-mail: commits-h...@cassandra.apache.org
