This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 8b58e5522e1 IGNITE-29165 [ducktests] Add reproducers of a full disk
volume under a server node (#13708)
8b58e5522e1 is described below
commit 8b58e5522e1c7cb77be842107e4358eef32f70e3
Author: Maksim Davydov <[email protected]>
AuthorDate: Tue Oct 27 15:54:01 2026 +0300
IGNITE-29165 [ducktests] Add reproducers of a full disk volume under a
server node (#13708)
---
modules/ducktests/README.md | 3 +
.../tests/disk_full/DiskFullLoadApplication.java | 88 ++++
.../tests/disk_full/StdoutFloodApplication.java | 51 +++
.../tests/disk_full/StdoutFloodService.java | 26 ++
.../tests/disk_full/StdoutFloodServiceImpl.java | 62 +++
.../tests/checks/utils/check_ignite_spec.py | 20 +-
.../tests/checks/utils/check_parametrized.py | 30 +-
.../ignitetest/services/utils/ignite_aware.py | 31 ++
.../utils/ignite_configuration/data_storage.py | 1 +
.../tests/ignitetest/services/utils/ignite_spec.py | 24 +-
.../tests/ignitetest/tests/disk_full_test.py | 458 +++++++++++++++++++++
.../ducktests/tests/ignitetest/utils/__init__.py | 4 +-
modules/ducktests/tests/ignitetest/utils/_mark.py | 34 ++
13 files changed, 823 insertions(+), 9 deletions(-)
diff --git a/modules/ducktests/README.md b/modules/ducktests/README.md
index c789bc25d98..66dc5ccb063 100644
--- a/modules/ducktests/README.md
+++ b/modules/ducktests/README.md
@@ -173,6 +173,9 @@ You can modify test environments at execution time using
global flags injected t
| **cluster_size** | Controls the cluster size for tests. Overrides the
default num_nodes value passed in the @cluster annotation. Default value is
determined by the test framework. | ```{"cluster_size": 13}``` |
| **ignite_versions** | List of Ignite versions for testing. Tests run for
each version in the list. Values are folder names in /opt/ (e.g., "ignite-dev"
for master branch, "ignite-2.18.0" for release). Default is determined by test
annotations. | ```{"ignite_versions": ["ignite-dev", "ignite-2.18.0"]}``` |
| **failure_detection_timeout** | Timeout in milliseconds for failure
detection in discovery. Used to detect node failures in the cluster. Default
value is 10000 ms (10 seconds). | ```{"failure_detection_timeout": 20000}``` |
+| **disk_full_tests** | Boolean flag to run the reproducers in
`disk_full_test.py`, which fill a disk volume of a server node and check what
the node does. Default is False: the tests are ignored. |
```{"disk_full_tests": true}``` |
+
+*Note on reproducers:* A test marked with `@reproducer("<global>")` runs only
when that global is set to true; otherwise it is ignored. The mark is for tests
that reproduce a behavior on demand and guard nothing on a regular run.
#### Security & Authentication
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java
new file mode 100644
index 00000000000..875b17ca122
--- /dev/null
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/DiskFullLoadApplication.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.disk_full;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.ignite.client.ClientCache;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Puts values under growing keys, with a new binary type every 1000 puts, so
the partitions, the WAL and the binary
+ * metadata keep growing. Retries failed puts and records {@code
progress-><puts> <errors><-} once a second.
+ */
+public class DiskFullLoadApplication extends IgniteAwareApplication {
+ /** Value size, in bytes. */
+ private static final int VALUE_SIZE = 1024;
+
+ /** Puts per new binary type. */
+ private static final int BINARY_TYPE_EVERY = 1000;
+
+ /** Successful puts after which the application reports it is initialized.
*/
+ private static final int WARM_UP_PUTS = 1000;
+
+ /** Pause after a failed put, in milliseconds. */
+ private static final long ERROR_PAUSE = 200;
+
+ /** {@inheritDoc} */
+ @Override protected void run(JsonNode jsonNode) throws Exception {
+ ClientCache<Long, Object> cache =
client.getOrCreateCache(jsonNode.get("cacheName").asText());
+
+ byte[] val = new byte[VALUE_SIZE];
+
+ long key = 0;
+ long puts = 0;
+ long errs = 0;
+ long nextReport = System.currentTimeMillis() + 1000;
+
+ while (active()) {
+ try {
+ if (key % BINARY_TYPE_EVERY == 0)
+ cache.put(key, client.binary().builder("DiskFullType" +
key).setField("val", val).build());
+ else
+ cache.put(key, val);
+
+ key++;
+ puts++;
+
+ // The first record comes before the marker, so the test can
read the counters once start() returns.
+ if (puts == WARM_UP_PUTS) {
+ recordResult("progress", puts + " " + errs);
+
+ markInitialized();
+ }
+ }
+ catch (Exception e) {
+ errs++;
+
+ log.warn("Put failed [key=" + key + ", err=" + e + ']');
+
+ Thread.sleep(ERROR_PAUSE);
+ }
+
+ long now = System.currentTimeMillis();
+
+ if (now >= nextReport) {
+ recordResult("progress", puts + " " + errs);
+
+ nextReport = now + 1000;
+ }
+ }
+
+ markFinished();
+ }
+}
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java
new file mode 100644
index 00000000000..1272054346e
--- /dev/null
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodApplication.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.disk_full;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.apache.ignite.internal.ducktest.utils.IgniteAwareApplication;
+
+/**
+ * Deploys {@link StdoutFloodServiceImpl} on every server node, records {@code
lines-><n><-}, the lines the service
+ * wrote, once a second, and cancels the service on stop.
+ */
+public class StdoutFloodApplication extends IgniteAwareApplication {
+ /** Service name. */
+ private static final String SVC_NAME = "stdout-flood";
+
+ /** {@inheritDoc} */
+ @Override protected void run(JsonNode jsonNode) throws Exception {
+
ignite.services(ignite.cluster().forServers()).deployNodeSingleton(SVC_NAME,
new StdoutFloodServiceImpl());
+
+ StdoutFloodService svc = ignite.services().serviceProxy(SVC_NAME,
StdoutFloodService.class, false);
+
+ recordResult("lines", svc.lines());
+
+ markInitialized();
+
+ while (active()) {
+ Thread.sleep(1000);
+
+ recordResult("lines", svc.lines());
+ }
+
+ ignite.services().cancel(SVC_NAME);
+
+ markFinished();
+ }
+}
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java
new file mode 100644
index 00000000000..e3b277cbd9a
--- /dev/null
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodService.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.disk_full;
+
+/**
+ * Writes lines to {@code System.out} without end, as user code left with
debug output does.
+ */
+public interface StdoutFloodService {
+ /** @return Lines written so far. */
+ public long lines();
+}
diff --git
a/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java
new file mode 100644
index 00000000000..3904e8a8409
--- /dev/null
+++
b/modules/ducktests/src/main/java/org/apache/ignite/internal/ducktest/tests/disk_full/StdoutFloodServiceImpl.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.ducktest.tests.disk_full;
+
+import java.util.concurrent.atomic.AtomicLong;
+import org.apache.ignite.resources.ServiceContextResource;
+import org.apache.ignite.services.Service;
+import org.apache.ignite.services.ServiceContext;
+
+/**
+ * Writes 1 KB lines to {@code System.out} at about 10 MB/s until cancelled.
+ */
+public class StdoutFloodServiceImpl implements Service, StdoutFloodService {
+ /** Serial version UID. */
+ private static final long serialVersionUID = 0L;
+
+ /** Line, 1 KB with its line separator. */
+ private static final String LINE = "x".repeat(1023);
+
+ /** Lines written per millisecond. */
+ private static final int LINES_PER_MS = 10;
+
+ /** Service context. */
+ @ServiceContextResource
+ private ServiceContext ctx;
+
+ /** Lines written. */
+ private final AtomicLong lines = new AtomicLong();
+
+ /** {@inheritDoc} */
+ @Override public void execute() throws Exception {
+ while (!ctx.isCancelled()) {
+ for (int i = 0; i < LINES_PER_MS; i++) {
+ System.out.println(LINE);
+
+ lines.incrementAndGet();
+ }
+
+ Thread.sleep(1);
+ }
+ }
+
+ /** {@inheritDoc} */
+ @Override public long lines() {
+ return lines.get();
+ }
+}
diff --git a/modules/ducktests/tests/checks/utils/check_ignite_spec.py
b/modules/ducktests/tests/checks/utils/check_ignite_spec.py
index 1e2cc4d3920..41ff5909202 100644
--- a/modules/ducktests/tests/checks/utils/check_ignite_spec.py
+++ b/modules/ducktests/tests/checks/utils/check_ignite_spec.py
@@ -17,13 +17,14 @@
Checks Spec class that describes config and command line to start Ignite-aware
service.
"""
import os
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
import pytest
from ignitetest.services.utils import IgniteServiceType
from ignitetest.services.utils.gc_params import CLIENT_ROLE, SERVER_ROLE
-from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec,
IgniteNodeSpec, service_role
+from ignitetest.services.utils.ignite_spec import IgniteApplicationSpec,
IgniteNodeSpec, IgniteNodeSpecFileOutput, \
+ service_role
from ignitetest.services.utils.jvm_utils import GC_PROFILES, GC_G1,
MultipleGcSelectedError
from ignitetest.utils.ignite_test import JFR_ENABLED
@@ -287,3 +288,18 @@ def
check_heap_override__from_jvm_opts__still_wins(service):
assert "-Xms4G" in spec.jvm_opts
assert "-Xmx1G" not in spec.jvm_opts
assert "-Xms1G" not in spec.jvm_opts
+
+
+"""
+Checks the shell redirection of a node's output.
+"""
+
+
+def
check_node_spec_file_output__appends_stdout_and_stderr_to_console_log(service):
+ service.log_dir = "/logs"
+
+ with patch.object(IgniteNodeSpecFileOutput, "envs", return_value={}):
+ cmd = IgniteNodeSpecFileOutput(service).command(None)
+
+ assert cmd.endswith(f" >> {os.path.join(service.log_dir, 'console.log')}
2>&1 &")
+ assert "tee" not in cmd
diff --git a/modules/ducktests/tests/checks/utils/check_parametrized.py
b/modules/ducktests/tests/checks/utils/check_parametrized.py
index 7c814fec457..3bc84d161fd 100644
--- a/modules/ducktests/tests/checks/utils/check_parametrized.py
+++ b/modules/ducktests/tests/checks/utils/check_parametrized.py
@@ -24,7 +24,7 @@ import pytest
from ducktape.mark import parametrized, parametrize, matrix, ignore
from ducktape.mark.mark_expander import MarkedFunctionExpander
-from ignitetest.utils import ignite_versions, ignore_if
+from ignitetest.utils import ignite_versions, ignore_if, reproducer
from ignitetest.utils._mark import IgniteVersionParametrize
from ignitetest.utils.version import IgniteVersion, V_2_8_0, V_2_8_1, V_2_7_6,
DEV_BRANCH
@@ -267,3 +267,31 @@ class CheckVersionIf:
assert next(filter(lambda x: x.injected_args['ver'] == str(V_2_8_0),
context_list)).ignore
assert not next(filter(lambda x: x.injected_args['ver'] ==
str(DEV_BRANCH), context_list)).ignore
+
+
+class CheckReproducer:
+ """
+ Checks @reproducer.
+ """
+ @pytest.mark.parametrize(['global_args', 'ignored'], [({}, True),
({'my_tests': False}, True),
+ ({'my_tests': True},
False)])
+ def check_ignored_unless_global_is_set(self, global_args, ignored):
+ """
+ Check that a test is ignored unless its global is set, with and
without @ignite_versions.
+ """
+ @reproducer('my_tests')
+ @ignite_versions(str(DEV_BRANCH), str(V_2_8_0))
+ def versioned(ignite_version):
+ return IgniteVersion(ignite_version)
+
+ @reproducer('my_tests')
+ def plain():
+ pass
+
+ sess_ctx = mock_session_ctx(global_args=global_args)
+
+ context_list = expand_function(func=versioned, sess_ctx=sess_ctx)
+ context_list += MarkedFunctionExpander(session_context=sess_ctx,
function=plain).expand()
+
+ assert len(context_list) == 3
+ assert all(ctx.ignore == ignored for ctx in context_list)
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
index e0f81ab35b7..822d422fad9 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_aware.py
@@ -48,6 +48,9 @@ class IgniteAwareService(BackgroundThreadService,
IgnitePathAware, JvmProcessMix
"""
The base class to build services aware of Ignite.
"""
+ # Mount source of the volumes mount_tmpfs makes: clean_node finds them by
it.
+ TMPFS_SOURCE = "ignitetest-tmpfs"
+
@constructible
class NetPart(IntEnum):
"""
@@ -79,6 +82,8 @@ class IgniteAwareService(BackgroundThreadService,
IgnitePathAware, JvmProcessMix
self.disconnected_nodes = []
+ self.tmpfs_volumes = {}
+
@property
def product(self):
return str(self.config.version)
@@ -109,7 +114,15 @@ class IgniteAwareService(BackgroundThreadService,
IgnitePathAware, JvmProcessMix
self.await_event("Topology snapshot", self.startup_timeout_sec,
nodes=nodes, from_the_beginning=True)
+ def mount_tmpfs(self, path, size_mb):
+ """
+ Mounts a tmpfs of size_mb over the path on every node when it starts,
so writes there fail once it is full.
+ """
+ self.tmpfs_volumes[path] = size_mb
+
def start_node(self, node, **kwargs):
+ self.__mount_tmpfs_volumes(node)
+
self.init_shared(node)
self.init_persistent(node)
@@ -186,8 +199,26 @@ class IgniteAwareService(BackgroundThreadService,
IgnitePathAware, JvmProcessMix
def clean_node(self, node, **kwargs):
super().clean_node(node, **kwargs)
+ self.__unmount_tmpfs_volumes(node)
+
node.account.ssh("rm -rf -- %s" % self.persistent_root,
allow_fail=False)
+ def __mount_tmpfs_volumes(self, node):
+ """
+ Mounts the volumes registered by mount_tmpfs, unless they are still
mounted from the node's previous start.
+ """
+ for path, size_mb in self.tmpfs_volumes.items():
+ node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -M {path} >
/dev/null || (mkdir -p {path} && "
+ f"sudo mount -t tmpfs -o size={size_mb}m,uid=$(id
-u),gid=$(id -g) {self.TMPFS_SOURCE} "
+ f"{path})", allow_fail=False)
+
+ def __unmount_tmpfs_volumes(self, node):
+ """
+ Unmounts every volume mount_tmpfs made on the node, also one left by
an aborted run.
+ """
+ node.account.ssh(f"findmnt -rn -S {self.TMPFS_SOURCE} -o TARGET |
xargs -r -n1 sudo umount -l",
+ allow_fail=False)
+
def init_persistent(self, node):
"""
Init persistent directory.
diff --git
a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py
b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py
index 54e5be42166..0484c95aafd 100644
---
a/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py
+++
b/modules/ducktests/tests/ignitetest/services/utils/ignite_configuration/data_storage.py
@@ -50,6 +50,7 @@ class DataStorageConfiguration(NamedTuple):
metrics_rate_time_interval: int = None
page_size: int = None
regions: list = []
+ storage_path: str = None
wal_archive_path: str = None
wal_buffer_size: int = None
wal_compaction_enabled: bool = None
diff --git a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
index ae7036df48c..891c56619fc 100644
--- a/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
+++ b/modules/ducktests/tests/ignitetest/services/utils/ignite_spec.py
@@ -373,6 +373,12 @@ class IgniteSpec(metaclass=ABCMeta):
opts = ["-J%s" % o for o in self.jvm_opts]
return " ".join(opts)
+ def _output_redirect(self):
+ """
+ :return: shell redirection of the service's stdout and stderr: through
tee, appended to console.log.
+ """
+ return "2>&1 | tee -a %s" % os.path.join(self.service.log_dir,
"console.log")
+
def _runcmd(self, cmd):
self.service.logger.debug(cmd)
proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
@@ -388,16 +394,26 @@ class IgniteNodeSpec(IgniteSpec):
"""
def command(self, node):
- cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \
+ cmd = "%s %s %s %s %s &" % \
(envs_to_exports(self.envs()),
self.service.script("ignite.sh"),
self._jvm_opts(),
self.config_file_path(),
- os.path.join(self.service.log_dir, "console.log"))
+ self._output_redirect())
return cmd
+class IgniteNodeSpecFileOutput(IgniteNodeSpec):
+ """
+ Spec to run ignite node with stdout and stderr appended to console.log by
the shell, as nohup does, so the JVM
+ writes them itself instead of a pipe to tee.
+ """
+
+ def _output_redirect(self):
+ return ">> %s 2>&1" % os.path.join(self.service.log_dir, "console.log")
+
+
class IgniteApplicationSpec(IgniteSpec):
"""
Spec to run ignite application
@@ -422,12 +438,12 @@ class IgniteApplicationSpec(IgniteSpec):
str(base64.b64encode(json.dumps(self.service.params).encode('utf-8')), 'utf-8')
]
- cmd = "%s %s %s %s 2>&1 | tee -a %s &" % \
+ cmd = "%s %s %s %s %s &" % \
(envs_to_exports(self.envs()),
self.service.script("ignite.sh"),
self._jvm_opts(),
",".join(args),
- os.path.join(self.service.log_dir, "console.log"))
+ self._output_redirect())
return cmd
diff --git a/modules/ducktests/tests/ignitetest/tests/disk_full_test.py
b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py
new file mode 100644
index 00000000000..adedac946a8
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/tests/disk_full_test.py
@@ -0,0 +1,458 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Fills one volume of a persistent server node under load and checks whether the
node survives.
+
+The tests reproduce what Ignite does today and guard no behavior, so they run
only with
+--globals '{"disk_full_tests": true}'.
+"""
+import os
+import time
+
+from ducktape.mark import matrix
+from ducktape.utils.util import wait_until
+
+from ignitetest.services.ignite import IgniteService
+from ignitetest.services.ignite_app import IgniteApplicationService
+from ignitetest.services.utils.control_utility import ControlUtility
+from ignitetest.services.utils.ignite_configuration import
IgniteConfiguration, IgniteThinClientConfiguration, \
+ DataStorageConfiguration
+from ignitetest.services.utils.ignite_configuration.data_storage import
DataRegionConfiguration
+from ignitetest.services.utils.ignite_configuration.discovery import
from_ignite_cluster
+from ignitetest.services.utils.ignite_spec import IgniteNodeSpecFileOutput
+from ignitetest.services.utils.jvm_utils import merge_jvm_settings
+from ignitetest.services.utils.ssl.client_connector_configuration import
ClientConnectorConfiguration
+from ignitetest.utils import cluster, ignite_versions, reproducer
+from ignitetest.utils.ignite_test import IgniteTest
+from ignitetest.utils.version import DEV_BRANCH, IgniteVersion
+
+MB = 1024 * 1024
+
+# A full tmpfs still takes writes into the rest of a file's last page.
+PAGE = 4096
+
+FILLER = "filler"
+
+# Failure type and failed writer the node logs when the volume fills.
+FAILURES = {
+ "wal_archive": ("SYSTEM_WORKER_TERMINATION", "Failed to archive WAL
segment"),
+ "partitions": ("CRITICAL_ERROR", "StorageException"),
+ "work": ("CRITICAL_ERROR", "BinaryMetadataFileStore")
+}
+
+
+class DiskFullTest(IgniteTest):
+ """
+ Each test puts a size-limited tmpfs over a directory of the node, fills it
and records what the node does today.
+ """
+ LOAD_APP =
"org.apache.ignite.internal.ducktest.tests.disk_full.DiskFullLoadApplication"
+
+ FLOOD_APP =
"org.apache.ignite.internal.ducktest.tests.disk_full.StdoutFloodApplication"
+
+ @cluster(num_nodes=2)
+ @reproducer("disk_full_tests")
+ @ignite_versions(str(DEV_BRANCH))
+ def test_node_survives_and_restarts_on_full_log_volume(self,
ignite_version):
+ """
+ The node serves load and restarts while the volume of its log4j2 log,
console.log and gc.log is full.
+ """
+ ignite = self.__ignite(ignite_version)
+ ignite.mount_tmpfs(ignite.log_dir, 32)
+
+ ignite.start()
+ ControlUtility(ignite).activate()
+
+ load = self.__start_load_client(ignite)
+
+ fill_volume(ignite, ignite.log_dir)
+
+ check_serves_load_for(ignite, load, seconds=15)
+ assert ControlUtility(ignite).cluster_state().state == "ACTIVE"
+
+ for log in log_files(ignite):
+ check_write_failed(ignite, log)
+
+ ignite.stop()
+
+ # Waits for a failed put, so the count of puts read next is final
until the node is back.
+ _, errors = progress(load)
+ wait_until(lambda: progress(load)[1] > errors, timeout_sec=30,
err_msg="The load still reaches the node")
+
+ puts, _ = progress(load)
+
+ # start() awaits a line in the node's new log, which the full volume
refuses.
+ ignite.start_async(clean=False)
+
+ wait_until(lambda: progress(load)[0] > puts, timeout_sec=60,
err_msg="No puts after the restart")
+
+ check_serves_load_for(ignite, load, seconds=15)
+ assert ControlUtility(ignite).cluster_state().state == "ACTIVE"
+ assert file_size(ignite, ignite.nodes[0].log_file) == 0, "The
restarted node wrote its log"
+
+ free_volume(ignite, ignite.log_dir)
+
+ for log in log_files(ignite):
+ check_grows(ignite, log)
+
+ load.stop()
+
+ @cluster(num_nodes=3)
+ @reproducer("disk_full_tests")
+ @ignite_versions(str(DEV_BRANCH))
+ def test_node_survives_log_volume_filled_by_stdout(self, ignite_version):
+ """
+ A service on the node writes to System.out until its output fills the
log volume.
+ """
+ ignite = self.__ignite(ignite_version)
+ ignite.mount_tmpfs(ignite.log_dir, 32)
+
+ ignite.start()
+ ControlUtility(ignite).activate()
+
+ load = self.__start_load_client(ignite)
+
+ flood = self.__start_flood_service(ignite)
+
+ wait_until(lambda: free_space(ignite, ignite.log_dir) == 0,
timeout_sec=60,
+ err_msg="System.out did not fill the log volume")
+
+ lines = printed_lines(flood)
+
+ check_serves_load_for(ignite, load, seconds=15)
+ assert ControlUtility(ignite).cluster_state().state == "ACTIVE"
+ assert printed_lines(flood) > lines, "The service stopped writing to
System.out"
+
+ for log in log_files(ignite):
+ check_write_failed(ignite, log)
+
+ flood.stop()
+
+ enlarge_volume(ignite, ignite.log_dir, 128)
+
+ for log in log_files(ignite):
+ check_grows(ignite, log)
+
+ load.stop()
+
+ @cluster(num_nodes=2)
+ @reproducer("disk_full_tests")
+ @ignite_versions(str(DEV_BRANCH))
+ def test_jvm_exits_on_full_jfr_volume(self, ignite_version):
+ """
+ JFR ends the JVM once it can't write its recording.
+ """
+ ignite = self.__ignite(ignite_version)
+ repository = os.path.join(ignite.persistent_root, "jfr_repository")
+ jfr_opts = ["-XX:StartFlightRecording",
f"-XX:FlightRecorderOptions=repository={repository}"]
+ ignite.spec.jvm_opts = merge_jvm_settings(ignite.spec.jvm_opts,
jfr_opts)
+ ignite.mount_tmpfs(repository, 32)
+
+ ignite.start()
+ ControlUtility(ignite).activate()
+
+ load = self.__start_load_client(ignite)
+
+ fill_volume(ignite, repository)
+
+ ignite.await_event("An irrecoverable error in Jfr", 60,
from_the_beginning=True, log_file="console.log")
+ check_stops(ignite)
+
+ load.stop()
+
+ @cluster(num_nodes=2)
+ @reproducer("disk_full_tests")
+ @ignite_versions(str(DEV_BRANCH))
+ def test_node_stops_on_full_volume_shared_by_logs_and_data(self,
ignite_version):
+ """
+ One volume holds the logs and all the data, as in Ignite's default
layout.
+ """
+ ignite = self.__ignite(ignite_version)
+ ignite.mount_tmpfs(ignite.persistent_root, 256)
+
+ ignite.start()
+ ControlUtility(ignite).activate()
+
+ load = self.__start_load_client(ignite)
+
+ fill_volume(ignite, ignite.persistent_root)
+
+ check_stops(ignite)
+
+ load.stop()
+
+ @cluster(num_nodes=2)
+ @reproducer("disk_full_tests")
+ @ignite_versions(str(DEV_BRANCH))
+ def test_node_survives_full_wal_volume(self, ignite_version):
+ """
+ WAL segments are preallocated, so puts go on into a full WAL volume.
+ """
+ ignite = self.__ignite(ignite_version)
+ wal = os.path.join(ignite.persistent_root, "wal")
+ ignite.mount_tmpfs(wal, 64)
+
+ ignite.start()
+ ControlUtility(ignite).activate()
+
+ load = self.__start_load_client(ignite)
+
+ segments = ignite.exec_command(ignite.nodes[0], f"ls {wal}/*/*.wal
2>/dev/null | wc -l")
+ assert int(segments) > 0, "No WAL segments on the volume"
+
+ fill_volume(ignite, wal)
+
+ check_serves_load_for(ignite, load, seconds=15)
+
+ load.stop()
+
+ @cluster(num_nodes=1)
+ @reproducer("disk_full_tests")
+ @ignite_versions(str(DEV_BRANCH))
+ def test_node_stops_on_wal_volume_smaller_than_its_segments(self,
ignite_version):
+ """
+ The node formats its WAL segments at start, and they don't fit the
volume.
+ """
+ ignite = self.__ignite(ignite_version)
+
+ # Ten 4 MB segments need 40 MB.
+ ignite.mount_tmpfs(os.path.join(ignite.persistent_root, "wal"), 16)
+
+ ignite.start_async()
+
+ check_stops(ignite)
+ check_failure(ignite, "CRITICAL_ERROR", "Failed to format WAL segment
file")
+
+ @cluster(num_nodes=2)
+ @reproducer("disk_full_tests")
+ @ignite_versions(str(DEV_BRANCH))
+ @matrix(volume=["wal_archive", "partitions", "work"])
+ def test_node_stops_on_full_volume(self, ignite_version, volume):
+ """
+ The node stops once it can't write the WAL archive, the partitions or
the binary metadata (in work).
+ """
+ ignite = self.__ignite(ignite_version)
+ path = os.path.join(ignite.persistent_root, volume)
+ ignite.mount_tmpfs(path, 64)
+
+ ignite.start()
+ ControlUtility(ignite).activate()
+
+ load = self.__start_load_client(ignite)
+
+ fill_volume(ignite, path)
+
+ check_stops(ignite)
+ check_failure(ignite, *FAILURES[volume])
+
+ load.stop()
+
+ @cluster(num_nodes=2)
+ @reproducer("disk_full_tests")
+ @ignite_versions(str(DEV_BRANCH))
+ def test_node_stops_when_cdc_fills_wal_archive_volume(self,
ignite_version):
+ """
+ CDC with no consumer and no cdcWalDirectoryMaxSize keeps every
archived segment.
+ """
+ ignite = self.__ignite(ignite_version)
+ archive = os.path.join(ignite.persistent_root, "wal_archive")
+ data_storage = ignite.config.data_storage
+
+ # CDC hard-links archived segments, so its directory must be on the
archive's volume.
+ ignite.config =
ignite.config._replace(data_storage=data_storage._replace(
+ cdc_wal_path=os.path.join(archive, "cdc"),
default=data_storage.default._replace(cdc_enabled=True)))
+ ignite.mount_tmpfs(archive, 256)
+
+ ignite.start()
+ ControlUtility(ignite).activate()
+
+ load = self.__start_load_client(ignite)
+
+ check_stops(ignite, timeout_sec=120)
+ check_failure(ignite, *FAILURES["wal_archive"])
+
+ load.stop()
+
+ def __ignite(self, ignite_version):
+ """
+ :return: Persistent server node that writes its stdout and stderr into
console.log itself, with the
+ partitions, the WAL and its archive in directories of their
own.
+ """
+ ignite = IgniteService(self.test_context, IgniteConfiguration(
+ version=IgniteVersion(ignite_version),
+ client_connector_configuration=ClientConnectorConfiguration(),
+ metrics_log_frequency=1000,
+ data_storage=DataStorageConfiguration(
+ checkpoint_frequency=1000,
+ wal_segment_size=4 * MB,
+ max_wal_archive_size=32 * MB,
+ default=DataRegionConfiguration(persistence_enabled=True,
max_size=256 * MB))), num_nodes=1)
+
+ ignite.spec = ignite.spec.rebuild_as(IgniteNodeSpecFileOutput)
+
+ root = ignite.persistent_root
+
+ ignite.config =
ignite.config._replace(data_storage=ignite.config.data_storage._replace(
+ storage_path=os.path.join(root, "partitions"),
+ wal_path=os.path.join(root, "wal"),
+ wal_archive_path=os.path.join(root, "wal_archive")))
+
+ return ignite
+
+ def __start_load_client(self, ignite):
+ """
+ Starts a load that records its puts and errors once a second.
+ :return: Load application service.
+ """
+ address =
f"{ignite.nodes[0].account.hostname}:{ignite.config.client_connector_configuration.port}"
+
+ load = IgniteApplicationService(
+ self.test_context,
+ IgniteThinClientConfiguration(addresses=[address],
version=ignite.config.version),
+ java_class_name=self.LOAD_APP,
+ params={"cacheName": "disk-full"})
+
+ load.start()
+
+ return load
+
+ def __start_flood_service(self, ignite):
+ """
+ Starts the application that deploys the System.out flood service on
the server.
+ :return: Flood application service.
+ """
+ flood = IgniteApplicationService(
+ self.test_context,
+ IgniteConfiguration(version=ignite.config.version,
client_mode=True,
+ discovery_spi=from_ignite_cluster(ignite)),
+ java_class_name=self.FLOOD_APP)
+
+ flood.start()
+
+ return flood
+
+
+def log_files(ignite):
+ """
+ :return: The node's log4j2 log, console.log and gc.log, all on the log
volume.
+ """
+ return [ignite.nodes[0].log_file] + [os.path.join(ignite.log_dir, log) for
log in ("console.log", "gc.log")]
+
+
+def fill_volume(ignite, path):
+ """
+ Writes a file into the path until its volume refuses the write.
+ """
+ out = ignite.exec_command(ignite.nodes[0], f"dd if=/dev/zero
of={os.path.join(path, FILLER)} bs=1M 2>&1")
+
+ assert "No space left on device" in out, f"{path} did not fill up: {out}"
+
+
+def free_volume(ignite, path):
+ """
+ Removes the file fill_volume wrote.
+ """
+ ignite.nodes[0].account.ssh(f"rm -f {os.path.join(path, FILLER)}",
allow_fail=False)
+
+
+def enlarge_volume(ignite, path, size_mb):
+ """
+ Grows the tmpfs mounted over the path, keeping its files and what they
hold, so writes there go on.
+ """
+ ignite.nodes[0].account.ssh(f"sudo mount -o remount,size={size_mb}m
{path}", allow_fail=False)
+
+
+def free_space(ignite, path):
+ """
+ :return: Bytes free on the volume of the path.
+ """
+ return int(ignite.exec_command(ignite.nodes[0], f"df --output=avail -B1
{path} | tail -1"))
+
+
+def printed_lines(flood):
+ """
+ :return: Lines the service wrote to System.out, as the flood application
recorded last.
+ """
+ return int(flood.extract_results("lines")[-1])
+
+
+def file_size(ignite, path):
+ """
+ :return: Bytes written into the file on the node.
+ """
+ return int(ignite.exec_command(ignite.nodes[0], f"stat -c %s {path}"))
+
+
+def check_write_failed(ignite, path):
+ """
+ Checks that the file ends on a page boundary: a write the full tmpfs
refused filled its last page.
+ """
+ assert file_size(ignite, path) % PAGE == 0, f"No write into {path} hit the
full volume"
+
+
+def check_grows(ignite, path):
+ """
+ Checks that the file grows within 15 s.
+ """
+ size = file_size(ignite, path)
+
+ wait_until(lambda: file_size(ignite, path) > size, timeout_sec=15,
err_msg=f"{path} did not grow")
+
+
+def check_serves_load_for(ignite, load, seconds):
+ """
+ Waits the given seconds and checks that the node is alive and the puts
went on without errors.
+ """
+ puts, errors = progress(load)
+
+ time.sleep(seconds)
+
+ assert ignite.alive(ignite.nodes[0]), "The node stopped"
+
+ puts_after, errors_after = progress(load)
+
+ assert puts_after > puts, f"No puts in {seconds} s"
+ assert errors_after == errors, f"{errors_after - errors} puts failed in
{seconds} s"
+
+
+def check_stops(ignite, timeout_sec=60):
+ """
+ Checks that the node's process ends.
+ """
+ wait_until(lambda: not ignite.alive(ignite.nodes[0]),
timeout_sec=timeout_sec,
+ err_msg=f"The node is still up {timeout_sec} s after its volume
filled up")
+
+
+def check_failure(ignite, failure_type, cause):
+ """
+ Checks that the node logged a FailureContext of the type, caused by the
writer and "No space left on device".
+ """
+ node = ignite.nodes[0]
+
+ failure = ignite.exec_command(node, f"grep -h -m1 -F -A60 'FailureContext
[type={failure_type}' "
+ f"{node.log_file} || true")
+
+ assert failure, f"No {failure_type} failure in the node's log"
+ assert cause in failure, f"The {failure_type} failure is not caused by
'{cause}': {failure[:2000]}"
+ assert "No space left on device" in failure, f"The {failure_type} failure
is not ENOSPC: {failure[:2000]}"
+
+
+def progress(load):
+ """
+ :return: Puts and failed puts the load application recorded last.
+ """
+ puts, errors = load.extract_results("progress")[-1].split()
+
+ return int(puts), int(errors)
diff --git a/modules/ducktests/tests/ignitetest/utils/__init__.py
b/modules/ducktests/tests/ignitetest/utils/__init__.py
index 6d6da0d14ed..2cc208c44ba 100644
--- a/modules/ducktests/tests/ignitetest/utils/__init__.py
+++ b/modules/ducktests/tests/ignitetest/utils/__init__.py
@@ -17,6 +17,6 @@
This module contains convenient utils for test.
"""
-from ._mark import ignore_if, ignite_versions, cluster
+from ._mark import ignore_if, ignite_versions, cluster, reproducer
-__all__ = ['ignore_if', 'ignite_versions', 'cluster']
+__all__ = ['ignore_if', 'ignite_versions', 'cluster', 'reproducer']
diff --git a/modules/ducktests/tests/ignitetest/utils/_mark.py
b/modules/ducktests/tests/ignitetest/utils/_mark.py
index 9d0a06df4ed..5964312fcc3 100644
--- a/modules/ducktests/tests/ignitetest/utils/_mark.py
+++ b/modules/ducktests/tests/ignitetest/utils/_mark.py
@@ -51,6 +51,26 @@ class IgnoreIf(Ignore):
return super().__eq__(other) and self.condition == other.condition
+class Reproducer(Ignore):
+ """
+ Ignore test unless the global that enables it is set.
+ """
+ def __init__(self, global_name):
+ super().__init__()
+ self.global_name = global_name
+
+ def apply(self, seed_context, context_list):
+ assert len(context_list) > 0, "reproducer decorator is not being
applied to any test cases"
+
+ for ctx in context_list:
+ ctx.ignore = ctx.ignore or not ctx.globals.get(self.global_name)
+
+ return context_list
+
+ def __eq__(self, other):
+ return super().__eq__(other) and self.global_name == other.global_name
+
+
class IgniteVersionParametrize(Mark):
"""
Parametrize function with ignite_version
@@ -219,6 +239,20 @@ def ignore_if(condition, *,
variable_name='ignite_version'):
return ignorer
+def reproducer(global_name):
+ """
+ Mark decorated test method as IGNORE unless the global is set, e.g.
--globals '{"<global_name>": true}'.
+ For tests that reproduce a behavior on demand rather than guard it on
every run.
+
+ :param global_name: name of the global that enables the test.
+ """
+ def ignorer(func):
+ Mark.mark(func, Reproducer(global_name))
+ return func
+
+ return ignorer
+
+
def cluster(**kwargs):
"""Test method decorator used to provide hints about how the test will use
the given cluster.