This is an automated email from the ASF dual-hosted git repository.
anton-vinogradov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new d30df8c0e80 IGNITE-28976 [ducktests] Add optional demo breakpoints to
pause a running scenario for inspection (#13470)
d30df8c0e80 is described below
commit d30df8c0e80207808962a0ea080acb05f8c16c91
Author: Maksim Davydov <[email protected]>
AuthorDate: Tue Sep 8 15:23:03 2026 +0300
IGNITE-28976 [ducktests] Add optional demo breakpoints to pause a running
scenario for inspection (#13470)
---
.gitignore | 1 +
modules/ducktests/README.md | 66 ++-
.../requirements-dev.txt => checks/__init__.py} | 4 -
.../services/__init__.py} | 4 -
.../services/network_group/__init__.py} | 4 -
.../support/__init__.py} | 4 -
.../tests/checks/support/demo_pause_control.py | 108 ++++
.../tests/checks/support/ducktape_doubles.py | 116 +++++
.../utils/__init__.py} | 4 -
.../tests/checks/utils/check_ignite_spec.py | 16 +-
.../utils/pause/__init__.py} | 4 -
.../tests/checks/utils/pause/check_banner.py | 131 +++++
.../tests/checks/utils/pause/check_control.py | 139 ++++++
.../tests/checks/utils/pause/check_control_dir.py | 225 +++++++++
.../tests/checks/utils/pause/check_selector.py | 94 ++++
.../tests/checks/utils/pause/check_timeout.py | 72 +++
modules/ducktests/tests/docker/demo_console.py | 195 ++++++++
.../ducktests/tests/docker/requirements-dev.txt | 1 +
modules/ducktests/tests/docker/run_tests.sh | 9 +
.../tests/ignitetest/services/mdc/mdc_cluster.py | 30 ++
.../ignitetest/services/network_group/manager.py | 32 +-
.../tests/mdc/partition_resilience_test.py | 10 +
.../tests/ignitetest/utils/ignite_test.py | 45 ++
modules/ducktests/tests/ignitetest/utils/pause.py | 541 +++++++++++++++++++++
.../tests/ignitetest/utils/pause_control.py | 250 ++++++++++
modules/ducktests/tests/setup.py | 2 +-
modules/ducktests/tests/tox.ini | 6 +
27 files changed, 2074 insertions(+), 39 deletions(-)
diff --git a/.gitignore b/.gitignore
index 1e684823db1..8ab90f323cf 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,6 +90,7 @@ CMakeSettings.json
#Ducktape
/results
+/.ducktests-demo
.ducktape
*.pyc
/tests/venv
diff --git a/modules/ducktests/README.md b/modules/ducktests/README.md
index ab29714ac8e..2c4d2b690aa 100644
--- a/modules/ducktests/README.md
+++ b/modules/ducktests/README.md
@@ -132,9 +132,8 @@ To locally simulate validation matrices across distinct
target runtimes (e.g., P
pyenv install 3.9
pyenv shell 3.8 3.9
```
-3. Install `tox` and run the validation suite:
+3. Run the validation suite:
```bash
- pip install tox
tox
tox -r -e codestyle,py3
```
@@ -239,6 +238,69 @@ You can target specific cross-product version
compatibility combinations inside
--global-json '{"safepoint_log_enabled": true}'
```
+### Demo Mode (Breakpoints)
+
+Scenarios can be frozen at named breakpoints, so a cluster can be shown to an
audience in exactly that state and then resumed. Ducktape runs the test inside
`ducker01` with stdin closed, so the keyboard lives in a second terminal.
+
+Terminal 1 - run the test with the `demo_pause` global:
+```bash
+./docker/run_tests.sh -n 12 -gj '{"demo_pause": "*"}' \
+ -t
./ignitetest/tests/mdc/partition_resilience_test.py::MdcPartitionResilienceTest.test_mdc_cluster_partition_resilience
+```
+
+Terminal 2 - drive the breakpoints:
+```bash
+python docker/demo_console.py
+```
+
+At every breakpoint the console prints the step, the elapsed time, every
service node with its liveness, the live network state (netem delays and
partition drops as they are actually applied), and ready-to-paste commands for
entering nodes and reading their logs and configs. `Enter` continues, `c` runs
the rest unattended, `a` aborts the test.
+
+While paused the cluster keeps running and any network impairment stays in
effect, so nodes can be inspected freely:
+```bash
+./docker/ducker-ignite ssh ducker03
+docker exec ducker03 bash -c "tail -n 50 /mnt/service/logs/ignite*.log"
+docker exec ducker03 cat /mnt/service/config/ignite-config.xml
+```
+
+The console is optional - the test communicates through files under
`<repository root>/.ducktests-demo`, which is shared with the host by the same
bind mount that carries the repository into the containers. Each file is a
simple signal - its presence is the command, its content (where any) is the
data:
+
+| File | Meaning |
+|------|---------|
+| `paused.txt` | The banner of the breakpoint currently held, human readable. |
+| `paused.json` | The same banner plus run metadata, for the console to read. |
+| `continue-<N>` | Resume breakpoint `N`, where `N` is the number shown in the
banner's `PAUSED N` line. The banner itself prints the exact `touch` command to
paste. |
+| `continue-all` | Resume and skip every remaining breakpoint. |
+| `abort` | Fail the test and tear down the cluster. |
+
+```bash
+cat .ducktests-demo/paused.txt # read the banner of the breakpoint
currently held
+touch .ducktests-demo/continue-3 # resume breakpoint 3 (the number matches
PAUSED 3 in the banner)
+touch .ducktests-demo/continue-all # resume and skip the remaining breakpoints
+touch .ducktests-demo/abort # fail the test and tear down
+```
+
+Both sides find that directory on their own, so by default nothing has to be
configured. `demo_pause_dir` overrides it - but the two sides name the same
directory differently, since the test runs inside `ducker01`, where the
repository is mounted at `/opt/ignite-dev`, while the console runs on the host.
Override it and the console has to be pointed at the host side of it:
+```bash
+./docker/run_tests.sh -gj '{"demo_pause": "*", "demo_pause_dir":
"/opt/ignite-dev/.demo"}' -t ./ignitetest/tests/<some_test.py>
+
+python docker/demo_console.py -d .demo
+```
+
+Breakpoints are added to a test with `self.pause("name", mdc, net)` and cost
nothing when the global is absent, which is how they stay in the tests without
affecting CI. Run one test at a time in demo mode (no `--max-parallel`): a
single control directory holds one breakpoint at a time.
+
+A held test reports nothing back to ducktape, which kills a session it has
heard nothing from for `--test-runner-timeout` (30 minutes by default) - and
that budget is measured from the construction of the test (setup included),
which is deliberately conservative: ducktape's kill timer is actually reset by
the last client event before the pause, which fires after setup, so the real
window is longer. Breakpoints therefore auto-continue while the runner is still
waiting, shortening themselv [...]
+```bash
+./docker/run_tests.sh -n 12 --test-runner-timeout 7200000 \
+ -gj '{"demo_pause": "*", "demo_pause_timeout_sec": 1800}' \
+ -t ./ignitetest/tests/mdc/partition_resilience_test.py
+```
+
+| Global Parameter Key | Definition | Example Configuration |
+|---------------------|------------|----------------------|
+| **demo_pause** | Which breakpoints stop the scenario. Absent or `false`
disables them all (the default); `true` or `"*"` stops at every one; a list or
comma separated string stops only at the named ones, matched case
insensitively. | ```{"demo_pause": "split-brain,healed"}``` |
+| **demo_pause_timeout_sec** | How long one breakpoint may hold the scenario
before it resumes on its own. Default is 600, and it is capped by what is left
of `--test-runner-timeout`. | ```{"demo_pause_timeout_sec": 1800}``` |
+| **demo_pause_dir** | Control directory shared with the host, named as the
test sees it - inside the containers the repository is `/opt/ignite-dev`.
Default is `<repository root>/.ducktests-demo`; anything else has to be passed
to the console as well, with `-d` and the host path. | ```{"demo_pause_dir":
"/opt/ignite-dev/.demo"}``` |
+
### Security Settings
```bash
# Enable built-in authentication overrides
diff --git a/modules/ducktests/tests/docker/requirements-dev.txt
b/modules/ducktests/tests/checks/__init__.py
similarity index 94%
copy from modules/ducktests/tests/docker/requirements-dev.txt
copy to modules/ducktests/tests/checks/__init__.py
index f868e43d674..ec2014340d7 100644
--- a/modules/ducktests/tests/docker/requirements-dev.txt
+++ b/modules/ducktests/tests/checks/__init__.py
@@ -12,7 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
--r requirements.txt
-pytest==6.2.5
-flake8==6.1.0
diff --git a/modules/ducktests/tests/docker/requirements-dev.txt
b/modules/ducktests/tests/checks/services/__init__.py
similarity index 94%
copy from modules/ducktests/tests/docker/requirements-dev.txt
copy to modules/ducktests/tests/checks/services/__init__.py
index f868e43d674..ec2014340d7 100644
--- a/modules/ducktests/tests/docker/requirements-dev.txt
+++ b/modules/ducktests/tests/checks/services/__init__.py
@@ -12,7 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
--r requirements.txt
-pytest==6.2.5
-flake8==6.1.0
diff --git a/modules/ducktests/tests/docker/requirements-dev.txt
b/modules/ducktests/tests/checks/services/network_group/__init__.py
similarity index 94%
copy from modules/ducktests/tests/docker/requirements-dev.txt
copy to modules/ducktests/tests/checks/services/network_group/__init__.py
index f868e43d674..ec2014340d7 100644
--- a/modules/ducktests/tests/docker/requirements-dev.txt
+++ b/modules/ducktests/tests/checks/services/network_group/__init__.py
@@ -12,7 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
--r requirements.txt
-pytest==6.2.5
-flake8==6.1.0
diff --git a/modules/ducktests/tests/docker/requirements-dev.txt
b/modules/ducktests/tests/checks/support/__init__.py
similarity index 94%
copy from modules/ducktests/tests/docker/requirements-dev.txt
copy to modules/ducktests/tests/checks/support/__init__.py
index f868e43d674..ec2014340d7 100644
--- a/modules/ducktests/tests/docker/requirements-dev.txt
+++ b/modules/ducktests/tests/checks/support/__init__.py
@@ -12,7 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
--r requirements.txt
-pytest==6.2.5
-flake8==6.1.0
diff --git a/modules/ducktests/tests/checks/support/demo_pause_control.py
b/modules/ducktests/tests/checks/support/demo_pause_control.py
new file mode 100644
index 00000000000..97d5d31b676
--- /dev/null
+++ b/modules/ducktests/tests/checks/support/demo_pause_control.py
@@ -0,0 +1,108 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+The host side of a demo breakpoint, for checks that drive one.
+
+A held breakpoint blocks the thread that reached it, so everything the host
does - reading the
+published banner, dropping a resume file - has to happen from another one
while the check
+itself sits inside :meth:`ignitetest.utils.pause.DemoPause.pause`.
+
+The protocol itself is spoken through
:class:`ignitetest.utils.pause_control.ControlDir`, the
+same class the test and ``docker/demo_console.py`` use - a check that hand
rolled the file
+names would stop checking the protocol and start checking its own copy of it.
+"""
+
+import threading
+import time
+from contextlib import contextmanager
+
+from ignitetest.utils.pause import DEMO_PAUSE_TIMEOUT_SEC, DemoPause
+from ignitetest.utils.pause_control import ControlDir
+
+from checks.support.ducktape_doubles import FakeLogger
+
+# Stands for the test a breakpoint was reached in; breakpoints report it to
the host.
+TEST_NAME = "check.CheckPause.check_something"
+
+# Far longer than the fraction of a second a check actually holds a breakpoint
for, and far
+# shorter than the framework's own default: a resume that never arrives has to
fail the check
+# that expected it rather than hold the suite for ten minutes.
+RESUME_TIMEOUT_SEC = 30
+
+
+def new_demo_pause(control_dir, started_at=None, runner_timeout_sec=None,
**test_globals):
+ """
+ :return: A DemoPause over the given control directory, logging into a
FakeLogger its
+ ``logger`` attribute hands back to the check.
+ """
+ test_globals.setdefault(DEMO_PAUSE_TIMEOUT_SEC, RESUME_TIMEOUT_SEC)
+
+ return DemoPause(FakeLogger(), test_globals, TEST_NAME,
control_dir=str(control_dir),
+ started_at=started_at,
runner_timeout_sec=runner_timeout_sec)
+
+
+def resume_with(control_dir, name, delay_sec=.05):
+ """
+ Creates a resume file from another thread, the way the host does while the
test blocks.
+ """
+ control = ControlDir(control_dir)
+
+ timer = threading.Timer(delay_sec, lambda: control.resume(name))
+ timer.daemon = True
+ timer.start()
+
+
+@contextmanager
+def published_status(control_dir, resume=None, timeout_sec=30):
+ """
+ Reads the published breakpoint while the check blocks on it, the way the
host console
+ does, and optionally resumes it.
+
+ Polls for the file rather than reading it once after a fixed delay: a
breakpoint that is
+ only held for a fraction of a second - which is what these checks hold
them for - would
+ otherwise be a race against the machine the checks happen to run on.
+
+ :param resume: Name of the resume file to create once the breakpoint has
been read, None
+ to leave it held.
+ :return: A dict, empty on entry and filled with the published breakpoint
by the time the
+ block is left.
+ """
+ control = ControlDir(control_dir)
+ published = {}
+
+ def read():
+ deadline = time.monotonic() + timeout_sec
+
+ while time.monotonic() < deadline:
+ status = control.read_status()
+
+ if status is not None:
+ published.update(status)
+
+ break
+
+ time.sleep(.01)
+
+ if resume:
+ control.resume(resume)
+
+ reader = threading.Thread(target=read, daemon=True)
+ reader.start()
+
+ try:
+ yield published
+ finally:
+ reader.join(timeout_sec)
diff --git a/modules/ducktests/tests/checks/support/ducktape_doubles.py
b/modules/ducktests/tests/checks/support/ducktape_doubles.py
new file mode 100644
index 00000000000..7a8621e5831
--- /dev/null
+++ b/modules/ducktests/tests/checks/support/ducktape_doubles.py
@@ -0,0 +1,116 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Stand-ins for the ducktape objects a test is handed - a logger, nodes,
services and the
+registry they are collected in - for checks of framework code that only reads
them.
+
+Each double is as poor as the real thing is at the point of use, so that a
check fails on
+code reaching for more than a test actually offers it.
+"""
+
+from types import SimpleNamespace
+
+from ignitetest.services.utils.path import IgnitePathAware
+
+
+class FakeLogger:
+ """
+ Collects what the code under check would have logged.
+ """
+ def __init__(self):
+ self.messages = []
+
+ def info(self, msg):
+ """Records an info message."""
+ self.messages.append(msg)
+
+ def warn(self, msg):
+ """Records a warning."""
+ self.messages.append(msg)
+
+ debug = info
+ error = warn
+
+
+def fake_nodes(*hostnames):
+ """
+ :return: Nodes carrying the account attributes that ducktape's do.
+ """
+ return [SimpleNamespace(account=SimpleNamespace(hostname=host,
externally_routable_ip=host))
+ for host in hostnames]
+
+
+class FakeService:
+ """
+ Stands in for a non-Ignite service of the test registry, e.g. a zookeeper
one: it carries
+ paths of its own, which code following the Ignite services must not hand
out for Ignite
+ nodes.
+ """
+ log_dir = "/mnt/service/zk-logs"
+ config_file = "/mnt/service/zookeeper.properties"
+
+ def __init__(self, *hostnames):
+ self.nodes = fake_nodes(*hostnames)
+
+ def who_am_i(self, node):
+ """Names the node the way a ducktape service does."""
+ return f"{self.__class__.__name__}-{node.account.hostname}"
+
+
+class FakeIgniteService(IgnitePathAware):
+ """
+ Stands in for an Ignite service, with the real path layout behind it.
+ """
+ def __init__(self, *hostnames):
+ self.nodes = fake_nodes(*hostnames)
+
+ def who_am_i(self, node):
+ """Names the node the way a ducktape service does."""
+ return f"{self.__class__.__name__}-{node.account.hostname}"
+
+ @property
+ def product(self):
+ return "ignite-dev"
+
+ @property
+ def globals(self):
+ return {}
+
+
+class FakeBrokenService:
+ """
+ Stands in for a service that can no longer answer for the nodes it still
holds, the way a
+ ducktape one does once a node has been freed from it: ``who_am_i`` goes
through ``idx()``,
+ which raises for a node the service does not own.
+ """
+ def __init__(self, *hostnames):
+ self.nodes = fake_nodes(*hostnames)
+
+ def who_am_i(self, node):
+ """Fails the way a ducktape service does for a node it does not own."""
+ raise RuntimeError(f"Could not find node {node}")
+
+
+class FakeRegistry:
+ """
+ Stands in for ducktape's ServiceRegistry, which is what a test hands to
the framework: it
+ is iterable and nothing else, so code reading it may not index it or ask
it for a length.
+ """
+ def __init__(self, *services):
+ self._services = services
+
+ def __iter__(self):
+ return iter(self._services)
diff --git a/modules/ducktests/tests/docker/requirements-dev.txt
b/modules/ducktests/tests/checks/utils/__init__.py
similarity index 94%
copy from modules/ducktests/tests/docker/requirements-dev.txt
copy to modules/ducktests/tests/checks/utils/__init__.py
index f868e43d674..ec2014340d7 100644
--- a/modules/ducktests/tests/docker/requirements-dev.txt
+++ b/modules/ducktests/tests/checks/utils/__init__.py
@@ -12,7 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
--r requirements.txt
-pytest==6.2.5
-flake8==6.1.0
diff --git a/modules/ducktests/tests/checks/utils/check_ignite_spec.py
b/modules/ducktests/tests/checks/utils/check_ignite_spec.py
index a3c8e521049..b60fce333e1 100644
--- a/modules/ducktests/tests/checks/utils/check_ignite_spec.py
+++ b/modules/ducktests/tests/checks/utils/check_ignite_spec.py
@@ -16,6 +16,7 @@
"""
Checks Spec class that describes config and command line to start Ignite-aware
service.
"""
+import os
from unittest.mock import Mock
import pytest
@@ -90,10 +91,15 @@ def
check_boolean_options__go_after_default_ones_and_overwrite_them__if_passed_v
def
check_colon_options__go_after_default_ones_and_overwrite_them__if_passed_via_jvm_opt(service):
service.log_dir = "/default-path"
+
+ # The default is built with os.path.join, which follows the control
machine rather than the
+ # node - so the expectation is joined the same way instead of being
written out, or the check
+ # would only hold where the separator happens to be "/".
+ default_gc_log = ("-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:"
+ f"{os.path.join(service.log_dir,
'gc.log')}:uptime,time,level,tags")
+
spec = IgniteApplicationSpec(service,
jvm_opts=["-Xlog:gc:/some-non-default-path/gc.log"])
+
assert "-Xlog:gc:/some-non-default-path/gc.log" in spec.jvm_opts
- assert
"-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:/default-path/gc.log:uptime,time,level,tags"
\
- in spec.jvm_opts
- assert spec.jvm_opts.index("-Xlog:gc:/some-non-default-path/gc.log") > \
- spec.jvm_opts.index(
-
"-Xlog:gc*=debug,gc+stats*=debug,gc+ergo*=debug:/default-path/gc.log:uptime,time,level,tags")
+ assert default_gc_log in spec.jvm_opts
+ assert spec.jvm_opts.index("-Xlog:gc:/some-non-default-path/gc.log") >
spec.jvm_opts.index(default_gc_log)
diff --git a/modules/ducktests/tests/docker/requirements-dev.txt
b/modules/ducktests/tests/checks/utils/pause/__init__.py
similarity index 94%
copy from modules/ducktests/tests/docker/requirements-dev.txt
copy to modules/ducktests/tests/checks/utils/pause/__init__.py
index f868e43d674..ec2014340d7 100644
--- a/modules/ducktests/tests/docker/requirements-dev.txt
+++ b/modules/ducktests/tests/checks/utils/pause/__init__.py
@@ -12,7 +12,3 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-
--r requirements.txt
-pytest==6.2.5
-flake8==6.1.0
diff --git a/modules/ducktests/tests/checks/utils/pause/check_banner.py
b/modules/ducktests/tests/checks/utils/pause/check_banner.py
new file mode 100644
index 00000000000..d32443388de
--- /dev/null
+++ b/modules/ducktests/tests/checks/utils/pause/check_banner.py
@@ -0,0 +1,131 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks the banner a held demo breakpoint renders: how long the scenario has
been running, the
+nodes it is made of, and the commands offered for looking into them.
+"""
+
+import os
+import time
+
+from ignitetest.utils.pause import ALL, DemoPause, _resume_command
+from ignitetest.utils.pause_control import ControlDir, repo_root
+
+from checks.support.demo_pause_control import new_demo_pause, published_status
+from checks.support.ducktape_doubles import FakeBrokenService,
FakeIgniteService, FakeRegistry, FakeService
+
+
+def check_elapsed_is_counted_from_test_start(tmp_path):
+ """
+ Check that the banner counts from the start of the test rather than from
the first
+ breakpoint: the setup phase of a multi-node scenario is minutes long, and
a demo that
+ reports t+00:00 after it hides exactly the part worth showing.
+ """
+ demo = new_demo_pause(tmp_path, started_at=time.monotonic() - 600,
demo_pause=ALL, demo_pause_timeout_sec=.3)
+
+ with published_status(tmp_path) as published:
+ demo.pause("split-brain")
+
+ assert published["elapsed_sec"] >= 600
+ assert any("t+10:00 since test start" in line for line in
published["banner"])
+
+
+def check_banner_is_rendered_from_the_service_registry(tmp_path):
+ """
+ Check that the banner is built by iterating the services alone: what a
test passes is
+ ducktape's ServiceRegistry, which supports nothing else.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3)
+
+ with published_status(tmp_path) as published:
+ demo.pause("split-brain",
services=FakeRegistry(FakeService("ducker02"), FakeIgniteService("ducker03")))
+
+ banner = "\n".join(published["banner"]).replace("\\", "/")
+
+ assert "FakeService-ducker02" in banner
+ assert "FakeIgniteService-ducker03" in banner
+ assert "/mnt/service/logs/ignite*.log" in banner, "the hints must still
follow the Ignite service"
+ assert "ducker02 ducker03" in banner
+
+
+def check_a_service_that_cannot_answer_is_degraded_not_raised(tmp_path):
+ """
+ Check that a service which cannot answer for its nodes costs the banner
those lines and
+ nothing more. A breakpoint only observes the cluster, so one that throws
while rendering
+ would fail the scenario at exactly the point the demo was added to show.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3)
+
+ with published_status(tmp_path) as published:
+ demo.pause("split-brain",
+ services=FakeRegistry(FakeBrokenService("ducker02"),
FakeIgniteService("ducker03")))
+
+ banner = "\n".join(published["banner"])
+
+ assert "FakeBrokenService-ducker02" in banner, "the node must still be
named, without asking its service"
+ assert "RuntimeError" in banner, "and what could not be read must say so"
+ assert "FakeIgniteService-ducker03" in banner, "a service after the broken
one must still be listed"
+ assert "ducker02 ducker03" in banner, "and every node must still be
offered by the hints"
+
+
+def check_the_resume_command_is_named_where_it_will_be_typed():
+ """
+ Check that the command a breakpoint offers is one its reader can actually
run. The banner
+ is rendered inside ducker01, where the repository is mounted at
/opt/ignite-dev, and the
+ console prints it verbatim on the host, where that path does not exist -
so the shared
+ repository root is the only anchor the two sides have in common.
+ """
+ inside = ControlDir(os.path.join(repo_root(), ".ducktests-demo"))
+
+ command = _resume_command(inside, 3)
+
+ assert command == "touch .ducktests-demo/continue-3 (from the repository
root)"
+ assert repo_root() not in command, "an absolute path here names a
directory the reader may not have"
+
+
+def check_the_resume_command_falls_back_to_the_whole_path():
+ """
+ Check that a control directory outside the repository is still named in
full: there is no
+ shared anchor left to make it relative to, so a whole path is the honest
answer.
+ """
+ outside = ControlDir(os.path.join(os.path.dirname(repo_root()),
"elsewhere", "demo"))
+
+ command = _resume_command(outside, 3)
+
+ assert command.endswith(os.path.join("elsewhere", "demo", "continue-3"))
+ assert "from the repository root" not in command
+
+
+def check_hints_follow_the_ignite_services():
+ """
+ Check that the copy-pasteable commands name the Ignite paths even when a
service of
+ another kind was registered first, as the zookeeper discovery scenarios do.
+ """
+ # noinspection PyProtectedMember
+ hints = "\n".join(DemoPause._hints_section([FakeService("ducker02"), #
pylint: disable=protected-access
+
FakeIgniteService("ducker03")]))
+
+ # The service paths come from os.path.join, which follows the control
machine rather than
+ # the nodes - a check that runs on Windows would otherwise see its
separators.
+ hints = hints.replace("\\", "/")
+
+ assert "/mnt/service/config/ignite-config.xml" in hints
+ assert "/mnt/service/logs/ignite*.log" in hints
+ assert "zookeeper.properties" not in hints
+ assert "zk-logs" not in hints
+
+ # Every node is still offered, whichever service it belongs to.
+ assert "ducker02 ducker03" in hints
diff --git a/modules/ducktests/tests/checks/utils/pause/check_control.py
b/modules/ducktests/tests/checks/utils/pause/check_control.py
new file mode 100644
index 00000000000..e84c83478a6
--- /dev/null
+++ b/modules/ducktests/tests/checks/utils/pause/check_control.py
@@ -0,0 +1,139 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks the file protocol a demo breakpoint and the host speak over the control
directory:
+what a held breakpoint publishes, what resumes it, and what is swept up
afterwards.
+"""
+
+import os
+
+import pytest
+
+from ignitetest.utils.pause import ALL
+from ignitetest.utils.pause_control import ABORT, CONTINUE_ALL, STATUS_JSON,
STATUS_TXT, continue_file
+
+from checks.support.demo_pause_control import TEST_NAME, new_demo_pause,
published_status, resume_with
+
+
+def check_publishes_and_consumes_status(tmp_path):
+ """
+ Check the published breakpoint - what the host reads - and that the test
cleans it up
+ once resumed, so a stale banner never outlives the pause it describes.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause=True)
+
+ with published_status(tmp_path, resume=continue_file(1)) as published:
+ demo.pause("split-brain", services=[])
+
+ assert published["seq"] == 1
+ assert published["run"] == demo.run
+ assert published["name"] == "split-brain"
+ assert published["test"] == TEST_NAME
+ assert any("PAUSED 1 split-brain" in line for line in
published["banner"])
+
+ for leftover in (STATUS_JSON, STATUS_TXT, continue_file(1)):
+ assert not os.path.exists(str(tmp_path / leftover)), leftover
+
+
+def check_continue_all_skips_the_rest(tmp_path):
+ """
+ Check that continue-all resumes the current breakpoint and disables every
later one, so
+ a demo can be cut short without restarting the scenario.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause=ALL)
+
+ resume_with(tmp_path, CONTINUE_ALL)
+
+ demo.pause("split-brain")
+
+ assert demo.seq == 1
+ assert not demo.enabled
+
+ demo.pause("healed")
+
+ assert demo.seq == 1, "breakpoints after continue-all must not stop the
scenario"
+ assert not os.path.exists(str(tmp_path / CONTINUE_ALL))
+
+
+def check_abort_fails_the_test(tmp_path):
+ """
+ Check that abort ends the scenario through an assertion, so ducktape tears
the cluster
+ down instead of leaving it running.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause=ALL)
+
+ resume_with(tmp_path, ABORT)
+
+ with pytest.raises(AssertionError, match="split-brain"):
+ demo.pause("split-brain")
+
+ assert not os.path.exists(str(tmp_path / ABORT))
+ assert not os.path.exists(str(tmp_path / STATUS_JSON))
+
+
+def
check_a_control_directory_that_cannot_be_made_does_not_fail_the_scenario(tmp_path):
+ """
+ Check that a control directory which cannot even be created costs the demo
and nothing
+ else. It is a bind mount of the host repository, so it can be read only or
owned by
+ another user - neither of which says anything about the cluster under
test, and a
+ breakpoint must not be what turns a passing run red.
+ """
+ # A file where the directory should go: the portable way to make
os.makedirs fail.
+ (tmp_path / "in-the-way").write_text("not a directory", encoding="utf-8")
+
+ demo = new_demo_pause(tmp_path / "in-the-way" / "control", demo_pause=ALL)
+
+ demo.pause("split-brain")
+
+ assert demo.seq == 0, "the scenario must have carried straight on"
+ assert not demo.enabled, "and the later breakpoints must not try it again"
+ assert any("control directory cannot be used" in msg for msg in
demo.logger.messages), demo.logger.messages
+
+
+def check_a_breakpoint_that_cannot_be_published_is_skipped(tmp_path):
+ """
+ Check the same where the directory exists but the banner cannot be
written. Blocking would
+ be no better than raising here: nothing reached the host, so there would
be nothing on
+ screen to resume, and the scenario would sit there for the whole timeout.
+ """
+ # A directory where the banner goes makes the write fail wherever these
checks run. The
+ # sweep cannot remove it either, so it is still in the way when the
breakpoint publishes.
+ os.mkdir(str(tmp_path / STATUS_TXT))
+
+ demo = new_demo_pause(tmp_path, demo_pause=ALL)
+
+ demo.pause("split-brain")
+
+ assert not demo.enabled
+ assert not os.path.exists(str(tmp_path / STATUS_JSON)), "half a breakpoint
must not be left published"
+ assert any("control directory cannot be used" in msg for msg in
demo.logger.messages), demo.logger.messages
+
+
+def check_stale_resume_file_is_cleared(tmp_path):
+ """
+ Check that a resume file left by a previous run does not skip the first
breakpoint of
+ this one - the control directory outlives a test, its contents must not.
+ """
+ open(str(tmp_path / continue_file(1)), "w").close()
+ open(str(tmp_path / STATUS_TXT), "w").close()
+
+ demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3)
+
+ demo.pause("split-brain")
+
+ assert demo.seq == 1
+ assert any("timed out" in msg for msg in demo.logger.messages), \
+ "the stale file must have been cleared, leaving the breakpoint to time
out"
diff --git a/modules/ducktests/tests/checks/utils/pause/check_control_dir.py
b/modules/ducktests/tests/checks/utils/pause/check_control_dir.py
new file mode 100644
index 00000000000..ea00c45c11d
--- /dev/null
+++ b/modules/ducktests/tests/checks/utils/pause/check_control_dir.py
@@ -0,0 +1,225 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks the control directory itself - the files a paused test and the host
exchange, and the
+polling that waits for them - with no breakpoint on top of it.
+"""
+
+import os
+import time
+
+from ignitetest.utils.pause_control import ABORT, CONTINUE_ALL, ControlDir,
STATUS_JSON, STATUS_TXT, continue_file
+
+
+def check_consuming_a_file_removes_it(tmp_path):
+ """
+ Check that consuming a control file reports it and removes it: the test is
the only party
+ that deletes what the host wrote, so consuming one is what acknowledges it.
+ """
+ control = ControlDir(tmp_path)
+
+ assert not control.consume(ABORT), "nothing was written yet"
+
+ control.resume(ABORT)
+
+ assert control.exists(ABORT)
+ assert control.consume(ABORT)
+ assert not control.exists(ABORT), "a consumed file must not be left for
the next breakpoint"
+ assert not control.consume(ABORT)
+
+
+def check_sweep_spares_a_held_breakpoint(tmp_path):
+ """
+ Check the asymmetry the two sides need: both clear the resume files an
earlier run left,
+ but only the test drops the banner as well. The console is just as likely
to have been
+ started against a test that is already holding one.
+ """
+ control = ControlDir(tmp_path)
+
+ control.publish(["PAUSED 1 split-brain"], {"seq": 1})
+ control.resume(continue_file(3))
+ control.resume(CONTINUE_ALL)
+
+ control.sweep()
+
+ assert not control.exists(continue_file(3))
+ assert not control.exists(CONTINUE_ALL)
+ assert control.read_status() is not None, "the console must not withdraw a
breakpoint it did not hold"
+
+ control.sweep(status=True)
+
+ assert control.read_status() is None
+ assert not control.exists(STATUS_TXT)
+
+
+def check_sweeping_a_directory_that_is_not_there(tmp_path):
+ """
+ Check that sweeping is safe before anything has been created: without the
global no
+ breakpoint ever makes the directory, and the console may well be started
first.
+ """
+ control = ControlDir(tmp_path / "never-created")
+
+ control.sweep(status=True)
+
+ assert control.read_status() is None
+ assert not os.path.exists(control.path), "sweeping must not create what it
was asked to clean"
+
+
+def check_sweep_clears_what_an_interrupted_write_left(tmp_path):
+ """
+ Check that the temporary name a write goes through is swept too, whichever
file it
+ belonged to: a sweep is the only thing that ever visits the directory
without knowing
+ what it expects to find, so it must also be the thing that cleans such a
leftover.
+ """
+ control = ControlDir(tmp_path)
+
+ for interrupted in (continue_file(1), CONTINUE_ALL, ABORT, STATUS_TXT,
STATUS_JSON):
+ control.write(interrupted + ".tmp", "")
+
+ control.sweep(status=True)
+
+ assert os.listdir(control.path) == []
+
+
+def check_sweep_spares_what_the_protocol_cannot_create(tmp_path):
+ """
+ Check that a sweep removes only names the protocol itself can leave:
matching a fixed
+ name by prefix would take a foreign file that merely starts like one of
its own.
+ """
+ control = ControlDir(tmp_path)
+
+ foreign = ["abort-note.txt", "paused.txt.bak", "resumed.json", "stray.tmp"]
+
+ for name in foreign:
+ control.write(name, "")
+
+ control.sweep(status=True)
+
+ assert sorted(os.listdir(control.path)) == sorted(foreign)
+
+
+def check_a_resume_file_lands_without_a_temporary(tmp_path):
+ """
+ Check that resuming creates the file and nothing besides: a resume file is
empty, so an
+ atomic replace would buy nothing and leave a transient behind for the
sweep to know about.
+ """
+ control = ControlDir(tmp_path)
+
+ control.resume(ABORT)
+
+ assert os.listdir(control.path) == [ABORT]
+
+
+def check_publishing_round_trips(tmp_path):
+ """
+ Check that what is published is what a reader gets back, and that
withdrawing it leaves
+ nothing of either file behind.
+ """
+ control = ControlDir(tmp_path)
+
+ control.publish(["PAUSED 1 split-brain", " test some.Test"], {"seq":
1, "name": "split-brain"})
+
+ assert control.read_status() == {"seq": 1, "name": "split-brain"}
+
+ with open(control.file(STATUS_TXT), encoding="utf-8") as file:
+ assert file.read() == "PAUSED 1 split-brain\n test some.Test\n"
+
+ control.clear_status()
+
+ assert control.read_status() is None
+ assert not control.exists(STATUS_TXT)
+ assert not control.exists(STATUS_JSON)
+
+
+def check_unreadable_status_reads_as_nothing_published(tmp_path):
+ """
+ Check that a status file caught half written reads as "not paused" rather
than raising:
+ the host polls this in a loop and simply comes back.
+ """
+ control = ControlDir(tmp_path)
+
+ control.write(STATUS_JSON, '{"seq": 1')
+
+ assert control.read_status() is None
+
+
+def check_awaiting_takes_the_file_that_arrived(tmp_path):
+ """
+ Check the wait ends on the file that appears, and hands back which one it
was.
+ """
+ control = ControlDir(tmp_path)
+
+ control.resume(continue_file(1))
+
+ assert control.await_any([ABORT, continue_file(1)], 5) == continue_file(1)
+ assert not control.exists(continue_file(1)), "the wait must consume what
ended it"
+
+
+def check_awaiting_honours_the_order_it_was_given(tmp_path):
+ """
+ Check that the first name wins when several are already there: abort has
to beat a
+ continue that landed in the same interval, or a demo would be resumed
instead of ended.
+ """
+ control = ControlDir(tmp_path)
+
+ control.resume(CONTINUE_ALL)
+ control.resume(ABORT)
+
+ assert control.await_any([ABORT, CONTINUE_ALL], 5) == ABORT
+ assert control.exists(CONTINUE_ALL), "only the file that ended the wait
may be consumed"
+
+
+def check_awaiting_gives_up(tmp_path):
+ """
+ Check that a wait nobody answers ends on its own rather than holding the
scenario until
+ ducktape kills it.
+ """
+ control = ControlDir(tmp_path)
+
+ started_at = time.monotonic()
+
+ assert control.await_any([ABORT], .3) is None
+ assert time.monotonic() - started_at >= .3, "it must have waited for what
it was given"
+
+
+def check_awaiting_ticks_without_being_told_how_often(tmp_path):
+ """
+ Check that a caller who wants to hear that the wait is still running need
not also pick an
+ interval - asking for one and getting a TypeError out of a held breakpoint
would be a poor
+ way to find out that the two arguments go together.
+ """
+ control = ControlDir(tmp_path)
+
+ ticks = []
+
+ assert control.await_any([ABORT], .6, tick=ticks.append) is None
+ assert ticks, "a wait longer than one poll must report itself"
+
+
+def check_awaiting_reports_that_it_is_still_waiting(tmp_path):
+ """
+ Check that the caller is ticked while the wait runs, which is how a held
breakpoint says
+ in the test log that it is paused rather than stuck - without this class
having to know
+ what a log is.
+ """
+ control = ControlDir(tmp_path)
+
+ ticks = []
+
+ assert control.await_any([ABORT], 1.2, tick=ticks.append, tick_sec=.01) is
None
+ assert ticks, "a wait longer than the tick interval must report itself"
+ assert all(0 < left <= 1.2 for left in ticks), ticks
+ assert ticks == sorted(ticks, reverse=True), "each tick must report less
time left than the last"
diff --git a/modules/ducktests/tests/checks/utils/pause/check_selector.py
b/modules/ducktests/tests/checks/utils/pause/check_selector.py
new file mode 100644
index 00000000000..f01c5685626
--- /dev/null
+++ b/modules/ducktests/tests/checks/utils/pause/check_selector.py
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks which demo breakpoints stop a scenario: how the ``demo_pause`` global
is read, and
+which of the breakpoints in the test it then selects.
+"""
+
+import os
+
+from ignitetest.utils.pause import ALL, parse_selector
+from ignitetest.utils.pause_control import continue_file
+
+from checks.support.demo_pause_control import new_demo_pause, resume_with
+
+
+def check_selector_parsing():
+ """
+ Check that every shape the demo_pause global can arrive in is understood:
-g passes it as
+ a string, -gj as whatever the json holds.
+ """
+ for disabled in (None, False, "", "false", "off", "0", [], " "):
+ assert parse_selector(disabled) is None, disabled
+
+ for every in (True, "*", "all", "true", "ON", "1"):
+ assert parse_selector(every) == ALL, every
+
+ assert parse_selector("split-brain") == {"split-brain"}
+ assert parse_selector("split-brain, healed ,") == {"split-brain", "healed"}
+ assert parse_selector(["split-brain", "healed"]) == {"split-brain",
"healed"}
+
+ # The global is typed by hand, the names live in the test source - the two
meet case insensitively.
+ assert parse_selector("Split-Brain, HEALED") == {"split-brain", "healed"}
+ assert parse_selector(["Split-Brain"]) == {"split-brain"}
+
+
+def check_names_are_matched_case_insensitively(tmp_path):
+ """
+ Check that a breakpoint is found however the global spells it.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause="Split-Brain")
+
+ resume_with(tmp_path, continue_file(1))
+
+ demo.pause("split-brain")
+
+ assert demo.seq == 1, "the global must not have to repeat the case of the
name in the test"
+
+
+def check_disabled_leaves_no_trace(tmp_path):
+ """
+ Check that without the global a breakpoint is a plain return: it must not
block, and it
+ must not even create the control directory, since every test carries
breakpoints in CI.
+ """
+ control_dir = tmp_path / "control"
+
+ demo = new_demo_pause(control_dir)
+
+ assert not demo.enabled
+
+ demo.pause("split-brain")
+
+ assert not os.path.exists(str(control_dir))
+ assert demo.seq == 0
+
+
+def check_selected_breakpoints_only(tmp_path):
+ """
+ Check that only the named breakpoints stop the scenario.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause="split-brain")
+
+ demo.pause("cluster-up")
+ demo.pause("healed")
+
+ assert demo.seq == 0, "an unnamed breakpoint must not stop the scenario"
+
+ resume_with(tmp_path, continue_file(1))
+
+ demo.pause("split-brain")
+
+ assert demo.seq == 1
diff --git a/modules/ducktests/tests/checks/utils/pause/check_timeout.py
b/modules/ducktests/tests/checks/utils/pause/check_timeout.py
new file mode 100644
index 00000000000..e47b922281f
--- /dev/null
+++ b/modules/ducktests/tests/checks/utils/pause/check_timeout.py
@@ -0,0 +1,72 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Checks how long a demo breakpoint may hold a scenario: the timeout the demo
asks for, and
+ducktape's --test-runner-timeout budget that only ever shortens it.
+"""
+
+import os
+
+from ignitetest.utils.pause import ALL, RUNNER_TIMEOUT_MARGIN_SEC
+from ignitetest.utils.pause_control import STATUS_JSON
+
+from checks.support.demo_pause_control import new_demo_pause, published_status
+
+
+def check_timeout_resumes_on_its_own(tmp_path):
+ """
+ Check that a forgotten breakpoint gives up rather than holding the
scenario until
+ ducktape kills it.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3)
+
+ demo.pause("split-brain")
+
+ assert demo.seq == 1
+ assert demo.enabled, "a timed out breakpoint must not disable the later
ones"
+ assert not os.path.exists(str(tmp_path / STATUS_JSON))
+
+
+def check_timeout_stays_within_the_runner_budget(tmp_path):
+ """
+ Check that a breakpoint gives up while ducktape's runner is still waiting:
it hears
+ nothing from a paused test, and killing the client takes the whole session
down instead
+ of just cutting the demo short. The requested timeout only ever shrinks.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause=ALL,
demo_pause_timeout_sec=3600,
+ runner_timeout_sec=RUNNER_TIMEOUT_MARGIN_SEC + .3)
+
+ demo.pause("split-brain")
+
+ assert demo.seq == 1
+ assert any("timed out" in msg for msg in demo.logger.messages), \
+ "the breakpoint must not outsit the runner budget it was given"
+ assert any("--test-runner-timeout" in msg for msg in
demo.logger.messages), \
+ "shortening a breakpoint must say what to raise to keep it"
+
+
+def check_timeout_is_left_alone_within_the_runner_budget(tmp_path):
+ """
+ Check that the budget only ever caps the requested timeout - a demo that
fits must be
+ held for exactly as long as it asked for.
+ """
+ demo = new_demo_pause(tmp_path, demo_pause=ALL, demo_pause_timeout_sec=.3,
runner_timeout_sec=1800)
+
+ with published_status(tmp_path) as published:
+ demo.pause("split-brain")
+
+ assert published["timeout_sec"] == .3
+ assert not any("--test-runner-timeout" in msg for msg in
demo.logger.messages)
diff --git a/modules/ducktests/tests/docker/demo_console.py
b/modules/ducktests/tests/docker/demo_console.py
new file mode 100644
index 00000000000..0decc8c5c5e
--- /dev/null
+++ b/modules/ducktests/tests/docker/demo_console.py
@@ -0,0 +1,195 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Host side of the demo breakpoints - run it in a second terminal, next to the
one running
+``run_tests.sh``, when a test is started with the ``demo_pause`` global:
+
+ ./docker/run_tests.sh -gj '{"demo_pause": "*"}' -t
./ignitetest/tests/<some_test.py>
+
+ python docker/demo_console.py
+
+Ducktape runs the test with stdin on /dev/null inside the ``ducker01``
container, so this is
+where the keyboard lives. The console itself is deliberately dumb: the test
renders the
+banner and this only prints it and writes back a resume file. Everything it
does can be done
+by hand instead - ``cat .ducktests-demo/paused.txt``, then ``touch
.ducktests-demo/continue-<N>``
+where ``<N>`` is the number shown in the banner's ``PAUSED N`` line.
+
+Standard library only: it runs on the host, outside the ducktests virtualenv.
+"""
+
+import argparse
+import importlib.util
+import os
+import sys
+import time
+
+# Speak the protocol through the framework's own ControlDir rather than
restating it here:
+# the two sides of a shared directory have to agree file for file, and a
second copy of it
+# is a second thing to keep in step.
+#
+# The host has no ducktape and no installed ignitetest, so the module is
loaded by path -
+# importing ignitetest.utils.pause_control would pull in the package __init__
chain and its
+# ducktape imports. pause_control itself is standard library only, which is
what makes it
+# loadable like this; ignitetest.utils.pause, which holds what the files mean,
is not.
+_TESTS_DIR =
os.path.abspath(os.path.join(os.path.dirname(os.path.abspath(__file__)),
os.pardir))
+_PAUSE_CONTROL_PY = os.path.join(_TESTS_DIR, "ignitetest", "utils",
"pause_control.py")
+
+_SPEC = importlib.util.spec_from_file_location("ignitetest_pause_control",
_PAUSE_CONTROL_PY)
+pause = importlib.util.module_from_spec(_SPEC)
+_SPEC.loader.exec_module(pause)
+
+POLL_SEC = .3
+
+KEYS = """
+ [Enter] continue [c] continue, skipping the rest [a] abort the
test
+ [q] leave the console (the test stays paused)
+"""
+
+
+def breakpoint_key(status):
+ """
+ :return: What identifies the published breakpoint. Not the sequence number
on its own:
+ that one is per test, so it restarts at 1 for every test of a
session, and a
+ run that died while paused leaves behind a banner numbered like a
live one.
+ """
+ return status.get("run"), status.get("seq")
+
+
+def clear_stale(control):
+ """
+ Removes resume files left behind by an earlier run, which would otherwise
skip the first
+ breakpoint of this one. The test clears them too, on its side, at its
first breakpoint.
+
+ Only ever done while nothing is published: a console is just as likely to
be started
+ against a test that is already holding a breakpoint, and a resume file
that was written
+ for that one - by hand, or by a console that has just been closed - is the
host's answer
+ to it rather than a leftover. The published breakpoint itself is left
alone either way,
+ a stale banner is told apart by its run id.
+
+ :return: Whether the sweep was performed.
+ """
+ if control.read_status() is not None:
+ return False
+
+ control.sweep()
+
+ return True
+
+
+def prompt(control, seq):
+ """
+ Asks what to do with the breakpoint that is currently published.
+
+ :return: False when the console should stop, True to wait for the next
breakpoint.
+ """
+ while True:
+ try:
+ answer = input(" > ").strip().lower()
+ except EOFError:
+ return False
+
+ if answer in ("", "n", "next"):
+ control.resume(pause.continue_file(seq))
+
+ return True
+
+ if answer in ("c", "continue", "all"):
+ control.resume(pause.CONTINUE_ALL)
+
+ print(" continuing, remaining breakpoints skipped")
+
+ return False
+
+ if answer in ("a", "abort"):
+ control.resume(pause.ABORT)
+
+ print(" aborting the test")
+
+ return False
+
+ if answer in ("q", "quit", "exit"):
+ print(f" leaving the test paused, resume it with:\n"
+ f" touch {control.file(pause.continue_file(seq))}")
+
+ return False
+
+ print(KEYS)
+
+
+def main():
+ """
+ Waits for breakpoints and drives them until the test is resumed for good.
+ """
+ parser = argparse.ArgumentParser(description="Drives the ducktests demo
breakpoints.")
+ parser.add_argument("-d", "--control-dir",
default=pause.default_control_dir(),
+ help="control directory shared with the test, defaults
to "
+ f"<repository root>/{pause.CONTROL_DIR_NAME}")
+
+ args = parser.parse_args()
+ control = pause.ControlDir(args.control_dir)
+
+ swept = clear_stale(control)
+
+ print(f"Demo console, watching {control.path}")
+
+ if swept:
+ print("Waiting for the first breakpoint... (Ctrl-C to leave)")
+ else:
+ print("A breakpoint is already held, joining it as it is (Ctrl-C to
leave)")
+
+ last_key, resumed_at = None, None
+
+ while True:
+ status = control.read_status()
+
+ if status is None:
+ # The test removes its status files as it resumes, so this is also
what tells the
+ # console that the breakpoint it has just driven is over and the
next one - which
+ # may well repeat its number, in the next test of the session - is
a new one.
+ last_key = None
+
+ time.sleep(POLL_SEC)
+
+ continue
+
+ if breakpoint_key(status) == last_key:
+ time.sleep(POLL_SEC)
+
+ continue
+
+ last_key = breakpoint_key(status)
+
+ print()
+
+ if resumed_at is not None:
+ print(f" ({time.monotonic() - resumed_at:.0f}s since the previous
breakpoint)")
+
+ print("\n".join(status.get("banner", [])))
+ print(KEYS)
+
+ if not prompt(control, status.get("seq")):
+ return
+
+ resumed_at = time.monotonic()
+
+ print(" resumed, waiting for the next breakpoint...")
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except KeyboardInterrupt:
+ sys.exit(130)
diff --git a/modules/ducktests/tests/docker/requirements-dev.txt
b/modules/ducktests/tests/docker/requirements-dev.txt
index f868e43d674..8ce0758663a 100644
--- a/modules/ducktests/tests/docker/requirements-dev.txt
+++ b/modules/ducktests/tests/docker/requirements-dev.txt
@@ -16,3 +16,4 @@
-r requirements.txt
pytest==6.2.5
flake8==6.1.0
+tox
diff --git a/modules/ducktests/tests/docker/run_tests.sh
b/modules/ducktests/tests/docker/run_tests.sh
index 511c106701a..b23ca856dcd 100755
--- a/modules/ducktests/tests/docker/run_tests.sh
+++ b/modules/ducktests/tests/docker/run_tests.sh
@@ -84,6 +84,10 @@ The options are as follows:
--image
Set custom docker image to run tests on.
+--test-runner-timeout
+ Milliseconds ducktape waits for a sign of life from a running test before
killing the
+ session, 1800000 by default.
+
EOF
exit 0
}
@@ -131,6 +135,7 @@ while [[ $# -ge 1 ]]; do
--subnet) SUBNET="--subnet $2"; shift 2;;
--jdk) JDK_VERSION="$2"; shift 2;;
--image) IMAGE_NAME="$2"; shift 2;;
+ --test-runner-timeout) TEST_RUNNER_TIMEOUT="$2"; shift 2;;
-f|--force) FORCE=$1; shift;;
*) break;;
esac
@@ -169,5 +174,9 @@ if [[ -n "$REPEAT" ]]; then
DUCKTAPE_OPTIONS="$DUCKTAPE_OPTIONS --repeat $REPEAT"
fi
+if [[ -n "$TEST_RUNNER_TIMEOUT" ]]; then
+ DUCKTAPE_OPTIONS="$DUCKTAPE_OPTIONS --test-runner-timeout
$TEST_RUNNER_TIMEOUT"
+fi
+
"$SCRIPT_DIR"/ducker-ignite test $TC_PATHS "$DUCKTAPE_OPTIONS" \
|| die "ducker-ignite test failed"
diff --git a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py
b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py
index d37b9c045fa..a9ccbe4496d 100644
--- a/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py
+++ b/modules/ducktests/tests/ignitetest/services/mdc/mdc_cluster.py
@@ -195,6 +195,36 @@ class MdcCluster:
return registry
+ def describe(self) -> List[str]:
+ """
+ Describes the cluster per data center for a demo breakpoint banner
+ (see :meth:`ignitetest.utils.ignite_test.IgniteTest.pause`). The
generic banner sees
+ a flat list of services, which is where the DC each node belongs to
gets lost.
+
+ Structure only - which node is up is what the banner's own service
section reports,
+ and it pays an SSH probe per node to find out.
+
+ :return: Section lines, the first one being the section title.
+ """
+ lines = ["DATA CENTERS"]
+
+ for dc in DCS:
+ roles = [(label, [node.account.hostname for svc in services for
node in svc.nodes])
+ for label, services in (("server", [self.servers[dc]] if
dc in self.servers else []),
+ ("runner", self.runners.get(dc,
[])),
+ ("loader", self.loaders.get(dc,
[])),
+ ("extra", self.extras.get(dc,
[])))]
+
+ # A DC that holds nothing is not named at all: an empty header
reads as a DC whose
+ # nodes have gone, which is exactly what a partition demo is being
watched for.
+ if not any(hosts for _, hosts in roles):
+ continue
+
+ lines.append(f" {dc}")
+ lines.extend(f" {label:<7} {' '.join(hosts)}" for label, hosts
in roles if hosts)
+
+ return lines
+
def thin_client_addresses(self) -> List[str]:
"""
:return: Thin client addresses of all server nodes across all DCs.
diff --git
a/modules/ducktests/tests/ignitetest/services/network_group/manager.py
b/modules/ducktests/tests/ignitetest/services/network_group/manager.py
index 3f90b7f9450..819212456bd 100644
--- a/modules/ducktests/tests/ignitetest/services/network_group/manager.py
+++ b/modules/ducktests/tests/ignitetest/services/network_group/manager.py
@@ -297,6 +297,29 @@ class NetworkGroupManager:
"""
self.logger.debug(f"Network State Overview: [START][{log_tag}]")
+ # The per-node SSH probes flood the debug log with their own command
output. Collect
+ # first, print contiguously after: the overview must stay readable as
one block.
+ for node_status in self._probe_network():
+ self.logger.debug(node_status)
+
+ self.logger.debug(f"Network State Overview: [END][{log_tag}]")
+
+ def describe(self):
+ """
+ Describes the live network state for a demo breakpoint banner
+ (see :meth:`ignitetest.utils.ignite_test.IgniteTest.pause`).
+
+ :return: Section lines, the first one being the section title.
+ """
+ return ["NETWORK"] + [f" {status}" for status in
self._probe_network()]
+
+ def _probe_network(self):
+ """
+ Probes the actual network state of every node - the applied netem
constraints, what
+ they are filtered onto, and the partition drops - rather than what was
asked for.
+
+ :return: One line per node, grouped by network group.
+ """
entries = [(group, svc, node)
for group, services in self.network_group_registry.items()
for svc in services
@@ -314,7 +337,7 @@ class NetworkGroupManager:
constraints = self._parse_qdisc_constraints(qdisc_lines)
targets_str = f" -> to [{', '.join(dst_ips)}]" if dst_ips and
constraints != "noqueue" else ""
- node_ip = socket.gethostbyname(node.account.externally_routable_ip)
+ node_ip = node.account.externally_routable_ip
partition_str = self._format_partition_drops(
self._parse_partition_drops(iptables_lines))
@@ -322,12 +345,7 @@ class NetworkGroupManager:
node_statuses.append(f"[{group:<4}]
{svc.who_am_i(node):<45}[{node_ip}] : "
f"{constraints}{targets_str}{partition_str}")
- # The per-node SSH probes above flood the debug log with their own
command output.
- # Collect first, print contiguously after: the overview must stay
readable as one block.
- for node_status in node_statuses:
- self.logger.debug(node_status)
-
- self.logger.debug(f"Network State Overview: [END][{log_tag}]")
+ return node_statuses
def _to_network_probe_cmd(self, node) -> str:
"""
diff --git
a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py
b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py
index 2ac39298055..e67d807caea 100644
--- a/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py
+++ b/modules/ducktests/tests/ignitetest/tests/mdc/partition_resilience_test.py
@@ -63,17 +63,23 @@ class MdcPartitionResilienceTest(IgniteTest):
with cross_dc_network(self.logger, mdc, delay_ms=cross_dc_latency_ms)
as net:
mdc.start_servers()
+ self.pause("cluster-up", mdc, net)
+
mdc.generate_data(DC_1, CACHE_NAME, 0, 100, backups=BACKUPS)
mdc.generate_data(DC_2, CACHE_NAME, 100, 200, backups=BACKUPS)
mdc.verify_cache_distribution(CACHE_NAME, copies_per_dc=1)
+ self.pause("data-loaded", mdc, net)
+
net.enable_network_partition(DC_1, DC_2)
sleep(SPLIT_SETTLE_SECS)
mdc.verify_split_brain()
+ self.pause("split-brain", mdc, net)
+
# All data written before the split is readable in both halves.
mdc.check_data(DC_1, CACHE_NAME, 0, 200)
mdc.check_data(DC_2, CACHE_NAME, 0, 200)
@@ -82,6 +88,8 @@ class MdcPartitionResilienceTest(IgniteTest):
mdc.check_put_admissibility(DC_1, CACHE_NAME, True)
mdc.check_put_admissibility(DC_2, CACHE_NAME, False)
+ self.pause("secondary-read-only", mdc, net)
+
net.disable_network_partition(DC_1, DC_2)
# Split-brain does not self-heal: the read-only half rejoins via
restart.
@@ -94,6 +102,8 @@ class MdcPartitionResilienceTest(IgniteTest):
mdc.control(DC_1).idle_verify(CACHE_NAME)
+ self.pause("healed", mdc, net)
+
mdc.verify_servers_log_clean()
mdc.stop_servers()
diff --git a/modules/ducktests/tests/ignitetest/utils/ignite_test.py
b/modules/ducktests/tests/ignitetest/utils/ignite_test.py
index 1feda7859ea..d2a71c50e83 100644
--- a/modules/ducktests/tests/ignitetest/utils/ignite_test.py
+++ b/modules/ducktests/tests/ignitetest/utils/ignite_test.py
@@ -23,6 +23,7 @@ from ducktape.cluster.remoteaccount import RemoteCommandError
from ducktape.tests.test import Test, TestContext
from ignitetest.services.utils.ducktests_service import DucktestsService
+from ignitetest.utils.pause import DemoPause
# globals:
JFR_ENABLED = "jfr_enabled"
@@ -66,6 +67,50 @@ class IgniteTest(Test):
super().__init__(test_context=test_context)
+ self.__demo_pause = None
+
+ # Stamped here rather than at the first breakpoint: it is what demo
breakpoints count
+ # their elapsed time from, and what they measure the runner budget
against, and both
+ # of those mean the start of the test - setup included. This is
deliberately
+ # conservative: ducktape's kill timer is actually reset by the last
client event
+ # before a pause, which fires after setup, so the real window is
longer - but
+ # anchoring at construction needs no hook into the runner client and
never
+ # overestimates the budget.
+ self.__started_at = monotonic()
+
+ def pause(self, name, *describers):
+ """
+ Holds the scenario at a named demo breakpoint until it is resumed from
the host, so
+ that the cluster can be shown in exactly this state. Does nothing at
all unless the
+ `demo_pause` global names this breakpoint - see
:mod:`ignitetest.utils.pause`.
+
+ Must be called from the test body: :meth:`tearDown` kills every
service, so a
+ breakpoint placed after the body would only ever show a dead cluster.
+
+ :param name: Breakpoint name, matched against the `demo_pause` global.
+ :param describers: Objects exposing `describe() -> list of str`, each
contributing a
+ section to the banner shown while paused, on top of the service
list every
+ breakpoint reports. Any fixture a test drives can implement it.
+ """
+ if self.__demo_pause is None:
+ self.__demo_pause = DemoPause(self.logger,
self.test_context.globals, self.test_context.test_name,
+ started_at=self.__started_at,
+
runner_timeout_sec=self.__runner_timeout_sec())
+
+ self.__demo_pause.pause(name, describers, self.test_context.services)
+
+ def __runner_timeout_sec(self):
+ """
+ :return: Ducktape's `--test-runner-timeout` in seconds, None when the
session carries
+ none. A breakpoint has to give up inside it, since the runner
kills a test
+ client it hears nothing from for that long - see
+ :meth:`ignitetest.utils.pause.DemoPause._budgeted_timeout`.
+ """
+ session_context = getattr(self.test_context, "session_context", None)
+ timeout_ms = getattr(session_context, "test_runner_timeout", None)
+
+ return timeout_ms / 1000 if timeout_ms else None
+
@property
def available_cluster_size(self):
# noinspection PyUnresolvedReferences
diff --git a/modules/ducktests/tests/ignitetest/utils/pause.py
b/modules/ducktests/tests/ignitetest/utils/pause.py
new file mode 100644
index 00000000000..82e1096097e
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/utils/pause.py
@@ -0,0 +1,541 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Demo breakpoints: freezing a scenario at a named point so a live audience can
be shown the
+cluster in that exact state.
+
+Ducktape runs the test inside the ``ducker01`` container with stdin on
/dev/null, so a test
+cannot read a keypress. What it can do is share files with the host:
``ducker-ignite`` bind
+mounts the whole Ignite repository into every container, so a control
directory below the
+repository root is visible to the test and to the host at the same time.
+
+The directory itself, and the file protocol over it, are
+:class:`ignitetest.utils.pause_control.ControlDir`. This module holds only
what those files
+*mean*: which breakpoint stops the scenario, what the banner says, and that
``abort`` ends
+the test.
+
+Between breakpoints both sides sweep the directory for files an earlier run
left behind,
+which would otherwise skip the next breakpoint: the test at its first one (see
+:meth:`DemoPause._prepare`), the console at startup - and only while nothing
is published, so
+a resume file meant for a breakpoint that is currently held is never swept.
+
+``docker/demo_console.py`` is the host side of it, but nothing depends on it:
reading
+``paused.txt`` and touching ``continue-<seq>`` by hand works just as well,
where ``<seq>`` is
+the breakpoint number shown in the banner's ``PAUSED <seq>`` line.
+
+Globals:
+
+ demo_pause - absent or false disables every breakpoint (the default, so
tests are
+ unaffected in CI); true or "*" stops at all of them; a list or a comma
separated string
+ stops only at the named ones, matched case insensitively.
+
+ demo_pause_timeout_sec - how long a single breakpoint may hold the
scenario before it
+ resumes on its own, 600 by default. Capped by what is left of ducktape's
+ ``--test-runner-timeout``, see :meth:`DemoPause._budgeted_timeout`.
+
+ demo_pause_dir - control directory, ``<repository root>/.ducktests-demo``
by default.
+"""
+
+import os
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+from ignitetest.services.utils.path import IgnitePathAware
+from ignitetest.utils.pause_control import ABORT, CONTINUE_ALL, ControlDir,
continue_file, default_control_dir, \
+ repo_root
+
+# globals:
+DEMO_PAUSE = "demo_pause"
+DEMO_PAUSE_TIMEOUT_SEC = "demo_pause_timeout_sec"
+DEMO_PAUSE_DIR = "demo_pause_dir"
+
+# Well below ducktape's own --test-runner-timeout (1800s), which a breakpoint
must not
+# outsit - see _budgeted_timeout().
+DEFAULT_TIMEOUT_SEC = 600
+
+# Kept free of the runner budget, so that resuming a breakpoint at the very
last moment
+# still leaves the scenario time to reach its next event.
+RUNNER_TIMEOUT_MARGIN_SEC = 60
+
+# Same limit as the network group manager: a breakpoint renders its banner
while the cluster
+# is held, so the SSH probes that report liveness are parallelised to keep
that render fast.
+_MAX_PARALLEL_SSH = 16
+
+# Timestamps the demo in the test log, so that a run can be read back
afterwards and it is
+# visible that the scenario is held rather than stuck. Deliberately NOT a
keepalive towards
+# ducktape: the test logger writes to files and to stdout only, while the
runner listens for
+# zmq events that just the runner client itself emits - _budgeted_timeout() is
what keeps a
+# paused test within the runner's patience.
+HEARTBEAT_SEC = 15
+
+# Every breakpoint name matches.
+ALL = "*"
+
+_WIDTH = 100
+
+
+def parse_selector(value):
+ """
+ Interprets the ``demo_pause`` global.
+
+ Names are matched case insensitively, both here and in
:meth:`DemoPause._stops_at`: the
+ global is typed by hand next to a test whose breakpoint names are written
in the source.
+
+ :return: None when demo pausing is disabled, :data:`ALL` to stop at every
breakpoint,
+ or the set of breakpoint names to stop at, lower cased.
+ """
+ if value is None or value is False:
+ return None
+
+ if value is True:
+ return ALL
+
+ if isinstance(value, (list, tuple, set, frozenset)):
+ names = {str(name).strip().lower() for name in value}
+ names.discard("")
+
+ return names or None
+
+ if isinstance(value, str):
+ text = value.strip().lower()
+
+ if text in ("", "false", "no", "off", "0"):
+ return None
+
+ if text in (ALL, "all", "true", "yes", "on", "1"):
+ return ALL
+
+ names = {name.strip() for name in text.split(",")}
+ names.discard("")
+
+ return names or None
+
+ return ALL if value else None
+
+
+def _fmt_duration(seconds):
+ """
+ :return: Duration as mm:ss, or hh:mm:ss once it no longer fits.
+ """
+ seconds = max(int(seconds), 0)
+
+ if seconds >= 3600:
+ return f"{seconds // 3600}:{seconds // 60 % 60:02d}:{seconds % 60:02d}"
+
+ return f"{seconds // 60:02d}:{seconds % 60:02d}"
+
+
+def _resume_command(control, seq):
+ """
+ :return: The command that resumes a breakpoint by hand, named where it
will be typed.
+
+ Relative to the repository root rather than absolute: the banner is
rendered inside
+ ``ducker01``, where the repository is mounted at ``/opt/ignite-dev``, and
read on the host,
+ where that path does not exist - so an absolute one would be a command
that cannot work
+ for the person it is offered to. Both sides see the same repository
through the bind
+ mount, which makes the relative form the one string that holds for both,
and it is the
+ form README documents.
+ """
+ path = control.file(continue_file(seq))
+
+ try:
+ relative = os.path.relpath(path, repo_root())
+ except ValueError:
+ # A control directory on another Windows drive than the repository:
nothing shared to
+ # be relative to, so the full path is all there is to offer.
+ relative = path
+
+ if relative.startswith(os.pardir) or relative == path:
+ return f"touch {path}"
+
+ return f"touch {relative.replace(os.sep, '/')} (from the repository
root)"
+
+
+def _node_host(node):
+ """
+ :return: Hostname of the node, None when it carries no account to read one
from.
+ """
+ return getattr(getattr(node, "account", None), "hostname", None)
+
+
+def _node_addr(node):
+ """
+ :return: The node's routable address when it adds anything to the name the
banner already
+ carries - under ducker the two are the same string.
+
+ Deliberately not resolved to an IP: a name that does not resolve costs a
DNS round trip
+ per node, and a breakpoint that takes seconds to print its banner defeats
the point.
+ """
+ addr = node.account.externally_routable_ip
+
+ return "" if addr == node.account.hostname else f"[{addr}]"
+
+
+def _node_state(service, node):
+ """
+ :return: Liveness of the node as far as its service can tell, "?" when the
probe itself
+ failed - a breakpoint must never fail the scenario it is only
observing.
+ """
+ alive = getattr(service, "alive", None)
+
+ if alive is None:
+ return ""
+
+ try:
+ return "UP" if alive(node) else "DOWN"
+ except Exception: # pylint: disable=broad-except
+ return "?"
+
+
+def _node_line(service, node, state):
+ """
+ :return: The node's line of the SERVICES section, degraded to the name
that can be read
+ without asking the service when the service itself cannot answer
for the node -
+ ``who_am_i`` goes through ``idx()``, which raises for a node the
service no
+ longer owns.
+
+ The liveness state is probed in parallel by :func:`_probe_liveness` and
passed in, so
+ that rendering the banner does not wait on one SSH round-trip per node in
sequence.
+
+ Like :func:`_node_state`, this lets no reading failure out: a breakpoint
must never fail
+ the scenario it is only observing, least of all while rendering the banner
it was added
+ for.
+ """
+ try:
+ return f" {service.who_am_i(node):<58} {_node_addr(node):<17}
{state}".rstrip()
+ except Exception as ex: # pylint: disable=broad-except
+ name = f"{type(service).__name__}-{_node_host(node) or '?'}"
+
+ return f" {name:<58} ({type(ex).__name__})"
+
+
+def _probe_liveness(services):
+ """
+ Probes the liveness of every node across all services concurrently,
returning a list of
+ state strings in the same order as the ``(service, node)`` pairs the
caller iterates.
+
+ Each ``alive(node)`` call is an SSH round-trip; on a multi-DC cluster
there are enough of
+ them that probing in sequence would delay the banner by several seconds.
The network
+ group manager already parallelises its probes the same way.
+ """
+ pairs = [(service, node) for service in services for node in service.nodes]
+
+ if not pairs:
+ return []
+
+ with ThreadPoolExecutor(max_workers=min(_MAX_PARALLEL_SSH, len(pairs))) as
pool:
+ return list(pool.map(lambda sn: _node_state(sn[0], sn[1]), pairs))
+
+
+class DemoPause:
+ """
+ Holds a scenario at named breakpoints.
+
+ Disabled unless the ``demo_pause`` global says otherwise, in which case
:meth:`pause` is
+ a plain return and nothing is written anywhere.
+ """
+ def __init__(self, logger, test_globals, test_name, control_dir=None,
started_at=None,
+ runner_timeout_sec=None):
+ """
+ :param started_at: Monotonic timestamp the test itself started at,
which is both what
+ the banner counts from and what the runner budget is spent
from. Defaults to
+ now, which is only right when the first breakpoint is the start
of the test.
+ Anchored at construction (setup included), which is
deliberately conservative:
+ ducktape's kill timer is actually reset by the last client
event before the
+ pause, which fires after setup, so the real window is longer.
+ :param runner_timeout_sec: Ducktape's ``--test-runner-timeout`` in
seconds, None when
+ unknown, in which case no breakpoint is cut short by it.
+ """
+ self.logger = logger
+ self.test_name = test_name
+
+ self.names = parse_selector(test_globals.get(DEMO_PAUSE))
+ self.timeout_sec = float(test_globals.get(DEMO_PAUSE_TIMEOUT_SEC,
DEFAULT_TIMEOUT_SEC))
+ self.runner_timeout_sec = runner_timeout_sec
+
+ self.control = ControlDir(control_dir or
test_globals.get(DEMO_PAUSE_DIR) or default_control_dir())
+
+ self.seq = 0
+
+ # Identifies this run of this test to the host console, which has no
other way of
+ # telling a breakpoint of the current run from one left published by a
run that
+ # died while paused: seq alone restarts at 1 for every test.
+ self.run = f"{os.getpid()}-{int(time.time())}"
+
+ self._started_at = time.monotonic() if started_at is None else
started_at
+ self._prepared = False
+ self._continue_all = False
+ self._unusable = False
+
+ @property
+ def enabled(self):
+ """
+ :return: Whether any breakpoint of this test can stop the scenario.
+ """
+ return self.names is not None and not self._continue_all and not
self._unusable
+
+ def pause(self, name, describers=(), services=()):
+ """
+ Blocks the scenario at the named breakpoint until the host resumes it.
+
+ A control directory that cannot be used costs the demo and nothing
else - see
+ :meth:`_give_up`.
+
+ :param name: Breakpoint name, matched against the ``demo_pause``
global.
+ :param describers: Objects exposing ``describe() -> list of str``,
each contributing
+ a section to the banner. The first line of a section is its
title.
+ :param services: Services to list in the banner, normally the test's
whole registry.
+ """
+ if not self._stops_at(name):
+ return
+
+ try:
+ self._prepare()
+
+ self.seq += 1
+
+ timeout_sec = self._budgeted_timeout()
+
+ banner = self._render(name, describers, services, timeout_sec)
+
+ self._publish(name, banner, timeout_sec)
+ except OSError as ex:
+ self._give_up(name, ex)
+
+ return
+
+ self.logger.info(f"Demo breakpoint reached [seq={self.seq},
name={name}, dir={self.control.path}]")
+
+ self._await_resume(name, timeout_sec)
+
+ def _give_up(self, name, error):
+ """
+ Turns the remaining breakpoints off, after the control directory
turned out to be
+ unusable.
+
+ A breakpoint observes a scenario; it must not be what ends one. The
directory is a
+ bind mount of the host repository, so it can be read only, be owned by
another user or
+ be full - none of which says anything about the cluster under test,
and all of which
+ would otherwise fail a run that was about to pass. Blocking would be
no better than
+ raising: a breakpoint whose banner never reached the host would hold
the scenario for
+ its whole timeout with nothing on screen to resume it.
+
+ Every later breakpoint would fail the same way, so they are dropped
here rather than
+ reported again at each one.
+ """
+ self._unusable = True
+
+ self.logger.warn(f"Demo breakpoints disabled, the control directory
cannot be used "
+ f"[dir={self.control.path}, error={error},
seq={self.seq}, name={name}, "
+ f"test={self.test_name}]")
+
+ def _stops_at(self, name):
+ if not self.enabled:
+ return False
+
+ return self.names == ALL or name.strip().lower() in self.names
+
+ def _budgeted_timeout(self):
+ """
+ :return: How long this breakpoint may actually hold the scenario.
+
+ ``demo_pause_timeout_sec`` is what the demo asks for, the runner
budget is what it is
+ allowed. Ducktape's runner kills a test client it has received no
event from for
+ ``--test-runner-timeout`` and takes the whole session down with it,
and a paused test
+ sends no events - so a breakpoint has to give up while the runner is
still waiting.
+ The budget is spent from the construction of the test (setup included)
rather than
+ from the breakpoint, hence a long setup, or a long earlier pause,
leaves less of it
+ for this one. This is deliberately conservative: ducktape resets its
kill timer on
+ every client event, the last one before a pause being ``"Running..."``
(after setup),
+ so the real window is longer by however long setup took - but
measuring from
+ construction needs no hook into the runner client and never
overestimates the budget.
+ """
+ if self.runner_timeout_sec is None:
+ return self.timeout_sec
+
+ left = self.runner_timeout_sec - (time.monotonic() - self._started_at)
- RUNNER_TIMEOUT_MARGIN_SEC
+
+ if left >= self.timeout_sec:
+ return self.timeout_sec
+
+ self.logger.warn(f"Demo breakpoint held for at most
{_fmt_duration(max(left, 0))} instead of the requested "
+ f"{_fmt_duration(self.timeout_sec)}: what is left of
ducktape's --test-runner-timeout "
+ f"({_fmt_duration(self.runner_timeout_sec)}) after
{_fmt_duration(self.elapsed_sec)} of "
+ f"this test. Raise --test-runner-timeout for a longer
demo "
+ f"[seq={self.seq}, test={self.test_name}]")
+
+ return max(left, 0.0)
+
+ @property
+ def elapsed_sec(self):
+ """
+ :return: Seconds since the test started.
+ """
+ return time.monotonic() - self._started_at
+
+ def _prepare(self):
+ """
+ Readies the control directory, once per test: a resume file left by a
previous run
+ would skip the very first breakpoint of this one.
+ """
+ if self._prepared:
+ return
+
+ self.control.prepare()
+
+ self._prepared = True
+
+ def _render(self, name, describers, services, timeout_sec):
+ """
+ :return: The banner as a list of lines.
+ """
+ elapsed = f" t+{_fmt_duration(self.elapsed_sec)} since test start"
+ auto = f"auto-continue in {_fmt_duration(timeout_sec)} "
+
+ lines = [
+ "=" * _WIDTH,
+ f" PAUSED {self.seq} {name}",
+ f" test {self.test_name}",
+ (elapsed + auto.rjust(max(_WIDTH - len(elapsed), 1))).rstrip(),
+ ]
+
+ for section in [self._services_section(services)] + [self._section(d)
for d in describers]:
+ if section:
+ lines.append("-" * _WIDTH)
+ lines.extend(section)
+
+ lines.append("-" * _WIDTH)
+ lines.extend(self._hints_section(services))
+
+ lines.append("-" * _WIDTH)
+ lines.append(" continue: [Enter] in the demo console")
+ lines.append(f" or {_resume_command(self.control,
self.seq)}")
+ lines.append("=" * _WIDTH)
+
+ return lines
+
+ def _section(self, describer):
+ try:
+ return list(describer.describe())
+ except Exception as ex: # pylint: disable=broad-except
+ self.logger.warn(f"Demo breakpoint describer failed
[describer={describer}, error={ex}]")
+
+ return []
+
+ @staticmethod
+ def _services_section(services):
+ lines = ["SERVICES"]
+
+ pairs = [(service, node) for service in services for node in
service.nodes]
+ states = _probe_liveness(services)
+
+ for (service, node), state in zip(pairs, states):
+ lines.append(_node_line(service, node, state))
+
+ if len(lines) == 1:
+ lines.append(" (none)")
+
+ return lines
+
+ @staticmethod
+ def _hints_section(services):
+ """
+ Node logs live only on the nodes while the test runs - ducktape copies
them into the
+ results directory at teardown - so every hint goes through the node
container.
+ """
+ # A node the banner could not name is left out rather than allowed to
fail the hints:
+ # the commands are offered for the nodes that can be entered, and one
unreadable node
+ # must not cost the demo the rest of them.
+ hosts = sorted({_node_host(node) for service in services for node in
service.nodes} - {None})
+
+ log_dir = "/mnt/service/logs"
+ config_file = "/mnt/service/config/ignite-config.xml"
+
+ # One set of copy-pasteable commands for a service list that is not
homogeneous, so
+ # they follow the Ignite services: a ZookeeperService or a
KafkaService registered
+ # ahead of them - which the discovery and CDC scenarios do - carries
paths of its own
+ # and would have the banner name a zookeeper.properties for nodes that
never had one.
+ for service in [s for s in services if isinstance(s, IgnitePathAware)]
or list(services):
+ try:
+ svc_log_dir = getattr(service, "log_dir", None)
+ svc_config_file = getattr(service, "config_file", None)
+ except Exception: # pylint: disable=broad-except
+ continue
+
+ log_dir = svc_log_dir or log_dir
+ config_file = svc_config_file or config_file
+
+ break
+
+ # Node paths, joined by hand: they are always POSIX, os.path.join is
not when the
+ # control machine happens to be Windows.
+ ignite_log = f"{log_dir.rstrip('/')}/ignite*.log"
+ console_log = f"{log_dir.rstrip('/')}/console.log"
+
+ return [
+ f" nodes {' '.join(hosts) if hosts else '(none)'}",
+ " shell ./docker/ducker-ignite ssh <node>",
+ f" logs docker exec <node> bash -c \"tail -n 50
{ignite_log}\"",
+ f" console docker exec <node> tail -n 50 {console_log}",
+ f" config docker exec <node> cat {config_file}",
+ ]
+
+ def _publish(self, name, banner, timeout_sec):
+ """
+ Publishes the breakpoint for the host: the banner to print, plus what
a reader needs
+ to tell this pause from any other.
+ """
+ self.control.publish(banner, {
+ "run": self.run,
+ "seq": self.seq,
+ "name": name,
+ "test": self.test_name,
+ "elapsed_sec": round(self.elapsed_sec, 1),
+ "timeout_sec": timeout_sec,
+ "banner": banner
+ })
+
+ def _await_resume(self, name, timeout_sec):
+ """
+ Holds the scenario until the host resumes the breakpoint, or until it
gives up on its
+ own.
+
+ What each resume file means lives here rather than in the control
directory: it is the
+ only part of the protocol that knows there is a scenario to end.
+ """
+ def still_waiting(left_sec):
+ self.logger.info(f"Still paused at demo breakpoint
[seq={self.seq}, name={name}, "
+ f"held={_fmt_duration(timeout_sec - left_sec)}, "
+ f"left={_fmt_duration(left_sec)}]")
+
+ taken = self.control.await_any([ABORT, CONTINUE_ALL,
continue_file(self.seq)], timeout_sec,
+ tick=still_waiting,
tick_sec=HEARTBEAT_SEC)
+
+ # Whatever ended the wait, the banner describes a breakpoint that is
over.
+ self.control.clear_status()
+
+ if taken == ABORT:
+ raise AssertionError(f"Demo aborted at breakpoint [seq={self.seq},
name={name}]")
+
+ if taken == CONTINUE_ALL:
+ self._continue_all = True
+
+ self.logger.info(f"Demo resumed, remaining breakpoints skipped
[seq={self.seq}, name={name}]")
+ elif taken is None:
+ self.logger.warn(f"Demo breakpoint timed out after {timeout_sec}s,
resuming "
+ f"[seq={self.seq}, name={name}]")
+ else:
+ self.logger.info(f"Demo resumed [seq={self.seq}, name={name}]")
diff --git a/modules/ducktests/tests/ignitetest/utils/pause_control.py
b/modules/ducktests/tests/ignitetest/utils/pause_control.py
new file mode 100644
index 00000000000..9a5cf96c0f2
--- /dev/null
+++ b/modules/ducktests/tests/ignitetest/utils/pause_control.py
@@ -0,0 +1,250 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+The directory a paused test and the host talk over, and the file protocol they
talk in.
+
+The protocol is one sided while a breakpoint is held - the host only ever
creates files, the
+test is the only party that deletes them - so no step of a held breakpoint can
race with the
+host:
+
+ - the test publishes ``paused.txt`` (a rendered banner) and
``paused.json`` (the same
+ content as data) and then blocks;
+ - the host creates ``continue-<seq>``, ``continue-all`` or ``abort``;
+ - the test consumes that file, removes it along with its own status files,
and proceeds.
+
+Both sides drive it through :class:`ControlDir`, which owns the mechanics
alone - paths,
+atomic writes, sweeping, polling. What a file *means* is the caller's business:
+:mod:`ignitetest.utils.pause` decides that ``abort`` fails the scenario, and
+``docker/demo_console.py`` decides which one to write.
+
+Standard library only, and deliberately free of every ignitetest import: the
console loads
+this module by path, on a host that has neither ducktape nor ignitetest
installed.
+"""
+
+import json
+import os
+import time
+
+CONTROL_DIR_NAME = ".ducktests-demo"
+
+STATUS_TXT = "paused.txt"
+STATUS_JSON = "paused.json"
+CONTINUE_PREFIX = "continue-"
+CONTINUE_ALL = "continue-all"
+ABORT = "abort"
+
+# The control directory is polled rather than watched: it is a bind mount
shared with the
+# host, where inotify is not dependable.
+POLL_SEC = .5
+
+
+def repo_root():
+ """
+ :return: Path of the Ignite repository root, derived from this module's
own location
+
(``<root>/modules/ducktests/tests/ignitetest/utils/pause_control.py``), so that
a
+ fork checked out elsewhere resolves its own root.
+ """
+ return os.path.abspath(os.path.join(os.path.dirname(__file__),
*[os.pardir] * 5))
+
+
+def default_control_dir():
+ """
+ :return: Path of the control directory shared between the test and the
host.
+ """
+ return os.path.join(repo_root(), CONTROL_DIR_NAME)
+
+
+def continue_file(seq):
+ """
+ :return: Name of the file that resumes the breakpoint with the given
sequence number.
+ """
+ return f"{CONTINUE_PREFIX}{seq}"
+
+
+class ControlDir:
+ """
+ One directory, shared between a paused test and the host driving it.
+
+ Every method here is mechanics. Nothing in this class knows what a
breakpoint is, which
+ is what lets the test, the console and the checks all speak the protocol
through the same
+ code rather than through three copies of it that have to agree.
+ """
+ def __init__(self, path):
+ self._path = str(path)
+
+ @property
+ def path(self):
+ """
+ :return: Path of the directory itself, as both sides name it.
+ """
+ return self._path
+
+ def file(self, name):
+ """
+ :return: Path of a control file, for reporting it to someone who will
type it.
+ """
+ return os.path.join(self._path, name)
+
+ def exists(self, name):
+ """
+ :return: Whether the control file is there.
+ """
+ return os.path.exists(self.file(name))
+
+ def consume(self, name):
+ """
+ Removes a control file if it is present, returning whether it was
there.
+
+ The test is the only party that deletes what the host writes, so
consuming
+ a file is what acknowledges the host's command.
+
+ :return: True if the file existed and was removed, False otherwise.
+ """
+ try:
+ os.remove(self.file(name))
+ except OSError:
+ return False
+
+ return True
+
+ def remove(self, name):
+ """
+ Removes a control file, if it is still there - the other side may have
just swept it.
+ """
+ try:
+ os.remove(self.file(name))
+ except OSError:
+ pass
+
+ def write(self, name, content):
+ """
+ Writes a control file whole: it lands through a temporary name, so the
other side
+ never reads a half written one.
+ """
+ path = self.file(name)
+ tmp = path + ".tmp"
+
+ with open(tmp, "w", encoding="utf-8") as file:
+ file.write(content)
+
+ os.replace(tmp, path)
+
+ def prepare(self):
+ """
+ Creates the directory and clears anything an earlier run left in it: a
stale resume
+ file would skip the very first breakpoint of this one.
+ """
+ os.makedirs(self._path, exist_ok=True)
+
+ self.sweep(status=True)
+
+ def sweep(self, status=False):
+ """
+ Removes what an earlier run left behind.
+
+ :param status: Whether to drop a published banner as well. The test
does, at its first
+ breakpoint. The console must not: it is just as likely to have
been started
+ against a test that is already holding one.
+ """
+ if not os.path.isdir(self._path):
+ return
+
+ # A write lands through the temporary ``<name>.tmp``: an interrupted
one leaves it
+ # behind, and this sweep is the only thing that ever visits the
directory without
+ # knowing what it expects to find, so it strips the suffix before the
match. Only
+ # ``continue-`` stays a prefix, for its open ended sequence number;
every other name
+ # matches exactly, so a file the protocol could never create is left
alone.
+ for name in os.listdir(self._path):
+ base = name[:-len(".tmp")] if name.endswith(".tmp") else name
+
+ stale = base.startswith(CONTINUE_PREFIX) or base == ABORT
+
+ if status:
+ stale = stale or base == STATUS_TXT or base == STATUS_JSON
+
+ if stale:
+ self.remove(name)
+
+ def publish(self, banner, payload):
+ """
+ Publishes a held breakpoint, banner first as text and then as data, so
that whoever
+ polls for the data never finds it ahead of the text it describes.
+ """
+ self.write(STATUS_TXT, "\n".join(banner) + "\n")
+ self.write(STATUS_JSON, json.dumps(payload, indent=2))
+
+ def read_status(self):
+ """
+ :return: The published breakpoint, or None when nothing is published.
A missing or
+ half written file simply reads as "not paused" and is retried
by the caller.
+ """
+ try:
+ with open(self.file(STATUS_JSON), encoding="utf-8") as file:
+ return json.load(file)
+ except (OSError, ValueError):
+ return None
+
+ def clear_status(self):
+ """
+ Withdraws the published breakpoint, so a stale banner never outlives
the pause it
+ describes.
+ """
+ for name in (STATUS_TXT, STATUS_JSON):
+ self.remove(name)
+
+ def resume(self, name):
+ """
+ Writes a resume file. The test consumes and removes it.
+
+ Created outright rather than through :meth:`write`: a resume file is
empty, so there
+ is no half written state for an atomic replace to hide, and the
temporary name that
+ replace goes through would be one more transient to leave lying about.
+ """
+ with open(self.file(name), "w", encoding="utf-8"):
+ pass
+
+ def await_any(self, names, timeout_sec, tick=None, tick_sec=POLL_SEC):
+ """
+ Polls until one of the named files appears, and consumes it.
+
+ :param names: Names to watch, in priority order - the first one
present wins when
+ several land between two polls.
+ :param tick: Called with the seconds left, every ``tick_sec`` that
passes without a
+ file. This is how a caller reports that it is still waiting
without this class
+ having to know what it would report to.
+ :param tick_sec: How often to do that, by default as often as the
directory is looked
+ at - which is as often as there is anything new to say.
+ :return: The name that ended the wait, None when the timeout ran out
first.
+ """
+ deadline = time.monotonic() + timeout_sec
+ next_tick = time.monotonic() + tick_sec if tick else None
+
+ while True:
+ for name in names:
+ if self.consume(name):
+ return name
+
+ now = time.monotonic()
+
+ if now >= deadline:
+ return None
+
+ if next_tick is not None and now >= next_tick:
+ next_tick = now + tick_sec
+
+ tick(deadline - now)
+
+ time.sleep(POLL_SEC)
diff --git a/modules/ducktests/tests/setup.py b/modules/ducktests/tests/setup.py
index 06d87990776..9065ccc9c98 100644
--- a/modules/ducktests/tests/setup.py
+++ b/modules/ducktests/tests/setup.py
@@ -28,7 +28,7 @@ setup(name="ignitetest",
author="Apache Ignite",
platforms=["any"],
license="apache2.0",
- packages=find_packages(exclude=["ignitetest.tests",
"ignitetest.tests.*"]),
+ packages=find_packages(exclude=["ignitetest.tests",
"ignitetest.tests.*", "checks", "checks.*"]),
include_package_data=True,
install_requires=open('docker/requirements.txt').read(),
tests_require=["pytest==6.2.5"])
diff --git a/modules/ducktests/tests/tox.ini b/modules/ducktests/tests/tox.ini
index e0fa4a74df8..c40e563eb26 100644
--- a/modules/ducktests/tests/tox.ini
+++ b/modules/ducktests/tests/tox.ini
@@ -21,6 +21,12 @@ usedevelop = True
envdir = {toxworkdir}/.virtualenvs/ignite-ducktests-{envname}
deps = -r{toxinidir}/docker/requirements-dev.txt
install_command = pip install --extra-index-url https://pypi.org/simple {opts}
{packages}
+# pytest builds its tmp_path root below a per user directory, and
getpass.getuser() has no
+# password database to fall back on when the name is missing from the
environment - which is
+# what a Windows control machine runs into, since tox passes neither variable
by default.
+passenv =
+ USER
+ USERNAME
commands = pytest {env:PYTESTARGS:} {posargs}
[testenv:codestyle]
