When introducing pseudo-random generation in the test runs we need to
ensure that these can be reproduced by setting a pre-defined seed.
This commits adds the ability to set one or allow for one to be
generated and reported back to the user.
Signed-off-by: Luca Vizzarro <luca.vizza...@arm.com>
Reviewed-by: Paul Szczepanek <paul.szczepa...@arm.com>
Reviewed-by: Alex Chapman <alex.chap...@arm.com>
---
doc/guides/tools/dts.rst | 5 +++++
dts/framework/config/__init__.py | 4 ++++
dts/framework/config/conf_yaml_schema.json | 4 ++++
dts/framework/config/types.py | 2 ++
dts/framework/runner.py | 8 ++++++++
dts/framework/settings.py | 17 +++++++++++++++++
6 files changed, 40 insertions(+)
diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
index 515b15e4d8..9b5ea9779c 100644
--- a/doc/guides/tools/dts.rst
+++ b/doc/guides/tools/dts.rst
@@ -251,6 +251,8 @@ DTS is run with ``main.py`` located in the ``dts``
directory after entering Poet
... | DTS_TEST_SUITES='suite, suite case, ...'
(default: [])
--re-run N_TIMES, --re_run N_TIMES
[DTS_RERUN] Re-run each test case the specified
number of times if a test failure occurs. (default: 0)
+ --random-seed NUMBER [DTS_RANDOM_SEED] The seed to use with the
pseudo-random generator. If not specified, the configuration value is
+ used instead. If that's also not specified, a
random seed is generated. (default: None)
The brackets contain the names of environment variables that set the same thing.
@@ -548,6 +550,9 @@ involved in the testing. These can be defined with the
following mappings:
+----------------------------+---------------+---------------------------------------------------+
| ``traffic_generator_node`` | Node name for the traffic generator node.
|
+----------------------------+-------------------------------------------------------------------+
+ | ``random_seed`` | (*optional*) *int* – Allows you to set a
seed for pseudo-random |
+ | | generation.
|
+
+----------------------------+-------------------------------------------------------------------+
``nodes``
`sequence
<https://docs.python.org/3/library/stdtypes.html#sequence-types-list-tuple-range>`_
listing
diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
index df60a5030e..269d9ec318 100644
--- a/dts/framework/config/__init__.py
+++ b/dts/framework/config/__init__.py
@@ -445,6 +445,7 @@ class TestRunConfiguration:
system_under_test_node: The SUT node to use in this test run.
traffic_generator_node: The TG node to use in this test run.
vdevs: The names of virtual devices to test.
+ random_seed: The seed to use for pseudo-random generation.
"""
build_targets: list[BuildTargetConfiguration]
@@ -455,6 +456,7 @@ class TestRunConfiguration:
system_under_test_node: SutNodeConfiguration
traffic_generator_node: TGNodeConfiguration
vdevs: list[str]
+ random_seed: int | None
@classmethod
def from_dict(
@@ -497,6 +499,7 @@ def from_dict(
vdevs = (
d["system_under_test_node"]["vdevs"] if "vdevs" in
d["system_under_test_node"] else []
)
+ random_seed = d.get("random_seed", None)
return cls(
build_targets=build_targets,
perf=d["perf"],
@@ -506,6 +509,7 @@ def from_dict(
system_under_test_node=system_under_test_node,
traffic_generator_node=traffic_generator_node,
vdevs=vdevs,
+ random_seed=random_seed,
)
def copy_and_modify(self, **kwargs) -> Self:
diff --git a/dts/framework/config/conf_yaml_schema.json
b/dts/framework/config/conf_yaml_schema.json
index f02a310bb5..df390e8ae2 100644
--- a/dts/framework/config/conf_yaml_schema.json
+++ b/dts/framework/config/conf_yaml_schema.json
@@ -379,6 +379,10 @@
},
"traffic_generator_node": {
"$ref": "#/definitions/node_name"
+ },
+ "random_seed": {
+ "type": "integer",
+ "description": "Optional field. Allows you to set a seed for
pseudo-random generation."
}
},
"additionalProperties": false,
diff --git a/dts/framework/config/types.py b/dts/framework/config/types.py
index cf16556403..ce7b784ac8 100644
--- a/dts/framework/config/types.py
+++ b/dts/framework/config/types.py
@@ -121,6 +121,8 @@ class TestRunConfigDict(TypedDict):
system_under_test_node: TestRunSUTConfigDict
#:
traffic_generator_node: str
+ #:
+ random_seed: int
class ConfigurationDict(TypedDict):
diff --git a/dts/framework/runner.py b/dts/framework/runner.py
index 6b6f6a05f5..34b1dad5c4 100644
--- a/dts/framework/runner.py
+++ b/dts/framework/runner.py
@@ -20,6 +20,7 @@
import importlib
import inspect
import os
+import random
import re
import sys
from pathlib import Path
@@ -147,6 +148,7 @@ def run(self) -> None:
self._logger.info(
f"Running test run with SUT
'{test_run_config.system_under_test_node.name}'."
)
+ self._init_random_seed(test_run_config)
test_run_result = self._result.add_test_run(test_run_config)
# we don't want to modify the original config, so create a
copy
test_run_test_suites = list(
@@ -723,3 +725,9 @@ def _exit_dts(self) -> None:
self._logger.info("DTS execution has ended.")
sys.exit(self._result.get_return_code())
+
+ def _init_random_seed(self, conf: TestRunConfiguration) -> None:
+ """Initialize the random seed to use for the test run."""
+ seed = SETTINGS.random_seed or conf.random_seed or
random.randrange(0xFFFF_FFFF)
+ self._logger.info(f"Initializing test run with random seed {seed}")
+ random.seed(seed)
diff --git a/dts/framework/settings.py b/dts/framework/settings.py
index f6303066d4..7744e37f54 100644
--- a/dts/framework/settings.py
+++ b/dts/framework/settings.py
@@ -66,6 +66,12 @@
Re-run each test case this many times in case of a failure.
+.. option:: --random-seed
+.. envvar:: DTS_RANDOM_SEED
+
+ The seed to use with the pseudo-random generator. If not specified, the
configuration value is
+ used instead. If that's also not specified, a random seed is generated.
+
The module provides one key module-level variable:
Attributes:
@@ -115,6 +121,8 @@ class Settings:
test_suites: list[TestSuiteConfig] = field(default_factory=list)
#:
re_run: int = 0
+ #:
+ random_seed: int | None = None
SETTINGS: Settings = Settings()
@@ -375,6 +383,15 @@ def _get_parser() -> _DTSArgumentParser:
)
_add_env_var_to_action(action, "RERUN")
+ action = parser.add_argument(
+ "--random-seed",
+ type=int,
+ help="The seed to use with the pseudo-random generator. If not specified,
the configuration"
+ " value is used instead. If that's also not specified, a random seed is
generated.",
+ metavar="NUMBER",
+ )
+ _add_env_var_to_action(action)
+
return parser
