osmith has submitted this change. ( 
https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email )

 (

1 is the latest approved patch-set.
No files were changed between the latest approved patch-set and the submitted 
one.
 )Change subject: testenv: add run --until-nok
......................................................................

testenv: add run --until-nok

Add an argument to run a specific test (if using --test) or a whole
testsuite until it fails with "failure" or "error". This helped me in
reproducing a race condition in the mgw testsuite (related issue).

Related: OS#3849
Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
---
M _testenv/testenv.py
M _testenv/testenv/__init__.py
M _testenv/testenv/testdir.py
M _testenv/testenv/testenv_cfg.py
M _testenv/testenv/testsuite.py
5 files changed, 76 insertions(+), 23 deletions(-)

Approvals:
  pespin: Looks good to me, but someone else must approve
  Jenkins Builder: Verified
  fixeria: Looks good to me, approved
  laforge: Looks good to me, but someone else must approve




diff --git a/_testenv/testenv.py b/_testenv/testenv.py
index 81b3aac..b9de4e8 100755
--- a/_testenv/testenv.py
+++ b/_testenv/testenv.py
@@ -16,6 +16,21 @@
 import testenv.testsuite


+def loop_continue_cond(loop_count):
+    if loop_count == 0:
+        return True
+
+    if testenv.args.until_nok:
+        logging.info("Checking testsuite logs for failures and errors")
+        for match_str in ["failures='0'", "errors='0'"]:
+            if not testenv.testsuite.check_junit_logs_have(loop_count - 1, 
match_str):
+                logging.critical("Stopping the loop")
+                return False
+        return True
+    else:
+        return False
+
+
 def run():
     testenv.testenv_cfg.init()

@@ -45,26 +60,34 @@
             testenv.osmo_dev.make(cfg)

     # Run the components + testsuite
-    cfg_count = 0
-    for cfg_name, cfg in testenv.testenv_cfg.cfgs.items():
-        # Restart podman container before running with another config
-        if testenv.args.podman and cfg_count:
+    loop_count = 0
+    while loop_continue_cond(loop_count):
+        # Restart podman container before running again
+        if testenv.args.podman and loop_count:
             testenv.podman.stop(True)

-        testenv.testenv_cfg.set_current(cfg_name)
+        cfg_count = 0
+        for cfg_name, cfg in testenv.testenv_cfg.cfgs.items():
+            # Restart podman container before running with another config
+            if testenv.args.podman and cfg_count:
+                testenv.podman.stop(True)

-        if testenv.args.binary_repo:
-            testenv.podman.enable_binary_repo()
-            testenv.podman_install.packages(cfg, cfg_name)
+            testenv.testenv_cfg.set_current(cfg_name, loop_count)

-        testenv.testdir.prepare(cfg_name, cfg)
-        testenv.daemons.start(cfg)
-        testenv.testsuite.run(cfg)
-        testenv.daemons.stop()
-        testenv.testdir.clean_run_scripts("finished")
+            if testenv.args.binary_repo:
+                testenv.podman.enable_binary_repo()
+                testenv.podman_install.packages(cfg, cfg_name)

-        cfg_count += 1
-        testenv.set_log_prefix("[testenv]")
+            testenv.testdir.prepare(cfg_name, cfg, loop_count)
+            testenv.daemons.start(cfg)
+            testenv.testsuite.run(cfg)
+            testenv.daemons.stop()
+            testenv.testdir.clean_run_scripts("finished")
+
+            cfg_count += 1
+            testenv.set_log_prefix("[testenv]")
+
+        loop_count += 1

     # Show test results
     testenv.testsuite.cat_junit_logs()
diff --git a/_testenv/testenv/__init__.py b/_testenv/testenv/__init__.py
index eb687f3..c1991ca 100644
--- a/_testenv/testenv/__init__.py
+++ b/_testenv/testenv/__init__.py
@@ -93,6 +93,14 @@
         help="use binary packages from this Osmocom OBS project instead (e.g. 
osmocom:nightly)",
     )

+    group = sub_run.add_argument_group("loop options", "Run the testsuite / a 
single test multiple times.")
+    group.add_argument(
+        "-u",
+        "--until-nok",
+        action="store_true",
+        help="run until there was either a failure or error",
+    )
+
     group = sub_run.add_argument_group(
         "QEMU options",
         "For some tests, the SUT can or must run in QEMU, typically to use 
kernel GTP-U.",
diff --git a/_testenv/testenv/testdir.py b/_testenv/testenv/testdir.py
index 88a2dca..b18efd1 100644
--- a/_testenv/testenv/testdir.py
+++ b/_testenv/testenv/testdir.py
@@ -51,14 +51,19 @@
         testenv.cmd.run(["ln", "-sf", testdir_topdir, "/tmp/logs"], 
no_podman=True)


-def prepare(cfg_name, cfg):
+def prepare(cfg_name, cfg, loop_count):
     global testdir
     global clean_scripts

+    topdir = testdir_topdir
+
+    if testenv.args.until_nok:
+        topdir = os.path.join(topdir, f"loop-{loop_count}")
+
     if len(testenv.testenv_cfg.cfgs) == 1:
-        testdir = testdir_topdir
+        testdir = topdir
     else:
-        testdir = os.path.join(testdir_topdir, cfg_name.replace("testenv_", 
"").replace(".cfg", ""))
+        testdir = os.path.join(topdir, cfg_name.replace("testenv_", 
"").replace(".cfg", ""))

     logging.info(f"Preparing testdir: {testdir}")
     testsuite_dir = os.path.join(testenv.testsuite.ttcn3_hacks_dir, 
testenv.args.testsuite)
diff --git a/_testenv/testenv/testenv_cfg.py b/_testenv/testenv/testenv_cfg.py
index fefaf5c..4044225 100644
--- a/_testenv/testenv/testenv_cfg.py
+++ b/_testenv/testenv/testenv_cfg.py
@@ -13,16 +13,20 @@
 current = None


-def set_current(cfg_name):
+def set_current(cfg_name, loop_count=0):
     global current
     current = cfg_name
+    loop_str = ""
+
+    if testenv.args.until_nok:
+        loop_str = f"[loop-{loop_count}]"

     if cfg_name == "testenv.cfg":
-        testenv.set_log_prefix("[testenv]")
+        testenv.set_log_prefix(f"[testenv]{loop_str}")
     else:
         cfg_name = cfg_name.replace("testenv_", "")
         cfg_name = cfg_name.replace(".cfg", "")
-        testenv.set_log_prefix(f"[testenv][{cfg_name}]")
+        testenv.set_log_prefix(f"[testenv]{loop_str}[{cfg_name}]")


 def exit_error_readme():
diff --git a/_testenv/testenv/testsuite.py b/_testenv/testenv/testsuite.py
index 1878b5c..0fa42ab 100644
--- a/_testenv/testenv/testsuite.py
+++ b/_testenv/testenv/testsuite.py
@@ -138,6 +138,11 @@
     testenv.cmd.run(cmd, cwd=cwd)


+def get_junit_logs(topdir):
+    pattern = os.path.join(topdir, "**", "junit-*.log")
+    return glob.glob(pattern, recursive=True)
+
+
 def cat_junit_logs():
     tool = "cat"

@@ -145,13 +150,21 @@
         colors = os.environ.get("TESTENV_SOURCE_HIGHLIGHT_COLORS", "esc256")
         tool = f"source-highlight -f {shlex.quote(colors)} -s xml -i"

-    pattern = os.path.join(testenv.testdir.testdir_topdir, "**", "junit-*.log")
-    for path in glob.glob(pattern, recursive=True):
+    for path in get_junit_logs(testenv.testdir.testdir_topdir):
         cmd = f"echo && {tool} {shlex.quote(path)} && echo"
         logging.info(f"Showing {os.path.relpath(path, 
testenv.testdir.testdir_topdir)}")
         testenv.cmd.run(cmd)


+def check_junit_logs_have(loop_count, match_str):
+    topdir = os.path.join(testenv.testdir.testdir_topdir, f"loop-{loop_count}")
+    for path in get_junit_logs(topdir):
+        cmd = ["grep", "-q", match_str, path]
+        if testenv.cmd.run(cmd, check=False).returncode:
+            return False
+    return True
+
+
 def run(cfg):
     global testsuite_proc


--
To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email
To unsubscribe, or for help writing mail filters, visit 
https://gerrit.osmocom.org/settings?usp=email

Gerrit-MessageType: merged
Gerrit-Project: osmo-ttcn3-hacks
Gerrit-Branch: master
Gerrit-Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
Gerrit-Change-Number: 38776
Gerrit-PatchSet: 2
Gerrit-Owner: osmith <osm...@sysmocom.de>
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: daniel <dwillm...@sysmocom.de>
Gerrit-Reviewer: fixeria <vyanits...@sysmocom.de>
Gerrit-Reviewer: laforge <lafo...@osmocom.org>
Gerrit-Reviewer: osmith <osm...@sysmocom.de>
Gerrit-Reviewer: pespin <pes...@sysmocom.de>

Reply via email to