[M] Change in osmo-ttcn3-hacks[master]: testenv: add run --until-nok

2024-11-18 Thread osmith 
osmith has submitted this change. ( 
https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email )

 (

1 is the latest approved patch-set.
No files were changed between the latest approved patch-set and the submitted 
one.
 )Change subject: testenv: add run --until-nok
..

testenv: add run --until-nok

Add an argument to run a specific test (if using --test) or a whole
testsuite until it fails with "failure" or "error". This helped me in
reproducing a race condition in the mgw testsuite (related issue).

Related: OS#3849
Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
---
M _testenv/testenv.py
M _testenv/testenv/__init__.py
M _testenv/testenv/testdir.py
M _testenv/testenv/testenv_cfg.py
M _testenv/testenv/testsuite.py
5 files changed, 76 insertions(+), 23 deletions(-)

Approvals:
  pespin: Looks good to me, but someone else must approve
  Jenkins Builder: Verified
  fixeria: Looks good to me, approved
  laforge: Looks good to me, but someone else must approve




diff --git a/_testenv/testenv.py b/_testenv/testenv.py
index 81b3aac..b9de4e8 100755
--- a/_testenv/testenv.py
+++ b/_testenv/testenv.py
@@ -16,6 +16,21 @@
 import testenv.testsuite


+def loop_continue_cond(loop_count):
+if loop_count == 0:
+return True
+
+if testenv.args.until_nok:
+logging.info("Checking testsuite logs for failures and errors")
+for match_str in ["failures='0'", "errors='0'"]:
+if not testenv.testsuite.check_junit_logs_have(loop_count - 1, 
match_str):
+logging.critical("Stopping the loop")
+return False
+return True
+else:
+return False
+
+
 def run():
 testenv.testenv_cfg.init()

@@ -45,26 +60,34 @@
 testenv.osmo_dev.make(cfg)

 # Run the components + testsuite
-cfg_count = 0
-for cfg_name, cfg in testenv.testenv_cfg.cfgs.items():
-# Restart podman container before running with another config
-if testenv.args.podman and cfg_count:
+loop_count = 0
+while loop_continue_cond(loop_count):
+# Restart podman container before running again
+if testenv.args.podman and loop_count:
 testenv.podman.stop(True)

-testenv.testenv_cfg.set_current(cfg_name)
+cfg_count = 0
+for cfg_name, cfg in testenv.testenv_cfg.cfgs.items():
+# Restart podman container before running with another config
+if testenv.args.podman and cfg_count:
+testenv.podman.stop(True)

-if testenv.args.binary_repo:
-testenv.podman.enable_binary_repo()
-testenv.podman_install.packages(cfg, cfg_name)
+testenv.testenv_cfg.set_current(cfg_name, loop_count)

-testenv.testdir.prepare(cfg_name, cfg)
-testenv.daemons.start(cfg)
-testenv.testsuite.run(cfg)
-testenv.daemons.stop()
-testenv.testdir.clean_run_scripts("finished")
+if testenv.args.binary_repo:
+testenv.podman.enable_binary_repo()
+testenv.podman_install.packages(cfg, cfg_name)

-cfg_count += 1
-testenv.set_log_prefix("[testenv]")
+testenv.testdir.prepare(cfg_name, cfg, loop_count)
+testenv.daemons.start(cfg)
+testenv.testsuite.run(cfg)
+testenv.daemons.stop()
+testenv.testdir.clean_run_scripts("finished")
+
+cfg_count += 1
+testenv.set_log_prefix("[testenv]")
+
+loop_count += 1

 # Show test results
 testenv.testsuite.cat_junit_logs()
diff --git a/_testenv/testenv/__init__.py b/_testenv/testenv/__init__.py
index eb687f3..c1991ca 100644
--- a/_testenv/testenv/__init__.py
+++ b/_testenv/testenv/__init__.py
@@ -93,6 +93,14 @@
 help="use binary packages from this Osmocom OBS project instead (e.g. 
osmocom:nightly)",
 )

+group = sub_run.add_argument_group("loop options", "Run the testsuite / a 
single test multiple times.")
+group.add_argument(
+"-u",
+"--until-nok",
+action="store_true",
+help="run until there was either a failure or error",
+)
+
 group = sub_run.add_argument_group(
 "QEMU options",
 "For some tests, the SUT can or must run in QEMU, typically to use 
kernel GTP-U.",
diff --git a/_testenv/testenv/testdir.py b/_testenv/testenv/testdir.py
index 88a2dca..b18efd1 100644
--- a/_testenv/testenv/testdir.py
+++ b/_testenv/testenv/testdir.py
@@ -51,14 +51,19 @@
 testenv.cmd.run(["ln", "-sf", testdir_topdir, "/tmp/logs"], 
no_podman=True)


-def prepare(cfg_name, cfg):
+def prepare(cfg_name, cfg, loop_count):
 global testdir
 global clean_scripts

+topdir = testdir_topdir
+
+if testenv.args.until_nok:
+topdir = os.path.join(topdir, f"loop-{loop_count}")
+
 if len(testenv.testenv_cfg.cfgs) == 1:
-testdir = testdir_topdir
+testdir = topdir

[M] Change in osmo-ttcn3-hacks[master]: testenv: add run --until-nok

2024-11-18 Thread osmith 
Attention is currently required from: daniel, osmith.

Hello Jenkins Builder, daniel, fixeria, laforge, pespin,

I'd like you to reexamine a change. Please visit

https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email

to look at the new patch set (#2).

The following approvals got outdated and were removed:
Verified+1 by Jenkins Builder

The change is no longer submittable: Verified is unsatisfied now.


Change subject: testenv: add run --until-nok
..

testenv: add run --until-nok

Add an argument to run a specific test (if using --test) or a whole
testsuite until it fails with "failure" or "error". This helped me in
reproducing a race condition in the mgw testsuite (related issue).

Related: OS#3849
Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
---
M _testenv/testenv.py
M _testenv/testenv/__init__.py
M _testenv/testenv/testdir.py
M _testenv/testenv/testenv_cfg.py
M _testenv/testenv/testsuite.py
5 files changed, 76 insertions(+), 23 deletions(-)


  git pull ssh://gerrit.osmocom.org:29418/osmo-ttcn3-hacks 
refs/changes/76/38776/2
--
To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email
To unsubscribe, or for help writing mail filters, visit 
https://gerrit.osmocom.org/settings?usp=email

Gerrit-MessageType: newpatchset
Gerrit-Project: osmo-ttcn3-hacks
Gerrit-Branch: master
Gerrit-Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
Gerrit-Change-Number: 38776
Gerrit-PatchSet: 2
Gerrit-Owner: osmith 
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: daniel 
Gerrit-Reviewer: fixeria 
Gerrit-Reviewer: laforge 
Gerrit-Reviewer: pespin 
Gerrit-Attention: osmith 
Gerrit-Attention: daniel

[M] Change in osmo-ttcn3-hacks[master]: testenv: add run --until-nok

2024-11-18 Thread fixeria 
Attention is currently required from: daniel, osmith.

fixeria has posted comments on this change by osmith. ( 
https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email )

Change subject: testenv: add run --until-nok
..


Patch Set 1: Code-Review+2


--
To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email
To unsubscribe, or for help writing mail filters, visit 
https://gerrit.osmocom.org/settings?usp=email

Gerrit-MessageType: comment
Gerrit-Project: osmo-ttcn3-hacks
Gerrit-Branch: master
Gerrit-Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
Gerrit-Change-Number: 38776
Gerrit-PatchSet: 1
Gerrit-Owner: osmith 
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: daniel 
Gerrit-Reviewer: fixeria 
Gerrit-Reviewer: laforge 
Gerrit-Reviewer: pespin 
Gerrit-Attention: osmith 
Gerrit-Attention: daniel 
Gerrit-Comment-Date: Mon, 18 Nov 2024 19:46:48 +
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes

[M] Change in osmo-ttcn3-hacks[master]: testenv: add run --until-nok

2024-11-18 Thread laforge 
Attention is currently required from: daniel, fixeria, osmith.

laforge has posted comments on this change by osmith. ( 
https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email )

Change subject: testenv: add run --until-nok
..


Patch Set 1: Code-Review+1

(1 comment)

Patchset:

PS1:
I don't really have much expericene with testenv...



--
To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email
To unsubscribe, or for help writing mail filters, visit 
https://gerrit.osmocom.org/settings?usp=email

Gerrit-MessageType: comment
Gerrit-Project: osmo-ttcn3-hacks
Gerrit-Branch: master
Gerrit-Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
Gerrit-Change-Number: 38776
Gerrit-PatchSet: 1
Gerrit-Owner: osmith 
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: daniel 
Gerrit-Reviewer: fixeria 
Gerrit-Reviewer: laforge 
Gerrit-Reviewer: pespin 
Gerrit-Attention: osmith 
Gerrit-Attention: fixeria 
Gerrit-Attention: daniel 
Gerrit-Comment-Date: Mon, 18 Nov 2024 16:58:43 +
Gerrit-HasComments: Yes
Gerrit-Has-Labels: Yes

[M] Change in osmo-ttcn3-hacks[master]: testenv: add run --until-nok

2024-11-14 Thread pespin 
Attention is currently required from: osmith.

pespin has posted comments on this change by osmith. ( 
https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email )

Change subject: testenv: add run --until-nok
..


Patch Set 1: Code-Review+1


--
To view, visit https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email
To unsubscribe, or for help writing mail filters, visit 
https://gerrit.osmocom.org/settings?usp=email

Gerrit-MessageType: comment
Gerrit-Project: osmo-ttcn3-hacks
Gerrit-Branch: master
Gerrit-Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
Gerrit-Change-Number: 38776
Gerrit-PatchSet: 1
Gerrit-Owner: osmith 
Gerrit-Reviewer: Jenkins Builder
Gerrit-Reviewer: pespin 
Gerrit-Attention: osmith 
Gerrit-Comment-Date: Thu, 14 Nov 2024 16:34:05 +
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes

[M] Change in osmo-ttcn3-hacks[master]: testenv: add run --until-nok

2024-11-14 Thread osmith 
osmith has uploaded this change for review. ( 
https://gerrit.osmocom.org/c/osmo-ttcn3-hacks/+/38776?usp=email )


Change subject: testenv: add run --until-nok
..

testenv: add run --until-nok

Add an argument to run a specific test (if using --test) or a whole
testsuite until it fails with "failure" or "error". This helped me in
reproducing a race condition in the mgw testsuite (related issue).

Related: OS#3849
Change-Id: I17e1ebcc5d6ff1b6a087c4d4c9405a02798212f1
---
M _testenv/testenv.py
M _testenv/testenv/__init__.py
M _testenv/testenv/testdir.py
M _testenv/testenv/testenv_cfg.py
M _testenv/testenv/testsuite.py
5 files changed, 76 insertions(+), 23 deletions(-)



  git pull ssh://gerrit.osmocom.org:29418/osmo-ttcn3-hacks 
refs/changes/76/38776/1

diff --git a/_testenv/testenv.py b/_testenv/testenv.py
index 81b3aac..b9de4e8 100755
--- a/_testenv/testenv.py
+++ b/_testenv/testenv.py
@@ -16,6 +16,21 @@
 import testenv.testsuite


+def loop_continue_cond(loop_count):
+if loop_count == 0:
+return True
+
+if testenv.args.until_nok:
+logging.info("Checking testsuite logs for failures and errors")
+for match_str in ["failures='0'", "errors='0'"]:
+if not testenv.testsuite.check_junit_logs_have(loop_count - 1, 
match_str):
+logging.critical("Stopping the loop")
+return False
+return True
+else:
+return False
+
+
 def run():
 testenv.testenv_cfg.init()

@@ -45,26 +60,34 @@
 testenv.osmo_dev.make(cfg)

 # Run the components + testsuite
-cfg_count = 0
-for cfg_name, cfg in testenv.testenv_cfg.cfgs.items():
-# Restart podman container before running with another config
-if testenv.args.podman and cfg_count:
+loop_count = 0
+while loop_continue_cond(loop_count):
+# Restart podman container before running again
+if testenv.args.podman and loop_count:
 testenv.podman.stop(True)

-testenv.testenv_cfg.set_current(cfg_name)
+cfg_count = 0
+for cfg_name, cfg in testenv.testenv_cfg.cfgs.items():
+# Restart podman container before running with another config
+if testenv.args.podman and cfg_count:
+testenv.podman.stop(True)

-if testenv.args.binary_repo:
-testenv.podman.enable_binary_repo()
-testenv.podman_install.packages(cfg, cfg_name)
+testenv.testenv_cfg.set_current(cfg_name, loop_count)

-testenv.testdir.prepare(cfg_name, cfg)
-testenv.daemons.start(cfg)
-testenv.testsuite.run(cfg)
-testenv.daemons.stop()
-testenv.testdir.clean_run_scripts("finished")
+if testenv.args.binary_repo:
+testenv.podman.enable_binary_repo()
+testenv.podman_install.packages(cfg, cfg_name)

-cfg_count += 1
-testenv.set_log_prefix("[testenv]")
+testenv.testdir.prepare(cfg_name, cfg, loop_count)
+testenv.daemons.start(cfg)
+testenv.testsuite.run(cfg)
+testenv.daemons.stop()
+testenv.testdir.clean_run_scripts("finished")
+
+cfg_count += 1
+testenv.set_log_prefix("[testenv]")
+
+loop_count += 1

 # Show test results
 testenv.testsuite.cat_junit_logs()
diff --git a/_testenv/testenv/__init__.py b/_testenv/testenv/__init__.py
index eb687f3..c1991ca 100644
--- a/_testenv/testenv/__init__.py
+++ b/_testenv/testenv/__init__.py
@@ -93,6 +93,14 @@
 help="use binary packages from this Osmocom OBS project instead (e.g. 
osmocom:nightly)",
 )

+group = sub_run.add_argument_group("loop options", "Run the testsuite / a 
single test multiple times.")
+group.add_argument(
+"-u",
+"--until-nok",
+action="store_true",
+help="run until there was either a failure or error",
+)
+
 group = sub_run.add_argument_group(
 "QEMU options",
 "For some tests, the SUT can or must run in QEMU, typically to use 
kernel GTP-U.",
diff --git a/_testenv/testenv/testdir.py b/_testenv/testenv/testdir.py
index 88a2dca..b18efd1 100644
--- a/_testenv/testenv/testdir.py
+++ b/_testenv/testenv/testdir.py
@@ -51,14 +51,19 @@
 testenv.cmd.run(["ln", "-sf", testdir_topdir, "/tmp/logs"], 
no_podman=True)


-def prepare(cfg_name, cfg):
+def prepare(cfg_name, cfg, loop_count):
 global testdir
 global clean_scripts

+topdir = testdir_topdir
+
+if testenv.args.until_nok:
+topdir = os.path.join(topdir, f"loop-{loop_count}")
+
 if len(testenv.testenv_cfg.cfgs) == 1:
-testdir = testdir_topdir
+testdir = topdir
 else:
-testdir = os.path.join(testdir_topdir, cfg_name.replace("testenv_", 
"").replace(".cfg", ""))
+testdir = os.path.join(topdir, cfg_name.replace("testenv_", 
"").replace(".cfg", ""))

 logging.info(