[Piglit] [PATCH] tests/deqp: add configuration option to control caselist generation
Right now, each time any deqp-like profile ask for the case list, it
calls to generate the caselist. As by default it doesn't pass any
parameter, it creates all the possible caselist. And there are some
test sets (like khr_gles) that calls this method several time. On slow
machines this means a significant startup time.
This commit adds a envvar that allows to keep the current
behaviour. But if disabled, if the caselist already exists, compare
the creation times with the deqp binary. If the binary is older that
the caselist, it assumes that the caselist is recent enough, and skips
calling the caselist generation method.
---
framework/test/deqp.py | 8
piglit.conf.example| 6 ++
2 files changed, 14 insertions(+)
diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 5db2a922f..aea335079 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -66,6 +66,9 @@ _EXTRA_ARGS = get_option('PIGLIT_DEQP_EXTRA_ARGS',
('deqp', 'extra_args'),
default='').split()
+force_caselist_regen = get_option('PIGLIT_DEQP_FORCE_CASELIST_REGEN',
+ ('deqp', 'force_caselist_regen'),
+ default='True')
def select_source(bin_, filename, mustpass, extra_args):
"""Return either the mustpass list or the generated list."""
@@ -116,6 +119,11 @@ def gen_caselist_txt(bin_, caselist, extra_args):
basedir = os.path.dirname(bin_)
caselist_path = os.path.join(basedir, caselist)
+if (force_caselist_regen == 'False' and
+os.path.exists(caselist_path) and
+os.path.getmtime(caselist_path) > os.path.getmtime(bin_)):
+return caselist_path
+
# TODO: need to catch some exceptions here...
with open(os.devnull, 'w') as d:
env = os.environ.copy()
diff --git a/piglit.conf.example b/piglit.conf.example
index 1877187df..5bb1ddff9 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -37,6 +37,12 @@ testB
; Options that affect all deqp based suites
;extra_args=--deqp-visibility=hidden
+; If we regenerate the caselist files everytime we ask them, or if we
+; reuse them if already present and more recent that the deqp
+; binary. The environment variable PIGLIT_DEQP_FORGE_CASELIST_REGEN
+; overrides de value set here.
+; force_caselist_regen=True
+
[deqp-egl]
; Path to the deqp-egl executable
; Can be overwritten by PIGLIT_DEQP_EGL_BIN environment variable
--
2.19.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit
Re: [Piglit] [PATCH] tests/deqp: use --deqp-case when generating case-list
I have realized that this solution have a big problem. You can pass extra args to deqp, so several people are already using it to pass --deqp-case through extra args. This patch would affect them. So Im dropping this patch. sorry for the noise. On 6/6/19 14:48, Alejandro Piñeiro wrote: Without this commit, generating the case list is called without any parameter by default, so it would generate all the caselist, even if you don't need it, or if you have just called the method before (like in khr_gles where you call several time gen_caselist). For the latter an alternative would just check if the caselist files are already created, and return if they exist. But in that case, we would need to add a mechanism to force creating them. --- framework/test/deqp.py | 6 -- tests/cts_gl.py| 20 ++-- tests/cts_gl45.py | 2 +- tests/cts_gles.py | 9 - tests/gtf_gles.py | 6 +++--- tests/khr_gl.py| 22 +++--- tests/khr_gles.py | 11 +-- tests/khr_noctx.py | 2 +- 8 files changed, 39 insertions(+), 39 deletions(-) diff --git a/framework/test/deqp.py b/framework/test/deqp.py index 5db2a922f..7c1a82c2a 100644 --- a/framework/test/deqp.py +++ b/framework/test/deqp.py @@ -114,7 +114,7 @@ def gen_caselist_txt(bin_, caselist, extra_args): # differ then we cannot pre-generate the caselist on the build host: # we must *dynamically* generate it during the testrun. basedir = os.path.dirname(bin_) -caselist_path = os.path.join(basedir, caselist) +caselist_path = os.path.join(basedir, caselist + '-cases.txt') # TODO: need to catch some exceptions here... with open(os.devnull, 'w') as d: @@ -123,8 +123,10 @@ def gen_caselist_txt(bin_, caselist, extra_args): env['MESA_GLES_VERSION_OVERRIDE'] = '3.2' subprocess.check_call( -[bin_, '--deqp-runmode=txt-caselist'] + extra_args, cwd=basedir, +[bin_, '--deqp-runmode=txt-caselist' , '--deqp-case=' , caselist, '.*'] ++ extra_args, cwd=basedir, stdout=d, stderr=d, env=env) + assert os.path.exists(caselist_path) return caselist_path diff --git a/tests/cts_gl.py b/tests/cts_gl.py index 935df7a7c..fa36ea4f9 100644 --- a/tests/cts_gl.py +++ b/tests/cts_gl.py @@ -70,24 +70,24 @@ class DEQPCTSTest(deqp.DEQPBaseTest): profile = deqp.make_profile( # pylint: disable=invalid-name itertools.chain( deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS', _EXTRA_ARGS)), ), DEQPCTSTest) diff --git a/tests/cts_gl45.py b/tests/cts_gl45.py index d033c3efc..070ed8026 100644 --- a/tests/cts_gl45.py +++ b/tests/cts_gl45.py @@ -64,6 +64,6 @@ class DEQPCTSTest(deqp.DEQPBaseTest): profile = deqp.make_profile( # pylint: disable=invalid-name itertools.chain( deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS', _EXTRA_ARGS)), ), DEQPCTSTest) diff --git a/tests/cts_gles.py b/tests/cts_gles.p
[Piglit] [PATCH] tests/deqp: use --deqp-case when generating case-list
Without this commit, generating the case list is called without any parameter by default, so it would generate all the caselist, even if you don't need it, or if you have just called the method before (like in khr_gles where you call several time gen_caselist). For the latter an alternative would just check if the caselist files are already created, and return if they exist. But in that case, we would need to add a mechanism to force creating them. --- framework/test/deqp.py | 6 -- tests/cts_gl.py| 20 ++-- tests/cts_gl45.py | 2 +- tests/cts_gles.py | 9 - tests/gtf_gles.py | 6 +++--- tests/khr_gl.py| 22 +++--- tests/khr_gles.py | 11 +-- tests/khr_noctx.py | 2 +- 8 files changed, 39 insertions(+), 39 deletions(-) diff --git a/framework/test/deqp.py b/framework/test/deqp.py index 5db2a922f..7c1a82c2a 100644 --- a/framework/test/deqp.py +++ b/framework/test/deqp.py @@ -114,7 +114,7 @@ def gen_caselist_txt(bin_, caselist, extra_args): # differ then we cannot pre-generate the caselist on the build host: # we must *dynamically* generate it during the testrun. basedir = os.path.dirname(bin_) -caselist_path = os.path.join(basedir, caselist) +caselist_path = os.path.join(basedir, caselist + '-cases.txt') # TODO: need to catch some exceptions here... with open(os.devnull, 'w') as d: @@ -123,8 +123,10 @@ def gen_caselist_txt(bin_, caselist, extra_args): env['MESA_GLES_VERSION_OVERRIDE'] = '3.2' subprocess.check_call( -[bin_, '--deqp-runmode=txt-caselist'] + extra_args, cwd=basedir, +[bin_, '--deqp-runmode=txt-caselist' , '--deqp-case=' , caselist, '.*'] ++ extra_args, cwd=basedir, stdout=d, stderr=d, env=env) + assert os.path.exists(caselist_path) return caselist_path diff --git a/tests/cts_gl.py b/tests/cts_gl.py index 935df7a7c..fa36ea4f9 100644 --- a/tests/cts_gl.py +++ b/tests/cts_gl.py @@ -70,24 +70,24 @@ class DEQPCTSTest(deqp.DEQPBaseTest): profile = deqp.make_profile( # pylint: disable=invalid-name itertools.chain( deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL30-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL31-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL32-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL33-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL40-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL41-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL42-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL43-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL44-CTS', _EXTRA_ARGS)), deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS', _EXTRA_ARGS)), ), DEQPCTSTest) diff --git a/tests/cts_gl45.py b/tests/cts_gl45.py index d033c3efc..070ed8026 100644 --- a/tests/cts_gl45.py +++ b/tests/cts_gl45.py @@ -64,6 +64,6 @@ class DEQPCTSTest(deqp.DEQPBaseTest): profile = deqp.make_profile( # pylint: disable=invalid-name itertools.chain( deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS-cases.txt', _EXTRA_ARGS)), +deqp.gen_caselist_txt(_CTS_BIN, 'GL45-CTS', _EXTRA_ARGS)), ), DEQPCTSTest) diff --git a/tests/cts_gles.py b/tests/cts_gles.py index 4ee28a89e..d65a9d56e 100644 --- a/tests/cts_gles.py +++ b/tests/cts_gles.py @@ -72,13 +72,12 @@ class DEQPCTSTest(deqp.DEQPBaseTest): profile = deqp.make_profile( # pylint: disable=invalid-name itertools.chain( deqp.iter_deqp_test_cases( -deqp.gen_caselist_txt(_CTS_BIN, 'ES2-CTS-cases.txt', _EXTRA_AR
