Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

Quoting Juan A. Suarez Romero (2018-05-02 09:49:08)
> Hi, Dylan.
> 
> I see you've pushed this series.
> 
> Now, when I'm trying to run some profiles (mainly, tests/crucible and
> tests/khr_gl* ), seems they are broken:
> 
> [/7776]
> Traceback (most recent call last):
>   File "./piglit", line 178, in 
> main()
>   File "./piglit", line 174, in main
> sys.exit(runner(args))
>   File "/home/igalia/jasuarez/piglit/framework/exceptions.py", line 51, in
> _inner
> func(*args, **kwargs)
>   File "/home/igalia/jasuarez/piglit/framework/programs/run.py", line 370, in
> run
> backend.finalize({'time_elapsed': time_elapsed.to_json()})
>   File "/home/igalia/jasuarez/piglit/framework/backends/json.py", line 163, in
> finalize
> assert data['tests']
> AssertionError
> 
> J.A.
> 

Dang.

I can't reproduce any failures with crucible, though I did make it thread safe
and fix the using a config file :)

I can't get the glcts binary to run, no matter what target I build for I run
into either EGL errors of GL errors.

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 1/5] framework/html: guard against errors writing individual test results

[Dylan Baker]

Quoting Marek Olšák (2018-05-02 13:32:43)
> From: Nicolai Hähnle 
> 
> ---
>  framework/summary/html_.py | 18 +++---
>  1 file changed, 11 insertions(+), 7 deletions(-)
> 
> diff --git a/framework/summary/html_.py b/framework/summary/html_.py
> index f7fdc8576..512b42c24 100644
> --- a/framework/summary/html_.py
> +++ b/framework/summary/html_.py
> @@ -24,20 +24,21 @@
>  
>  from __future__ import (
>  absolute_import, division, print_function, unicode_literals
>  )
>  import errno
>  import getpass
>  import os
>  import shutil
>  import sys
>  import tempfile
> +import traceback
>  
>  import mako
>  from mako.lookup import TemplateLookup
>  import six
>  
>  # a local variable status exists, prevent accidental overloading by renaming
>  # the module
>  from framework import backends, exceptions, core
>  
>  from .common import Results, escape_filename, escape_pathname
> @@ -106,27 +107,30 @@ def _make_testrun_info(results, destination, 
> exclude=None):
>  
>  # Then build the individual test results
>  for key, value in six.iteritems(each.tests):
>  html_path = os.path.join(destination, name,
>   escape_filename(key + ".html"))
>  temp_path = os.path.dirname(html_path)
>  
>  if value.result not in exclude:
>  core.check_dir(temp_path)
>  
> -with open(html_path, 'wb') as out:
> -out.write(_TEMPLATES.get_template(
> -'test_result.mako').render(
> -testname=key,
> -value=value,
> -css=os.path.relpath(result_css, temp_path),
> -index=os.path.relpath(index, temp_path)))
> +try:
> +with open(html_path, 'wb') as out:
> +out.write(_TEMPLATES.get_template(
> +'test_result.mako').render(
> +testname=key,
> +value=value,
> +css=os.path.relpath(result_css, temp_path),
> +index=os.path.relpath(index, temp_path)))
> +except OSError as e:
> +traceback.print_exc()

This makes me really nervous. What are you trying to catch, and why is it
a good idea to print a traceback and continue?

Dylan

>  
>  
>  def _make_comparison_pages(results, destination, exclude):
>  """Create the pages of comparisons."""
>  pages = frozenset(['changes', 'problems', 'skips', 'fixes',
> 'regressions', 'enabled', 'disabled'])
>  
>  # Index.html is a bit of a special case since there is index, all, and
>  # alltests, where the other pages all use the same name. ie,
>  # changes.html, changes, and page=changes.
> -- 
> 2.17.0
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 4/5] khr_gl45: add support for mustpass lists

[Dylan Baker]

Quoting Marek Olšák (2018-05-02 13:32:46)
> From: Nicolai Hähnle <nicolai.haeh...@amd.com>
> 
> ---
>  tests/khr_gl45.py | 15 +--
>  1 file changed, 9 insertions(+), 6 deletions(-)
> 
> diff --git a/tests/khr_gl45.py b/tests/khr_gl45.py
> index 08e6280bb..95f474220 100644
> --- a/tests/khr_gl45.py
> +++ b/tests/khr_gl45.py
> @@ -36,34 +36,37 @@ PIGLIT_KHR_GL_BIN -- environment equivalent of 
> [khr_gl]:bin
>  PIGLIT_KHR_GL_EXTRA_ARGS -- environment equivalent of [khr_gl]:extra_args

please update the docstring with the change from khr_gl to khr_gl45.

with that,
Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

>  
>  """
>  
>  from __future__ import (
>  absolute_import, division, print_function, unicode_literals
>  )
>  import itertools
>  
>  from framework.test import deqp
> +from framework.options import OPTIONS
>  
>  __all__ = ['profile']
>  
> -_KHR_BIN = deqp.get_option('PIGLIT_KHR_GL_BIN', ('khr_gl', 'bin'),
> +_KHR_BIN = deqp.get_option('PIGLIT_KHR_GL_BIN', ('khr_gl45', 'bin'),
> required=True)
>  
> -_EXTRA_ARGS = deqp.get_option('PIGLIT_KHR_GL_EXTRA_ARGS', ('khr_gl', 
> 'extra_args'),
> +_KHR_MUSTPASS = deqp.get_option('PIGLIT_KHRGL45_MUSTPASS',
> + ('khr_gl45', 'mustpasslist'),
> + required=OPTIONS.deqp_mustpass)
> +
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_KHR_GL_EXTRA_ARGS', ('khr_gl45', 
> 'extra_args'),
>default='').split()
>  
>  
>  class DEQPKHRTest(deqp.DEQPBaseTest):
>  deqp_bin = _KHR_BIN
>  
>  @property
>  def extra_args(self):
>  return super(DEQPKHRTest, self).extra_args + \
>  [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>  
>  profile = deqp.make_profile(  # pylint: disable=invalid-name
> -itertools.chain(
> -deqp.iter_deqp_test_cases(
> -deqp.gen_caselist_txt(_KHR_BIN, 'KHR-GL45-cases.txt', 
> _EXTRA_ARGS)),
> -),
> +deqp.select_source(_KHR_BIN, 'KHR-GL45-cases.txt', _KHR_MUSTPASS,
> +   _EXTRA_ARGS),
>  DEQPKHRTest)
> -- 
> 2.17.0
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 5/5] deqp_egl: add support for mustpass lists

[Dylan Baker]

Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

Quoting Marek Olšák (2018-05-02 13:32:47)
> From: Nicolai Hähnle <nicolai.haeh...@amd.com>
> 
> ---
>  tests/deqp_egl.py | 10 +++---
>  1 file changed, 7 insertions(+), 3 deletions(-)
> 
> diff --git a/tests/deqp_egl.py b/tests/deqp_egl.py
> index 7422c82e3..ccd40b2af 100644
> --- a/tests/deqp_egl.py
> +++ b/tests/deqp_egl.py
> @@ -19,36 +19,40 @@
>  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
> THE
>  # SOFTWARE.
>  
>  """Piglit integrations for dEQP EGL tests."""
>  
>  from __future__ import (
>  absolute_import, division, print_function, unicode_literals
>  )
>  
>  from framework.test import deqp
> +from framework.options import OPTIONS
>  
>  __all__ = ['profile']
>  
>  _EGL_BIN = deqp.get_option('PIGLIT_DEQP_EGL_BIN',
> ('deqp-egl', 'bin'),
> required=True)
>  
> +_DEQP_MUSTPASS = deqp.get_option('PIGLIT_DEQP_EGL_MUSTPASS',
> + ('deqp-egl', 'mustpasslist'),
> + required=OPTIONS.deqp_mustpass)
> +
>  _EXTRA_ARGS = deqp.get_option('PIGLIT_DEQP_EGL_EXTRA_ARGS',
>('deqp-egl', 'extra_args'),
>default='').split()
>  
>  
>  class DEQPEGLTest(deqp.DEQPBaseTest):
>  deqp_bin = _EGL_BIN
>  
>  @property
>  def extra_args(self):
>  return super(DEQPEGLTest, self).extra_args + \
>  [x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
>  
>  
>  profile = deqp.make_profile(  # pylint: disable=invalid-name
> -deqp.iter_deqp_test_cases(
> -deqp.gen_caselist_txt(_EGL_BIN, 'dEQP-EGL-cases.txt',
> -  _EXTRA_ARGS)),
> +deqp.select_source(_EGL_BIN, 'dEQP-EGL-cases.txt', _DEQP_MUSTPASS,
> +   _EXTRA_ARGS),
>  DEQPEGLTest)
> -- 
> 2.17.0
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 3/5] framework: handle UnicodeDecodeError

[Dylan Baker]

Quoting Marek Olšák (2018-05-02 13:32:45)
> From: Marek Olšák 
> 
> This happens due to LLVM printing colored text into stdout/stderr on error.
> ---
>  framework/test/base.py | 3 +++
>  1 file changed, 3 insertions(+)
> 
> diff --git a/framework/test/base.py b/framework/test/base.py
> index 134b87245..f187c0210 100644
> --- a/framework/test/base.py
> +++ b/framework/test/base.py
> @@ -369,20 +369,23 @@ class Test(object):
>  os.killpg(os.getpgid(proc.pid), signal.SIGKILL)
>  
>  # Since the process isn't running it's safe to get any remaining
>  # stdout/stderr values out and store them.
>  self.result.out, self.result.err = proc.communicate()
>  
>  raise TestRunError(
>  'Test run time exceeded timeout value ({} seconds)\n'.format(
>  self.timeout),
>  'timeout')
> +# LLVM prints colored text into stdout/stderr on error, which raises:
> +except UnicodeDecodeError as e:
> +raise TestRunError("UnicodeDecodeError.\n", 'crash')

This seems odd to me, Popen.communicate() returns bytes, and the conversion is
done right after this (it's hideous, but there is a setter function for
result.out and result.err that converts bytes to unicode), but is uses replace
for characters it doesn't understand.

>  
>  # The setter handles the bytes/unicode conversion
>  self.result.out = out
>  self.result.err = err
>  self.result.returncode = returncode
>  
>  def __eq__(self, other):
>  return self.command == other.command
>  
>  def __ne__(self, other):
> -- 
> 2.17.0
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 2/5] framework: fix grouptools.commonprefix when len(args) == 1

[Dylan Baker]

Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

Quoting Marek Olšák (2018-05-02 13:32:44)
> From: Nicolai Hähnle <nicolai.haeh...@amd.com>
> 
> ---
>  framework/grouptools.py | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/framework/grouptools.py b/framework/grouptools.py
> index f28241d3c..ce2e97c79 100644
> --- a/framework/grouptools.py
> +++ b/framework/grouptools.py
> @@ -82,21 +82,21 @@ def splitname(group):
>  i = group.rfind(SEPARATOR) + 1
>  head, tail = group[:i], group[i:]
>  head = head.rstrip(SEPARATOR)
>  
>  return head, tail
>  
>  
>  def commonprefix(args):
>  """Given a list of groups, returns the longest common leading 
> component."""
>  if len(args) == 1:
> -return args
> +return args[0]
>  elif any(e == '' for e in args):
>  return ''
>  
>  common = []
>  
>  for elems in zip(*[split(a) for a in args]):
>  iter_ = iter(elems)
>  first = next(iter_)
>  if all(first == r for r in iter_):
>  common.append(first)
> -- 
> 2.17.0
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH] find_static_tests.py: fix python2 compatibility

[Dylan Baker]

Because python2 uses bytes, but python3 uses unicode.

CC: Michel Dänzer 
Fixes: d42d909cd754d0e2c41eec60f3a1015f2d882b95
   ("tests: Add script to find all hand written test files")
---
 tests/find_static_tests.py | 11 ++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/tests/find_static_tests.py b/tests/find_static_tests.py
index 215273159..795a56dc9 100644
--- a/tests/find_static_tests.py
+++ b/tests/find_static_tests.py
@@ -28,6 +28,8 @@ import argparse
 import io
 import os
 
+import six
+
 
 def main():
 parser = argparse.ArgumentParser()
@@ -55,7 +57,14 @@ def main():
 for dirpath, _, filenames in os.walk(directory):
 for filename in filenames:
 if os.path.splitext(filename)[1] in exts:
-files.append(os.path.join(dirpath, filename))
+name = os.path.join(dirpath, filename)
+if six.PY2:
+# This might not be correct, but it's fine. As long as the
+# two files are the same it'll work, and utf-8 is what
+# everyone *should* be using, and as a superset of ascii
+# *should* cover most people
+name = name.decode('utf-8', 'replace')
+files.append(name)
 
 if os.path.exists(args.output):
 with io.open(args.output, 'rt', encoding='utf-8') as f:
-- 
2.17.0

___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

Quoting Rafael Antognolli (2018-04-25 11:01:29)
> On Mon, Apr 23, 2018 at 10:47:08AM -0700, Dylan Baker wrote:
> > I'm planning to just push this Wednesday if no one expresses any more 
> > concerns,
> > or signals that they want time to test or review this.
> 
> I've been running some tests this morning (regular runs, using filters,
> testing print-cmd and summary), and it all worked just fine for me. It's
> not an in deep testing, but at least I can say it works for my common
> use cases. So if you want, feel free to add
> 
> Tested-by: Rafael Antognolli <rafael.antogno...@intel.com>

Thank you for testing this!

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

I'm planning to just push this Wednesday if no one expresses any more concerns,
or signals that they want time to test or review this.

Dylan

Quoting Dylan Baker (2018-04-04 15:26:48)
> I don't expect everyone I've CC'd to give thorough review (or any
> review), I've mostly CC'd people who I think would be interested in this
> work, or who's work flow I might be altered by it.
> 
> Piglit has struggled to cope with the growing number of tests that it
> contains, especially with startup time. Piglit has always calculated
> tests at runtime, which was not a problem when there were only a few
> hundred or even thousand tests. Piglit now has roughly 55,000
> OpenGL/OpenGL ES tests, which is a lot to calculate at start up. It also
> means that piglit needs to keep a python object for each of those tests
> in memory, which has sent the resident memory usage soaring. We've also
> moved to automatic test discovery for glslparser, asmparser, and shader
> tests, which is very convenient and reduces typing, but further
> increases the amount of time spent starting up. This has even made
> features which decrease runtime, like fast skipping, hurt startup
> performance, making it a less than desirable tradeoff in some cases.
> Even on a relatively fast machine with an nvme disk 15-20 seconds is not
> an unheard of startup time. That might be okay to run 55,000 tests, but
> not if you only need a dozen, such as when bisecting.
> 
> This series is my proposal to fix that, mainly by moving much of that
> cost to build time. This series creates the infrastructure build XML
> base profiles at build time, which are installed with piglit instead of
> the python profiles. These profiles are lazily iterated over to ease
> memory usage, test objects are created as they are run, and python can
> garbage collect them as soon as they are done running. Along with that
> any filters applied to profiles (like removing 80% of the vs_in shader
> tests in quick) are done before the profile is serialized, and all fast
> skipping information is collected at build time as well, and encoded in
> the XML. All this means that start times are vastly reduced.
> 
> For example:
> XML profiles
> quick: 0.5
> shader: 0.5
> 
> master
> quick: 11.6
> shader: 7.3
> 
> This series also implements some optimizations for running without
> filters or test-lists, if you add a filter quick would take 2.5
> seconds, because that is necessary to calculate the total number of
> tests before starting.
> 
> To keep classic profiles like all, quick, quick_cl, gpu, cpu, and
> llvmpipe working this series adds meta profiles, small XML snippets that
> list other profiles. These can contain other meta profiles, xml
> profiles, or python profiles. This means that for most uses cases your
> existing command line will still work, `./piglit run quick out -c` will
> still do exactly the same thing as before, just faster.
> 
> The XML generated is dumb, there is no encoding of options or logic. An
> early version of this series did contain logic and options, but the
> result was pretty terrible. It was very hard to read, and the code to
> handle it was very complicated. I've chosen not to go down that path.
> There are drawbacks, some things that relied on run time generation have
> cannot be handled the same way, among them the "multi shader" concept,
> where shader_runner consumes a directory of shader_tests at a time. This
> was previously handled via a --process-isolation=false flag, now its
> encoded into profiles, "shader_multi" and "quick_shader_multi"; there
> was also an option to use glslparsertest with ES shaders and
> ARB_ES_compatibility, that is now "glslparser_arb_compat". I haven't
> added metaprofiles for these cases, although we certainly could (or you
> can write your own, the schema is dead simple), so `./piglit run quick
> out --process-isolation=false` is now `./piglit run quick_gl glslparser
> quick_shader_multi out`.
> 
> I've run this through our CI extensively, and gotten green results out
> of it across the board.
> 
> I know this is a big series, but piglit makes a lot of assumptions about the
> test profiles being created at runtime, and we've had to changes those
> assumptions.
> 
> 
> Dylan Baker (35):
>   update git ignore for this series
>   test/piglit_test: add ROOT_DIR variable
>   framework/profile: Allow a group manager class to be overwritten
>   framework/test: Use getter for altering PiglitBaseTest Command
>   framework/test: expose required and excluded platforms
>   framework/profile: Add a __len__ method to TestProfile
>   framework: Use custom class for ASM parser tests
>   framework/test: add a test class for bu

Re: [Piglit] [PATCH] unittests: mustpass is a text list

[Dylan Baker]

Quoting Juan A. Suarez Romero (2018-04-19 00:44:04)
> Commit 7cf9e743a updated deqp mustpass lists to use text based mustpass.
> 
> Hence unit tests should use text based mustpass too.
> 
> Fixes: 7cf9e743a ("framework: update deqp mustpass list for text based
> mustpass")
> ---
>  unittests/framework/test/test_deqp.py | 28 +---
>  1 file changed, 9 insertions(+), 19 deletions(-)
> 
> diff --git a/unittests/framework/test/test_deqp.py 
> b/unittests/framework/test/test_deqp.py
> index 1eb84b13f..1c24ab365 100644
> --- a/unittests/framework/test/test_deqp.py
> +++ b/unittests/framework/test/test_deqp.py
> @@ -299,28 +299,18 @@ class TestDEQPBaseTest(object):
>  class TestGenMustpassTests(object):
>  """Tests for the gen_mustpass_tests function."""
>  
> -_xml = textwrap.dedent("""\
> -
> - appPackageName="com.freedesktop.org.piglit.deqp" testType="deqpTest" 
> xmlns:deqp="http://drawelements.com/deqp; deqp:glesVersion="196608">
> -
> -
> -
> -
> -
> -
> -
> -
> -
> -
> -
> -
> -
> -""")
> +_txt = """\
> +dEQP.piglit.group1.test1
> +dEQP.piglit.group1.test2
> +dEQP.piglit.nested.group2.test3
> +dEQP.piglit.nested.group2.test4
> +"""
>  
>  def test_basic(self, tmpdir):
> -p = tmpdir.join('foo.xml')
> -p.write(self._xml)
> +p = tmpdir.join('foo.txt')
> +p.write(self._txt)
>  tests = set(deqp.gen_mustpass_tests(six.text_type(p)))
> +print(tests)
>  assert tests == {
>  'dEQP.piglit.group1.test1',
>  'dEQP.piglit.group1.test2',
> -- 
> 2.14.3
> 

Oops! Thanks for fixing this.

Reviewed-by: Dylan Baker <dy...@pnwbakers.com>


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

Quoting Dylan Baker (2018-04-04 15:26:48)
> I don't expect everyone I've CC'd to give thorough review (or any
> review), I've mostly CC'd people who I think would be interested in this
> work, or who's work flow I might be altered by it.
> 
> Piglit has struggled to cope with the growing number of tests that it
> contains, especially with startup time. Piglit has always calculated
> tests at runtime, which was not a problem when there were only a few
> hundred or even thousand tests. Piglit now has roughly 55,000
> OpenGL/OpenGL ES tests, which is a lot to calculate at start up. It also
> means that piglit needs to keep a python object for each of those tests
> in memory, which has sent the resident memory usage soaring. We've also
> moved to automatic test discovery for glslparser, asmparser, and shader
> tests, which is very convenient and reduces typing, but further
> increases the amount of time spent starting up. This has even made
> features which decrease runtime, like fast skipping, hurt startup
> performance, making it a less than desirable tradeoff in some cases.
> Even on a relatively fast machine with an nvme disk 15-20 seconds is not
> an unheard of startup time. That might be okay to run 55,000 tests, but
> not if you only need a dozen, such as when bisecting.
> 
> This series is my proposal to fix that, mainly by moving much of that
> cost to build time. This series creates the infrastructure build XML
> base profiles at build time, which are installed with piglit instead of
> the python profiles. These profiles are lazily iterated over to ease
> memory usage, test objects are created as they are run, and python can
> garbage collect them as soon as they are done running. Along with that
> any filters applied to profiles (like removing 80% of the vs_in shader
> tests in quick) are done before the profile is serialized, and all fast
> skipping information is collected at build time as well, and encoded in
> the XML. All this means that start times are vastly reduced.
> 
> For example:
> XML profiles
> quick: 0.5
> shader: 0.5
> 
> master
> quick: 11.6
> shader: 7.3
> 
> This series also implements some optimizations for running without
> filters or test-lists, if you add a filter quick would take 2.5
> seconds, because that is necessary to calculate the total number of
> tests before starting.
> 
> To keep classic profiles like all, quick, quick_cl, gpu, cpu, and
> llvmpipe working this series adds meta profiles, small XML snippets that
> list other profiles. These can contain other meta profiles, xml
> profiles, or python profiles. This means that for most uses cases your
> existing command line will still work, `./piglit run quick out -c` will
> still do exactly the same thing as before, just faster.
> 
> The XML generated is dumb, there is no encoding of options or logic. An
> early version of this series did contain logic and options, but the
> result was pretty terrible. It was very hard to read, and the code to
> handle it was very complicated. I've chosen not to go down that path.
> There are drawbacks, some things that relied on run time generation have
> cannot be handled the same way, among them the "multi shader" concept,
> where shader_runner consumes a directory of shader_tests at a time. This
> was previously handled via a --process-isolation=false flag, now its
> encoded into profiles, "shader_multi" and "quick_shader_multi"; there
> was also an option to use glslparsertest with ES shaders and
> ARB_ES_compatibility, that is now "glslparser_arb_compat". I haven't
> added metaprofiles for these cases, although we certainly could (or you
> can write your own, the schema is dead simple), so `./piglit run quick
> out --process-isolation=false` is now `./piglit run quick_gl glslparser
> quick_shader_multi out`.
> 
> I've run this through our CI extensively, and gotten green results out
> of it across the board.
> 
> I know this is a big series, but piglit makes a lot of assumptions about the
> test profiles being created at runtime, and we've had to changes those
> assumptions.
> 
> 
> Dylan Baker (35):
>   update git ignore for this series
>   test/piglit_test: add ROOT_DIR variable
>   framework/profile: Allow a group manager class to be overwritten
>   framework/test: Use getter for altering PiglitBaseTest Command
>   framework/test: expose required and excluded platforms
>   framework/profile: Add a __len__ method to TestProfile
>   framework: Use custom class for ASM parser tests
>   framework/test: add a test class for built-in constants
>   tests: use BuiltInConstantsClass
>   framework: use a class method for building test via parsing
>   fram

Re: [Piglit] [PATCH] framework: update deqp mustpass list for text based mustpass

[Dylan Baker]

Quoting Dylan Baker (2018-04-11 13:39:26)
> Quoting Eric Anholt (2018-04-11 12:09:34)
> > Dylan Baker <dy...@pnwbakers.com> writes:
> > 
> > > The current code has bit-rotted quite a bit, in late 2016 the format was
> > > changed from XML based to txt based, but the code was never updated.
> > > ---
> > 
> > I think you can remove the elementtree import now.
> > 
> > Other than that,
> > 
> > Reviewed-by: Eric Anholt <e...@anholt.net>
> > 
> > (though deqp seems much less important to me than VK-GL-CTS, given that
> > you need to be running from VK-GL-CTS for conformance submissions as far
> > as I understand)
> 
> Thanks!
> 
> This should fix all of the dEQP based suites, including the VK-GL-CTS.
> 
> Dylan

And I somehow rebased your r-b out of the patch when applying it to master,
but didn't notice until after I pushed. Sigh.

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] framework: update deqp mustpass list for text based mustpass

[Dylan Baker]

Quoting Dylan Baker (2018-04-11 13:39:26)
> Quoting Eric Anholt (2018-04-11 12:09:34)
> > Dylan Baker <dy...@pnwbakers.com> writes:
> > 
> > > The current code has bit-rotted quite a bit, in late 2016 the format was
> > > changed from XML based to txt based, but the code was never updated.
> > > ---
> > 
> > I think you can remove the elementtree import now.
> > 
> > Other than that,
> > 
> > Reviewed-by: Eric Anholt <e...@anholt.net>
> > 
> > (though deqp seems much less important to me than VK-GL-CTS, given that
> > you need to be running from VK-GL-CTS for conformance submissions as far
> > as I understand)
> 
> Thanks!
> 
> This should fix all of the dEQP based suites, including the VK-GL-CTS.

I say that, but actually we'd need to change some code to support mustpass lists
in some of those profiles.

> Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] framework: update deqp mustpass list for text based mustpass

[Dylan Baker]

Quoting Eric Anholt (2018-04-11 12:09:34)
> Dylan Baker <dy...@pnwbakers.com> writes:
> 
> > The current code has bit-rotted quite a bit, in late 2016 the format was
> > changed from XML based to txt based, but the code was never updated.
> > ---
> 
> I think you can remove the elementtree import now.
> 
> Other than that,
> 
> Reviewed-by: Eric Anholt <e...@anholt.net>
> 
> (though deqp seems much less important to me than VK-GL-CTS, given that
> you need to be running from VK-GL-CTS for conformance submissions as far
> as I understand)

Thanks!

This should fix all of the dEQP based suites, including the VK-GL-CTS.

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH] framework: update deqp mustpass list for text based mustpass

[Dylan Baker]

The current code has bit-rotted quite a bit, in late 2016 the format was
changed from XML based to txt based, but the code was never updated.
---
 framework/test/deqp.py | 22 ++
 piglit.conf.example|  6 +++---
 2 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/framework/test/deqp.py b/framework/test/deqp.py
index 871ce2545..e63ab9368 100644
--- a/framework/test/deqp.py
+++ b/framework/test/deqp.py
@@ -90,23 +90,13 @@ def make_profile(test_list, test_class):
 return profile
 
 
-def gen_mustpass_tests(mp_list):
+def gen_mustpass_tests(mustpass):
 """Return a testlist from the mustpass list."""
-root = et.parse(mp_list).getroot()
-group = []
-
-def gen(base):
-for elem in base:
-if elem.tag == 'Test':
-yield '{}.{}'.format('.'.join(group), elem.get('name'))
-else:
-group.append(elem.get('name'))
-for test in gen(elem):
-yield test
-del group[-1]
-
-for test in gen(root):
-yield test
+with open(mustpass, 'r') as f:
+for l in f:
+l = l.strip()
+if l:
+yield l
 
 
 def gen_caselist_txt(bin_, caselist, extra_args):
diff --git a/piglit.conf.example b/piglit.conf.example
index d592308b8..5d1ff5474 100644
--- a/piglit.conf.example
+++ b/piglit.conf.example
@@ -60,7 +60,7 @@ testB
 ; Path to the test case list of CTS for deqp-gles2. You can also set this with
 ; the environment variable PIGLIT_DEQP2_MUSTPASS. Piglit will run the subset of
 ; dEQP-GLES2 tests if this option is set.
-;mustpasslist=/home/knuth/src/deqp/android/cts/com.drawelements.deqp.gles2.xml
+;mustpasslist=/home/knuth/src/deqp/android/cts/master/gles2-master.txt
 
 [deqp-gles3]
 ;
@@ -77,7 +77,7 @@ testB
 ; Path to the test case list of CTS for deqp-gles3. You can also set this with
 ; the environment variable PIGLIT_DEQP3_MUSTPASS. Piglit will run the subset of
 ; dEQP-GLES3 tests if this option is set.
-;mustpasslist=/home/knuth/src/deqp/android/cts/com.drawelements.deqp.gles3.xml
+;mustpasslist=/home/knuth/src/deqp/android/cts/master/gles3-master.txt
 
 [deqp-gles31]
 ; Path to the deqp-gles31 executable
@@ -92,7 +92,7 @@ testB
 ; Path to the test case list of CTS for deqp-gles31. You can also set this with
 ; the environment variable PIGLIT_DEQP31_MUSTPASS. Piglit will run the subset 
of
 ; dEQP-GLES31 tests if this option is set.
-;mustpasslist=/home/knuth/src/deqp/android/cts/com.drawelements.deqp.gles31.xml
+;mustpasslist=/home/knuth/src/deqp/android/cts/master/gles31-master.txt
 
 [deqp-vk]
 ; Path to the deqp-vk executable
-- 
2.17.0

___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

Quoting Mark Janes (2018-04-10 18:54:46)
> Dylan Baker <dy...@pnwbakers.com> writes:
> 
> > Quoting Marek Olšák (2018-04-10 14:22:10)
> >> On Tue, Apr 10, 2018 at 2:15 PM, Dylan Baker <dy...@pnwbakers.com> wrote:
> >> 
> >> Quoting Eric Anholt (2018-04-09 17:10:35)
> >> > Marek Olšák <mar...@gmail.com> writes:
> >> >
> >> > > Is this use case affected?
> >> > >
> >> > > piglit run --deqp-mustpass-list --process-isolation 0 -p gbm -c 
> >> quick
> >> > > cts_gl45 deqp_gles2 deqp_gles3 deqp_gles31
> >> > >
> >> > > Yes, that is just 1 command to run all those test suites at the 
> >> same
> >> time.
> >> > >
> >> > > I use my personal "deqp" piglit branch that also disables process
> >> isolation
> >> > > for glcts and deqp, and parses the deqp mustpass lists which are 
> >> in txt
> >> > > files.
> >> >
> >> > Parsing the mustpass lists sounds really useful.  Trying to 
> >> construct an
> >> > appropriate command line otherwise has been quite a challenge.
> >> 
> >> That option is already in core piglit, you just need to configure your
> >> piglit.conf appropriately. I guess we really should have that in the
> >> piglit.conf.example file...
> >> 
> >> 
> >> I doubt it. Why would I have these then:
> >> 
> >> https://cgit.freedesktop.org/~mareko/piglit/commit/?h=deqp=
> >> 0b11344e0c18b9bf07ad12381b94f308f362eb88
> >> https://cgit.freedesktop.org/~mareko/piglit/commit/?h=deqp=
> >> b086d8f82d41338055ab48bdda78c4a0c1ee02d0
> >> https://cgit.freedesktop.org/~mareko/piglit/commit/?h=deqp=
> >> 90beefa825cda792eaa72bff2cefac463af6d08a
> >> 
> >> Marek
> >
> > Because in late 2016 (there is some internal stuff that happened so we 
> > stopped
> > using piglit to run deqp I'm not entirely happy about) they changed the 
> > mustpass
> > list from xml to txt and no one ever updated it in master.
> 
> I'm not sure what you are referring to here.  We stopped using piglit
> for dEQP because the run-time was 10X faster with a custom runner.  We
> don't have process isolation anymore, but we recover from
> crashes/assertions without paying the penalty of iterating the dEQP test
> list for each test run.
> 
> Google dEQP authors recommended this path.
> 
> > Dylan

I was concerned that we would end up re-implementing so much of piglit that we
would invest way more time in it than fixing piglit, and not have something as
well tested or as widely deployed. Anyway, that's in the past at this point.

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

Quoting Marek Olšák (2018-04-10 14:22:10)
> On Tue, Apr 10, 2018 at 2:15 PM, Dylan Baker <dy...@pnwbakers.com> wrote:
> 
> Quoting Eric Anholt (2018-04-09 17:10:35)
> > Marek Olšák <mar...@gmail.com> writes:
> >
> > > Is this use case affected?
> > >
> > > piglit run --deqp-mustpass-list --process-isolation 0 -p gbm -c quick
> > > cts_gl45 deqp_gles2 deqp_gles3 deqp_gles31
> > >
> > > Yes, that is just 1 command to run all those test suites at the same
> time.
> > >
> > > I use my personal "deqp" piglit branch that also disables process
> isolation
> > > for glcts and deqp, and parses the deqp mustpass lists which are in 
> txt
> > > files.
> >
> > Parsing the mustpass lists sounds really useful.  Trying to construct an
> > appropriate command line otherwise has been quite a challenge.
> 
> That option is already in core piglit, you just need to configure your
> piglit.conf appropriately. I guess we really should have that in the
> piglit.conf.example file...
> 
> 
> I doubt it. Why would I have these then:
> 
> https://cgit.freedesktop.org/~mareko/piglit/commit/?h=deqp=
> 0b11344e0c18b9bf07ad12381b94f308f362eb88
> https://cgit.freedesktop.org/~mareko/piglit/commit/?h=deqp=
> b086d8f82d41338055ab48bdda78c4a0c1ee02d0
> https://cgit.freedesktop.org/~mareko/piglit/commit/?h=deqp=
> 90beefa825cda792eaa72bff2cefac463af6d08a
> 
> Marek

Because in late 2016 (there is some internal stuff that happened so we stopped
using piglit to run deqp I'm not entirely happy about) they changed the mustpass
list from xml to txt and no one ever updated it in master.

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

Quoting Eric Anholt (2018-04-09 17:10:35)
> Marek Olšák  writes:
> 
> > Is this use case affected?
> >
> > piglit run --deqp-mustpass-list --process-isolation 0 -p gbm -c quick
> > cts_gl45 deqp_gles2 deqp_gles3 deqp_gles31
> >
> > Yes, that is just 1 command to run all those test suites at the same time.
> >
> > I use my personal "deqp" piglit branch that also disables process isolation
> > for glcts and deqp, and parses the deqp mustpass lists which are in txt
> > files.
> 
> Parsing the mustpass lists sounds really useful.  Trying to construct an
> appropriate command line otherwise has been quite a challenge.

That option is already in core piglit, you just need to configure your
piglit.conf appropriately. I guess we really should have that in the
piglit.conf.example file...

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 0/3] tests: intel blackhole render fixes

[Dylan Baker]

for the series:
Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

Quoting Lionel Landwerlin (2018-04-06 03:25:13)
> Hi,
> 
> I've made a few mistake when sending the initial patch. I'm not quite
> sure how I missed them :(
> 
> Thanks,
> 
> Lionel Landwerlin (3):
>   tests: fix INTEL_blackhole_render test names
>   tests: intel_blackhole_render: require extension in dispatch test
>   tests: nv_image_formats: remove unintended new files
> 
>  tests/all.py   | 8 
>  tests/spec/intel_blackhole_render/blackhole_dispatch.c | 1 +
>  tests/spec/nv_image_formats/CMakeLists.gl.txt  | 8 
>  3 files changed, 5 insertions(+), 12 deletions(-)
>  delete mode 100644 tests/spec/nv_image_formats/CMakeLists.gl.txt
> 
> --
> 2.17.0
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] vkrunner: Add the script used to generate vr-format-table.h

[Dylan Baker]

Thanks for making the changes I requested!

Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

Quoting Neil Roberts (2018-04-06 05:14:20)
> Note that the script is not run automatically as part of the build
> process and instead vr-format-table.h is checked into git.
> ---
> 
> I’ve split the script out to make it easier to review.
> 
> Thanks for the feedback Dylan.
> 
>  tests/vulkan/vkrunner/make-formats.py | 137 
> ++
>  1 file changed, 137 insertions(+)
>  create mode 100755 tests/vulkan/vkrunner/make-formats.py
> 
> diff --git a/tests/vulkan/vkrunner/make-formats.py 
> b/tests/vulkan/vkrunner/make-formats.py
> new file mode 100755
> index 0..01e9000de
> --- /dev/null
> +++ b/tests/vulkan/vkrunner/make-formats.py
> @@ -0,0 +1,137 @@
> +#!/usr/bin/env python
> +
> +# Copyright (C) 2018 Intel Corporation
> +
> +# Permission is hereby granted, free of charge, to any person obtaining a
> +# copy of this software and associated documentation files (the "Software"),
> +# to deal in the Software without restriction, including without limitation
> +# the rights to use, copy, modify, merge, publish, distribute, sublicense,
> +# and/or sell copies of the Software, and to permit persons to whom the
> +# Software is furnished to do so, subject to the following conditions:
> +
> +# The above copyright notice and this permission notice (including the next
> +# paragraph) shall be included in all copies or substantial portions of the
> +# Software.
> +
> +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
> +# DEALINGS IN THE SOFTWARE.
> +
> +from __future__ import (
> +absolute_import, division, print_function, unicode_literals
> +)
> +
> +# This script is used to generate vr-format-table.h from vulkan.h. It
> +# is not run automatically as part of the build process but if need be
> +# it can be used to update the file as follows:
> +#
> +# ./make-formats.py < /usr/include/vulkan/vulkan.h > vr-format-table.h
> +
> +import re
> +import sys
> +from mako.template import Template
> +
> +FORMAT_RE = re.compile(r'\bVK_FORMAT_([A-Z0-9_]+)\b')
> +SKIP_RE = re.compile(r'(?:_BLOCK(?:_IMG)?|_KHR|^UNDEFINED|'
> + r'^RANGE_SIZE|^MAX_ENUM|_RANGE)$')
> +COMPONENT_RE = re.compile('([A-Z]+)([0-9]+)')
> +PACK_RE = re.compile('PACK([0-9]+)$')
> +
> +SWIZZLES = ['RGBA', 'BGRA', 'ARGB', 'ABGR']
> +
> +TEMPLATE="""\
> +/* Automatically generated by make-formats.py */
> +static const struct vr_format
> +formats[] = {
> +% for format in formats:
> +   {
> +   .vk_format = VK_FORMAT_${format['name']},
> +   .name = "${format['name']}",
> +   .packed_size = ${format['packed_size']},
> +   .swizzle = VR_FORMAT_SWIZZLE_${format['swizzle']},
> +   .mode = VR_FORMAT_MODE_${format['mode']},
> +   .n_components = ${len(format['components'])},
> +   .components = {
> +   % for letter, size in format['components']:
> +   { .bits = ${size} },
> +   % endfor
> +   }
> +   },
> +% endfor
> +};"""
> +
> +
> +def get_format_names(data):
> +in_enum = False
> +
> +for line in data:
> +if line.startswith('typedef enum VkFormat '):
> +in_enum = True
> +elif line.startswith('}'):
> +in_enum = False
> +if not in_enum:
> +continue
> +
> +md = FORMAT_RE.search(line)
> +if md is None:
> +continue
> +name = md.group(1)
> +if SKIP_RE.search(name):
> +continue
> +yield name
> +
> +
> +def get_formats(data):
> +for name in sorted(set(get_format_names(data))):
> +parts = name.split('_')
> +
> +components = get_components(parts[0])
> +
> +if components is None:
> +continue
> +
> +swizzle = get_swizzle(components)
> +
> +if len(parts) >= 3:
> +md = PACK_RE.match(parts[2])
> +packed_size = int(md.group(1))
> +else:
> +packed_size = 0
> +
> +yield {'name': name,
> 

Re: [Piglit] [PATCH v2] framework: Add a vulkan tests profile

[Dylan Baker]

Looks good, thank you!

Quoting Neil Roberts (2018-04-06 05:19:57)
> This searches for files named *.vk_shader_test in the tests/vulkan
> directory and runs them with VkRunner.
> 
> Reviewed-by: Dylan Baker <dy...@pnwbakers.com>
> 
> v2: Move VkShaderTest to piglit_test.py and rename to VkRunnerTest.
> Add future imports. Remove unused import.
> ---
>  framework/test/piglit_test.py | 10 ++
>  tests/vulkan.py   | 33 +
>  2 files changed, 43 insertions(+)
>  create mode 100644 tests/vulkan.py
> 
> diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
> index b6cec71cd..d7767cb8b 100644
> --- a/framework/test/piglit_test.py
> +++ b/framework/test/piglit_test.py
> @@ -44,6 +44,7 @@ __all__ = [
>  'PiglitCLTest',
>  'PiglitGLTest',
>  'PiglitBaseTest',
> +'VkRunnerTest',
>  'CL_CONCURRENT',
>  'TEST_BIN_DIR',
>  ]
> @@ -184,3 +185,12 @@ class PiglitCLTest(PiglitBaseTest):  # pylint: 
> disable=too-few-public-methods
>  """
>  def __init__(self, command, run_concurrent=CL_CONCURRENT, **kwargs):
>  super(PiglitCLTest, self).__init__(command, run_concurrent, **kwargs)
> +
> +
> +class VkRunnerTest(PiglitBaseTest):
> +""" Make a PiglitTest instance for a VkRunner shader test file """
> +
> +def __init__(self, filename):
> +super(VkRunnerTest, self).__init__(
> +['vkrunner', filename],
> +run_concurrent=True)
> diff --git a/tests/vulkan.py b/tests/vulkan.py
> new file mode 100644
> index 0..7058f3108
> --- /dev/null
> +++ b/tests/vulkan.py
> @@ -0,0 +1,33 @@
> +#!/usr/bin/env python
> +# -*- coding: utf-8 -*-
> +"""All Vulkan tests that come with piglit, using default settings."""
> +
> +from __future__ import (
> +absolute_import, division, print_function, unicode_literals
> +)
> +
> +import os
> +
> +from framework.profile import TestProfile
> +from framework import grouptools
> +from framework.test.piglit_test import VkRunnerTest
> +from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
> +
> +__all__ = ['profile']
> +
> +profile = TestProfile()
> +
> +# Find and add all shader tests.
> +for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
> +_basedir = os.path.join(basedir, 'vulkan')
> +for dirpath, _, filenames in os.walk(_basedir):
> +groupname = grouptools.from_path(os.path.relpath(dirpath, _basedir))
> +for filename in filenames:
> +testname, ext = os.path.splitext(filename)
> +if ext != '.vk_shader_test':
> +continue
> +test = VkRunnerTest(os.path.join(dirpath, filename))
> +group = grouptools.join(groupname, testname)
> +assert group not in profile.test_list, group
> +
> +profile.test_list[group] = test
> -- 
> 2.14.3
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH v2 34/35] Generate xml for builtin profiles

[Dylan Baker]

This creates and installs xml for all builtin profiles. Using these
profiles I see startup times of ~1 second as opposed to more than 10
with current master, and runtimes that drop by ~1 minute.

v2: - Fix compilation with make. v1 only worked with ninja.
---
 CMakeLists.txt  |  8 +++-
 tests/CMakeLists.no_api.txt | 89 +
 tests/CMakeLists.txt|  2 +
 3 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100644 tests/CMakeLists.no_api.txt

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dc80ece9a..db047b4b2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -534,7 +534,13 @@ install (
DIRECTORY tests
DESTINATION ${PIGLIT_INSTALL_LIBDIR}
FILES_MATCHING REGEX 
".*\\.(xml|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
-   REGEX "CMakeFiles|CMakeLists" EXCLUDE
+   REGEX 
"CMakeFiles|CMakeLists|serializer.py|opengl.py|cl.py|quick_gl.py|glslparser.py|shader.py|quick_shader.py|no_error.py|llvmpipe_gl.py|sanity.py"
 EXCLUDE
+)
+
+install (
+   DIRECTORY ${CMAKE_BINARY_DIR}/tests
+   DESTINATION ${PIGLIT_INSTALL_LIBDIR}
+   FILES_MATCHING REGEX ".*\\.xml"
 )
 
 install (
diff --git a/tests/CMakeLists.no_api.txt b/tests/CMakeLists.no_api.txt
new file mode 100644
index 0..3004d09e6
--- /dev/null
+++ b/tests/CMakeLists.no_api.txt
@@ -0,0 +1,89 @@
+# Copyright © 2018 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+add_custom_target(
+   static-shader-tests
+   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/shader.list
+   COMMAND ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/find_static_tests.py ${CMAKE_CURRENT_SOURCE_DIR} 
shader ${CMAKE_CURRENT_BINARY_DIR}/shader.list
+   VERBATIM
+)
+add_custom_target(
+   static-glslparser-tests
+   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/glslparser.list
+   COMMAND ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/find_static_tests.py ${CMAKE_CURRENT_SOURCE_DIR} 
glslparser ${CMAKE_CURRENT_BINARY_DIR}/glslparser.list
+   VERBATIM
+)
+add_custom_target(
+   static-asmparser-tests
+   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/asmparser.list
+   COMMAND ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/find_static_tests.py ${CMAKE_CURRENT_SOURCE_DIR} 
asmparser ${CMAKE_CURRENT_BINARY_DIR}/asmparser.list
+   VERBATIM
+)
+add_custom_target(
+   static-program-tests
+   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/program.list
+   COMMAND ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/find_static_tests.py ${CMAKE_CURRENT_SOURCE_DIR} 
program ${CMAKE_CURRENT_BINARY_DIR}/program.list
+   VERBATIM
+)
+
+function(piglit_generate_xml name profile meta_target extra_args)
+   add_custom_command(
+   OUTPUT ${CMAKE_BINARY_DIR}/tests/${name}.xml
+   COMMAND ${CMAKE_COMMAND} -E env 
PIGLIT_BUILD_TREE=${CMAKE_BINARY_DIR} ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${name} 
${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py ${CMAKE_BINARY_DIR}/tests/${name}.xml 
 ${extra_args}
+   DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${ARGN}
+   VERBATIM
+   )
+   add_custom_target(
+   generate-${name}-xml
+   DEPENDS ${CMAKE_BINARY_DIR}/tests/${name}.xml
+   )
+   add_dependencies(${meta_target} generate-${name}-xml)
+endfunction()
+
+add_custom_target(gen-gl-xml)
+piglit_generate_xml(opengl opengl gen-gl-xml "")
+piglit_generate_xml(no_error no_error gen-gl-xml "")
+piglit_generate_xml(quick_gl quick_gl gen-gl-xml "")
+piglit_generate_xml(llvmpipe_gl llvmpipe_gl gen-gl-xml "")
+piglit_generate_xml(sanity sanity gen-gl-xml "")
+
+add_custom_target(gen-gl-gen-xml)
+piglit_generate_xml(glslparser glslparser gen-gl-gen-xml "" gen-gl-tests 
static-glslparser-tests static-asmparser-tests)
+piglit_generate_xml(glslparser_arb_compat 

Re: [Piglit] [PATCH 7/8] framework: Add a vulkan tests profile

[Dylan Baker]

Quoting Neil Roberts (2018-04-04 15:56:54)
> This searches for files named *.vk_shader_test in the tests/vulkan
> directory and runs them with VkRunner.
> ---
>  framework/test/vk_shader_test.py | 39 +++
>  tests/vulkan.py  | 30 ++
>  2 files changed, 69 insertions(+)
>  create mode 100644 framework/test/vk_shader_test.py
>  create mode 100644 tests/vulkan.py
> 
> diff --git a/framework/test/vk_shader_test.py 
> b/framework/test/vk_shader_test.py
> new file mode 100644
> index 0..e6eab0e15
> --- /dev/null
> +++ b/framework/test/vk_shader_test.py
> @@ -0,0 +1,39 @@
> +# Copyright (C) 2018 Intel Corporation
> +#
> +# Permission is hereby granted, free of charge, to any person
> +# obtaining a copy of this software and associated documentation
> +# files (the "Software"), to deal in the Software without
> +# restriction, including without limitation the rights to use,
> +# copy, modify, merge, publish, distribute, sublicense, and/or
> +# sell copies of the Software, and to permit persons to whom the
> +# Software is furnished to do so, subject to the following
> +# conditions:
> +#
> +# This permission notice shall be included in all copies or
> +# substantial portions of the Software.
> +#
> +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
> +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
> +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
> +# PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHOR(S) BE
> +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
> +# AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF
> +# OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
> +# DEALINGS IN THE SOFTWARE.
> +
> +""" This module enables running VkRunner shader tests. """
> +

future imports please

> +from .piglit_test import PiglitBaseTest
> +
> +__all__ = [
> +'VkShaderTest',
> +]
> +
> +
> +class VkShaderTest(PiglitBaseTest):
> +""" Make a PiglitTest instance for a VkRunner shader test file """
> +
> +def __init__(self, filename):
> +super(VkShaderTest, self).__init__(
> +['vkrunner', filename],
> +run_concurrent=True)

With how minimal this is you might just put in in piglit_test, shader and glsl
parser are split out for historical reasons and because the parsers are
gigantic. But I don't care either way.

> diff --git a/tests/vulkan.py b/tests/vulkan.py
> new file mode 100644
> index 0..e9c48f6a4
> --- /dev/null
> +++ b/tests/vulkan.py
> @@ -0,0 +1,30 @@
> +#!/usr/bin/env python
> +# -*- coding: utf-8 -*-
> +"""All Vulkan tests that come with piglit, using default settings."""

future imports here as well.

> +
> +import os
> +
> +from framework.profile import TestProfile
> +from framework.test import PiglitCLTest

unused import

otherwise this is:
Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

> +from framework import grouptools
> +from framework.test.vk_shader_test import VkShaderTest
> +from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
> +
> +__all__ = ['profile']
> +
> +profile = TestProfile()
> +
> +# Find and add all shader tests.
> +for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
> +_basedir = os.path.join(basedir, 'vulkan')
> +for dirpath, _, filenames in os.walk(_basedir):
> +groupname = grouptools.from_path(os.path.relpath(dirpath, _basedir))
> +for filename in filenames:
> +testname, ext = os.path.splitext(filename)
> +if ext != '.vk_shader_test':
> +continue
> +test = VkShaderTest(os.path.join(dirpath, filename))
> +group = grouptools.join(groupname, testname)
> +assert group not in profile.test_list, group
> +
> +profile.test_list[group] = test
> -- 
> 2.14.3
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 6/8] Import VkRunner

[Dylan Baker]

Quoting Neil Roberts (2018-04-04 15:56:53)
[snip]
> diff --git a/tests/vulkan/vkrunner/make-formats.py 
> b/tests/vulkan/vkrunner/make-formats.py
> new file mode 100755
> index 0..6bb1030c0
> --- /dev/null
> +++ b/tests/vulkan/vkrunner/make-formats.py
> @@ -0,0 +1,93 @@
> +#!/usr/bin/python3

Unfortunately python scripts in piglit need to be python 2/3 compatible, I've
also got a few style nits.

Please use all of the future imports like the other scripts do

> +
> +import re
> +import sys
> +
> +format_re = re.compile(r'\bVK_FORMAT_([A-Z0-9_]+)\b')
> +skip_re = re.compile(r'(?:_BLOCK(?:_IMG)?|_KHR|^UNDEFINED|'
> + r'^RANGE_SIZE|^MAX_ENUM|_RANGE)$')
> +component_re = re.compile('([A-Z]+)([0-9]+)')
> +pack_re = re.compile('PACK([0-9]+)$')

module constants should be ALL_CAPS

> +
> +swizzles = [ 'RGBA', 'BGRA', 'ARGB', 'ABGR' ]

no spaces between [ and '

two newlines between toplevel classes and functions

> +
> +def get_formats(data):
> +in_enum = False
> +
> +for line in data:
> +if line.startswith('typedef enum VkFormat '):
> +in_enum = True
> +elif line.startswith('}'):
> +in_enum = False
> +if not in_enum:
> +continue
> +
> +md = format_re.search(line)
> +if md is None:
> +continue
> +name = md.group(1)
> +if skip_re.search(name):
> +continue
> +yield(name)

no parens around returns or yields uless they're returning a tuple please.

> +
> +def get_components(name):
> +components = [(md.group(1), int(md.group(2)))
> +  for md in component_re.finditer(name)]
> +for letter, size in components:
> +if "RGBA".find(letter) == -1:

if letter not in 'RGBA'

> +return None
> +return components
> +
> +def get_swizzle(components):
> +component_letters = "".join((letter for letter, size in components))
> +for swizzle in swizzles:
> +if swizzle.startswith(component_letters):
> +return swizzle
> +print("Unknown swizzle {}".format(component_letters),
> +  file=sys.stderr)
> +sys.exit(1)
> +
> +formats = sorted(set(get_formats(sys.stdin)))
> +
> +print("""/* Automatically generated by make-formats.py */
> +static const struct vr_format
> +formats[] = {""")

mako is already a requirement for piglit, can we use it instead of printing a
file? It's much easier to understand and edit after the fact.

Also please use a main function and the if __name__ == "__main__" idiom.

Dylan

> +
> +for name in formats:
> +parts = name.split('_')
> +
> +components = get_components(parts[0])
> +
> +if components is None:
> +continue
> +
> +swizzle = get_swizzle(components)
> +
> +if len(parts) >= 3:
> +md = pack_re.match(parts[2])
> +packed_size = int(md.group(1))
> +else:
> +packed_size = 0
> +
> +print("""  {{
> +   .vk_format = VK_FORMAT_{},
> +   .name = "{}",
> +   .packed_size = {},
> +   .swizzle = VR_FORMAT_SWIZZLE_{},
> +   .mode = VR_FORMAT_MODE_{},
> +   .n_components = {},
> +   .components = {{""".format(
> +name,
> +name,
> +packed_size,
> +swizzle,
> +parts[1],
> +len(components)))
> +
> +for letter, size in components:
> +print("\t\t\t{{ .bits = {} }},".format(size))
> +
> +print("""  }
> +   },""")
> +
> +print("};")

[snip]


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

Quoting Marek Olšák (2018-04-04 16:37:47)
> Is this use case affected?
> 
> piglit run --deqp-mustpass-list --process-isolation 0 -p gbm -c quick cts_gl45
> deqp_gles2 deqp_gles3 deqp_gles31

You could either run that as:

piglit run --deqp-mustpass-list --process-isolation 0 -p gbm -c quick_gl
quick_shader_multi glslparser cts_gl45 deqp_gles2 deqp_gles3 deqp_gles31

or you cold write yourself a metaprofile that would look something like:



  quick_gl
  quick_shader_multi
  glslparser
  cts
  gl45
  deqp
  ...


We could probably put metaprofiles for the multiple-shader at a time profiles
in piglit, I just didn't want to get too over board with them.

> 
> Yes, that is just 1 command to run all those test suites at the same time.
> 
> I use my personal "deqp" piglit branch that also disables process isolation 
> for
> glcts and deqp, and parses the deqp mustpass lists which are in txt files.
> 
> Marek

I have not touched the --process-isolation flag since I knew that you had that
branch (and I have a branch for dEQP as well that I might try to dust off and
get working again).

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH v2 35/35] profile: use gz to compress profiles

[Dylan Baker]

This results in substantially smaller profiles and doesn't seem to
affect runtime.

v2: - install xml and xml.gz files. This is needed so that meta profiles
  will be installed.
---
 CMakeLists.txt  |  4 ++--
 framework/profile.py| 10 +-
 tests/CMakeLists.no_api.txt |  6 +++---
 tests/serializer.py |  4 +++-
 4 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index db047b4b2..8c7d16270 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -533,14 +533,14 @@ install (
 install (
DIRECTORY tests
DESTINATION ${PIGLIT_INSTALL_LIBDIR}
-   FILES_MATCHING REGEX 
".*\\.(xml|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
+   FILES_MATCHING REGEX 
".*\\.(xml|xml.gz|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
REGEX 
"CMakeFiles|CMakeLists|serializer.py|opengl.py|cl.py|quick_gl.py|glslparser.py|shader.py|quick_shader.py|no_error.py|llvmpipe_gl.py|sanity.py"
 EXCLUDE
 )
 
 install (
DIRECTORY ${CMAKE_BINARY_DIR}/tests
DESTINATION ${PIGLIT_INSTALL_LIBDIR}
-   FILES_MATCHING REGEX ".*\\.xml"
+   FILES_MATCHING REGEX ".*\\.xml.gz"
 )
 
 install (
diff --git a/framework/profile.py b/framework/profile.py
index 3975d0606..1c75025b3 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -34,8 +34,8 @@ import ast
 import collections
 import contextlib
 import copy
+import gzip
 import importlib
-import io
 import itertools
 import multiprocessing
 import multiprocessing.dummy
@@ -318,7 +318,7 @@ class XMLProfile(object):
 
 def __len__(self):
 if not (self.filters or self.forced_test_list):
-with io.open(self.filename, 'rt') as f:
+with gzip.open(self.filename, 'rt') as f:
 iter_ = et.iterparse(f, events=(b'start', ))
 for _, elem in iter_:
 if elem.tag == 'PiglitTestList':
@@ -333,7 +333,7 @@ class XMLProfile(object):
 
 def _itertests(self):
 """Always iterates tests instead of using the forced test_list."""
-with io.open(self.filename, 'rt') as f:
+with gzip.open(self.filename, 'rt') as f:
 doc = et.iterparse(f, events=(b'end', ))
 _, root = next(doc)  # get the root so we can keep clearing it
 for _, e in doc:
@@ -517,13 +517,13 @@ def load_test_profile(filename, python=None):
 if os.path.exists(meta):
 return MetaProfile(meta)
 
-xml = os.path.join(ROOT_DIR, 'tests', name + '.xml')
+xml = os.path.join(ROOT_DIR, 'tests', name + '.xml.gz')
 if os.path.exists(xml):
 return XMLProfile(xml)
 
 if python is False:
 raise exceptions.PiglitFatalError(
-'Cannot open "tests/{0}.xml" or "tests/{0}.meta.xml"'.format(name))
+'Cannot open "tests/{0}.xml.gz" or 
"tests/{0}.meta.xml"'.format(name))
 
 try:
 mod = importlib.import_module('tests.{0}'.format(name))
diff --git a/tests/CMakeLists.no_api.txt b/tests/CMakeLists.no_api.txt
index d7f95af0e..3ffba74b0 100644
--- a/tests/CMakeLists.no_api.txt
+++ b/tests/CMakeLists.no_api.txt
@@ -45,14 +45,14 @@ add_custom_target(
 
 function(piglit_generate_xml name profile meta_target extra_args)
add_custom_command(
-   OUTPUT ${CMAKE_BINARY_DIR}/tests/${name}.xml
-   COMMAND ${CMAKE_COMMAND} -E env 
PIGLIT_BUILD_TREE=${CMAKE_BINARY_DIR} ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${name} 
${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py ${CMAKE_BINARY_DIR}/tests/${name}.xml 
 ${extra_args}
+   OUTPUT ${CMAKE_BINARY_DIR}/tests/${name}.xml.gz
+   COMMAND ${CMAKE_COMMAND} -E env 
PIGLIT_BUILD_TREE=${CMAKE_BINARY_DIR} ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${name} 
${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py 
${CMAKE_BINARY_DIR}/tests/${name}.xml.gz  ${extra_args}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${ARGN}
VERBATIM
)
add_custom_target(
generate-${name}-xml
-   DEPENDS ${CMAKE_BINARY_DIR}/tests/${name}.xml
+   DEPENDS ${CMAKE_BINARY_DIR}/tests/${name}.xml.gz
)
add_dependencies(${meta_target} generate-${name}-xml)
 endfunction()
diff --git a/tests/serializer.py b/tests/serializer.py
index 58b3a14e9..32beb428c 100644
--- a/tests/serializer.py
+++ b/tests/serializer.py
@@ -22,6 +22,7 @@
 """Script for taking profiles in python format and serializing them to XML."""
 
 import argparse
+import gzip
 import os
 import sys
 import xml.etree.cElementTree as et
@@ -133,7 +134,8 @@ def serializer(name, profile, outfile):
 et.SubElement(env, 'env', name=k, value=v)
 
 tree = et.ElementTree(root)
-tree.write(outfile, encoding='utf-8', xml_declaration=True)
+with 

Re: [Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

Quoting Dylan Baker (2018-04-04 15:26:48)
> I don't expect everyone I've CC'd to give thorough review (or any
> review), I've mostly CC'd people who I think would be interested in this
> work, or who's work flow I might be altered by it.
> 
> Piglit has struggled to cope with the growing number of tests that it
> contains, especially with startup time. Piglit has always calculated
> tests at runtime, which was not a problem when there were only a few
> hundred or even thousand tests. Piglit now has roughly 55,000
> OpenGL/OpenGL ES tests, which is a lot to calculate at start up. It also
> means that piglit needs to keep a python object for each of those tests
> in memory, which has sent the resident memory usage soaring. We've also
> moved to automatic test discovery for glslparser, asmparser, and shader
> tests, which is very convenient and reduces typing, but further
> increases the amount of time spent starting up. This has even made
> features which decrease runtime, like fast skipping, hurt startup
> performance, making it a less than desirable tradeoff in some cases.
> Even on a relatively fast machine with an nvme disk 15-20 seconds is not
> an unheard of startup time. That might be okay to run 55,000 tests, but
> not if you only need a dozen, such as when bisecting.
> 
> This series is my proposal to fix that, mainly by moving much of that
> cost to build time. This series creates the infrastructure build XML
> base profiles at build time, which are installed with piglit instead of
> the python profiles. These profiles are lazily iterated over to ease
> memory usage, test objects are created as they are run, and python can
> garbage collect them as soon as they are done running. Along with that
> any filters applied to profiles (like removing 80% of the vs_in shader
> tests in quick) are done before the profile is serialized, and all fast
> skipping information is collected at build time as well, and encoded in
> the XML. All this means that start times are vastly reduced.
> 
> For example:
> XML profiles
> quick: 0.5
> shader: 0.5
> 
> master
> quick: 11.6
> shader: 7.3
> 
> This series also implements some optimizations for running without
> filters or test-lists, if you add a filter quick would take 2.5
> seconds, because that is necessary to calculate the total number of
> tests before starting.
> 
> To keep classic profiles like all, quick, quick_cl, gpu, cpu, and
> llvmpipe working this series adds meta profiles, small XML snippets that
> list other profiles. These can contain other meta profiles, xml
> profiles, or python profiles. This means that for most uses cases your
> existing command line will still work, `./piglit run quick out -c` will
> still do exactly the same thing as before, just faster.
> 
> The XML generated is dumb, there is no encoding of options or logic. An
> early version of this series did contain logic and options, but the
> result was pretty terrible. It was very hard to read, and the code to
> handle it was very complicated. I've chosen not to go down that path.
> There are drawbacks, some things that relied on run time generation have
> cannot be handled the same way, among them the "multi shader" concept,
> where shader_runner consumes a directory of shader_tests at a time. This
> was previously handled via a --process-isolation=false flag, now its
> encoded into profiles, "shader_multi" and "quick_shader_multi"; there
> was also an option to use glslparsertest with ES shaders and
> ARB_ES_compatibility, that is now "glslparser_arb_compat". I haven't
> added metaprofiles for these cases, although we certainly could (or you
> can write your own, the schema is dead simple), so `./piglit run quick
> out --process-isolation=false` is now `./piglit run quick_gl glslparser
> quick_shader_multi out`.
> 
> I've run this through our CI extensively, and gotten green results out
> of it across the board.
> 
> I know this is a big series, but piglit makes a lot of assumptions about the
> test profiles being created at runtime, and we've had to changes those
> assumptions.
> 
> 
> Dylan Baker (35):
>   update git ignore for this series
>   test/piglit_test: add ROOT_DIR variable
>   framework/profile: Allow a group manager class to be overwritten
>   framework/test: Use getter for altering PiglitBaseTest Command
>   framework/test: expose required and excluded platforms
>   framework/profile: Add a __len__ method to TestProfile
>   framework: Use custom class for ASM parser tests
>   framework/test: add a test class for built-in constants
>   tests: use BuiltInConstantsClass
>   framework: use a class method for building test via parsing
>   fram

[Piglit] [PATCH 22/35] framework/profile: Add support for meta profiles

[Dylan Baker]

MetaProfiles are a single profile composed of more than one profile.
This is designed to allow the old all-inclusive names to continue to
exist without having to keep duplicates of each profile around. A
metaprofile simply declares that it needs profiles a, b, and c and then
runs those.

This allows an "all" meta profile to include opengl, shader, and
glslparser, for example.

A metaprofile is allowed to include XML or python based profiles.
---
 framework/profile.py | 64 +++--
 1 file changed, 62 insertions(+), 2 deletions(-)

diff --git a/framework/profile.py b/framework/profile.py
index 2395b22..3975d06 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -359,6 +359,62 @@ class XMLProfile(object):
 return iter(self._itertests())
 
 
+class MetaProfile(object):
+
+"""Holds multiple profiles but acts like one.
+
+This is meant to allow classic profiles like all to exist after being
+split.
+"""
+
+def __init__(self, filename):
+self.forced_test_list = []
+self.filters = []
+self.options = {
+'dmesg': get_dmesg(False),
+'monitor': Monitoring(False),
+'ignore_missing': False,
+}
+
+tree = et.parse(filename)
+root = tree.getroot()
+self._profiles = [load_test_profile(p.text)
+  for p in root.findall('.//Profile')]
+
+for p in self._profiles:
+p.options = self.options
+
+def __len__(self):
+if self.forced_test_list or self.filters:
+return sum(1 for _ in self.itertests())
+return sum(len(p) for p in self._profiles)
+
+def setup(self):
+pass
+
+def teardown(self):
+pass
+
+def _itertests(self):
+for p in self._profiles:
+for k, v in p.itertests():
+if all(f(k, v) for f in self.filters):
+yield k, v
+
+def itertests(self):
+if self.forced_test_list:
+alltests = dict(self._itertests())
+opts = collections.OrderedDict()
+for n in self.forced_test_list:
+if self.options['ignore_missing'] and n not in alltests:
+opts[n] = DummyTest(n, status.NOTRUN)
+else:
+opts[n] = alltests[n]
+return six.iteritems(opts)
+else:
+return iter(self._itertests())
+
+
 class TestProfile(object):
 """Class that holds a list of tests for execution.
 
@@ -456,14 +512,18 @@ def load_test_profile(filename, python=None):
   XML is tried.
 """
 name = os.path.splitext(os.path.basename(filename))[0]
-xml = os.path.join('tests', name + '.xml')
 if not python:
+meta = os.path.join(ROOT_DIR, 'tests', name + '.meta.xml')
+if os.path.exists(meta):
+return MetaProfile(meta)
+
 xml = os.path.join(ROOT_DIR, 'tests', name + '.xml')
 if os.path.exists(xml):
 return XMLProfile(xml)
 
 if python is False:
-raise exceptions.PiglitFatalError('Cannot open 
"tests/{}.xml"'.format(name))
+raise exceptions.PiglitFatalError(
+'Cannot open "tests/{0}.xml" or "tests/{0}.meta.xml"'.format(name))
 
 try:
 mod = importlib.import_module('tests.{0}'.format(name))
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 01/35] update git ignore for this series

[Dylan Baker]

---
 .gitignore | 4 
 1 file changed, 4 insertions(+)

diff --git a/.gitignore b/.gitignore
index c005a5a..0f4c952 100644
--- a/.gitignore
+++ b/.gitignore
@@ -30,3 +30,7 @@ __pycache__/
 
 *.css
 *.html
+
+tests/*.xml
+tests/*.xml.gz
+!tests/*.meta.xml
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 24/35] tests/quick: fix filtering of vs_in shader tests

[Dylan Baker]

This filter is incorrect as is, in that it can mask tests that it can
remove tests it shouldn't, for example
spec@arb_enhanced_layouts@arb_enhanced_layouts-transform-feedback-layout-qualifiers_vs_interface.
Correcting this is problematic because it changes the way the filter
works and filter will now remove different tests than it did before.

However, it is necessary for a couple of reasons. 1) it's removing tests
it shouldn't, 2) this series is going to split reverse the
shader.py/all.py relationship, and this bug is going to surface there.

The good new is that while this changes the specific tests run, at least
on Intel hardware no failures are removed and no failures are added.
---
 tests/quick.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/quick.py b/tests/quick.py
index 73c4678..2d05d7a 100644
--- a/tests/quick.py
+++ b/tests/quick.py
@@ -17,6 +17,7 @@ import random
 
 from framework import grouptools
 from framework.test import PiglitGLTest
+from framework.test.shader_test import ShaderTest
 from tests.all import profile as _profile
 
 __all__ = ['profile']
@@ -32,8 +33,8 @@ class FilterVsIn(object):
 self.random = random.Random()
 self.random.seed(42)
 
-def __call__(self, name, _):
-if 'vs_in' in name:
+def __call__(self, name, test):
+if isinstance(test, ShaderTest) and 'vs_in' in grouptools.split(name):
 # 20%
 return self.random.random() <= .2
 return True
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 04/35] framework/test: Use getter for altering PiglitBaseTest Command

[Dylan Baker]

This removes mutation of the class, and pushes the change out to run
time instead of build time. This is better coding style, and is required
for XML serialization, since paths need to be relative in the XML, and
made absolute at runtime.
---
 framework/test/piglit_test.py | 5 -
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index cc129ed..3d5b156 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -70,8 +70,11 @@ class PiglitBaseTest(ValgrindMixin, Test):
 def __init__(self, command, run_concurrent=True, **kwargs):
 super(PiglitBaseTest, self).__init__(command, run_concurrent, **kwargs)
 
+@Test.command.getter
+def command(self):
 # Prepend TEST_BIN_DIR to the path.
-self._command[0] = os.path.join(TEST_BIN_DIR, self._command[0])
+cmd = os.path.join(TEST_BIN_DIR, super(PiglitBaseTest, 
self).command[0])
+return [cmd] + self._command[1:]
 
 def interpret_result(self):
 out = []
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 34/35] Generate xml for builtin profiles

[Dylan Baker]

This creates and installs xml for all builtin profiles. Using these
profiles I see startup times of ~1 second as opposed to more than 10
with current master, and runtimes that drop by ~1 minute.
---
 CMakeLists.txt  |  8 ++-
 tests/CMakeLists.no_api.txt | 89 ++-
 tests/CMakeLists.txt|  2 +-
 3 files changed, 98 insertions(+), 1 deletion(-)
 create mode 100644 tests/CMakeLists.no_api.txt

diff --git a/CMakeLists.txt b/CMakeLists.txt
index dc80ece..db047b4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -534,7 +534,13 @@ install (
DIRECTORY tests
DESTINATION ${PIGLIT_INSTALL_LIBDIR}
FILES_MATCHING REGEX 
".*\\.(xml|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
-   REGEX "CMakeFiles|CMakeLists" EXCLUDE
+   REGEX 
"CMakeFiles|CMakeLists|serializer.py|opengl.py|cl.py|quick_gl.py|glslparser.py|shader.py|quick_shader.py|no_error.py|llvmpipe_gl.py|sanity.py"
 EXCLUDE
+)
+
+install (
+   DIRECTORY ${CMAKE_BINARY_DIR}/tests
+   DESTINATION ${PIGLIT_INSTALL_LIBDIR}
+   FILES_MATCHING REGEX ".*\\.xml"
 )
 
 install (
diff --git a/tests/CMakeLists.no_api.txt b/tests/CMakeLists.no_api.txt
new file mode 100644
index 000..d7f95af
--- /dev/null
+++ b/tests/CMakeLists.no_api.txt
@@ -0,0 +1,89 @@
+# Copyright © 2018 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+add_custom_target(
+   static-shader-tests
+   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/shader.list
+   COMMAND ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/find_static_tests.py ${CMAKE_CURRENT_SOURCE_DIR} 
shader ${CMAKE_CURRENT_BINARY_DIR}/shader.list
+   VERBATIM
+)
+add_custom_target(
+   static-glslparser-tests
+   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/glslparser.list
+   COMMAND ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/find_static_tests.py ${CMAKE_CURRENT_SOURCE_DIR} 
glslparser ${CMAKE_CURRENT_BINARY_DIR}/glslparser.list
+   VERBATIM
+)
+add_custom_target(
+   static-asmparser-tests
+   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/asmparser.list
+   COMMAND ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/find_static_tests.py ${CMAKE_CURRENT_SOURCE_DIR} 
asmparser ${CMAKE_CURRENT_BINARY_DIR}/asmparser.list
+   VERBATIM
+)
+add_custom_target(
+   static-program-tests
+   BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/program.list
+   COMMAND ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/find_static_tests.py ${CMAKE_CURRENT_SOURCE_DIR} 
program ${CMAKE_CURRENT_BINARY_DIR}/program.list
+   VERBATIM
+)
+
+function(piglit_generate_xml name profile meta_target extra_args)
+   add_custom_command(
+   OUTPUT ${CMAKE_BINARY_DIR}/tests/${name}.xml
+   COMMAND ${CMAKE_COMMAND} -E env 
PIGLIT_BUILD_TREE=${CMAKE_BINARY_DIR} ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${name} 
${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py ${CMAKE_BINARY_DIR}/tests/${name}.xml 
 ${extra_args}
+   DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${ARGN}
+   VERBATIM
+   )
+   add_custom_target(
+   generate-${name}-xml
+   DEPENDS ${CMAKE_BINARY_DIR}/tests/${name}.xml
+   )
+   add_dependencies(${meta_target} generate-${name}-xml)
+endfunction()
+
+add_custom_target(gen-gl-xml)
+piglit_generate_xml(opengl opengl gen-gl-xml "")
+piglit_generate_xml(no_error no_error gen-gl-xml "")
+piglit_generate_xml(quick_gl quick_gl gen-gl-xml "")
+piglit_generate_xml(llvmpipe_gl llvmpipe_gl gen-gl-xml "")
+piglit_generate_xml(sanity sanity gen-gl-xml "")
+
+add_custom_target(gen-gl-gen-xml)
+piglit_generate_xml(glslparser glslparser gen-gl-gen-xml "" gen-gl-tests 
${CMAKE_CURRENT_BINARY_DIR}/glslparser.list 
${CMAKE_CURRENT_BINARY_DIR}/asmparser.list)
+piglit_generate_xml(glslparser_arb_compat glslparser gen-gl-gen-xml 

[Piglit] [PATCH 08/35] framework/test: add a test class for built-in constants

[Dylan Baker]

This has the same problem as asmparser tests.
---
 framework/test/piglit_test.py | 5 +
 1 file changed, 5 insertions(+)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index c53c3e5..0986ab0 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -203,3 +203,8 @@ class ASMParserTest(PiglitBaseTest):
 def command(self):
 command = super(ASMParserTest, self).command
 return command + [os.path.join(ROOT_DIR, self.filename)]
+
+
+class BuiltInConstantsTest(PiglitBaseTest):
+
+"""Test class for handling built in constants tests."""
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 06/35] framework/profile: Add a __len__ method to TestProfile

[Dylan Baker]

This exposes a standard interface for getting the number of tests in a
profile, which is itself nice. It will also allow us to encapsulate the
differences between the various profiles added in this series.
---
 framework/profile.py | 14 +++---
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/framework/profile.py b/framework/profile.py
index b6deed0..925271e 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -284,6 +284,9 @@ class TestProfile(object):
 'ignore_missing': False,
 }
 
+def __len__(self):
+return sum(1 for _ in self.itertests())
+
 def setup(self):
 """Method to do pre-run setup."""
 
@@ -381,15 +384,12 @@ def run(profiles, logger, backend, concurrency):
 """
 chunksize = 1
 
-# The logger needs to know how many tests are running. Because of filters
-# there's no way to do that without making a concrete list out of the
-# filters profiles.
-profiles = [(p, list(p.itertests())) for p in profiles]
-log = LogManager(logger, sum(len(l) for _, l in profiles))
+profiles = [(p, p.itertests()) for p in profiles]
+log = LogManager(logger, sum(len(p) for p, _ in profiles))
 
 # check that after the filters are run there are actually tests to run.
-if not any(l for _, l in profiles):
-raise exceptions.PiglitUserError('no matching tests')
+# if not any(l for _, l in profiles):
+# raise exceptions.PiglitUserError('no matching tests')
 
 def test(name, test, profile, this_pool=None):
 """Function to call test.execute from map"""
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 27/35] tests/glsl_parser_test.py: fix is_skip for serialized profiles

[Dylan Baker]

Currently is skip relies on runtime detection of which glslparsertest
binaries are built, but we can't assume that at build time. Instead
always assign the appropriate binary, and then check for the existence
of that binary at run time.
---
 framework/test/glsl_parser_test.py| 20 
 unittests/framework/test/test_glsl_parser_test.py | 13 --
 2 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/framework/test/glsl_parser_test.py 
b/framework/test/glsl_parser_test.py
index 35eadfe..88646c9 100644
--- a/framework/test/glsl_parser_test.py
+++ b/framework/test/glsl_parser_test.py
@@ -122,7 +122,7 @@ class Parser(object):
 # If GLES is requested, but piglit was not built with a gles version,
 # then ARB_ES3_compatibility is required. Add it to
 # self.gl_required
-if self.glsl_es_version and not _HAS_GLES_BIN:
+if self.glsl_es_version and _FORCE_DESKTOP_VERSION:
 if self.glsl_es_version == 1.0:
 ver = '2'
 elif self.glsl_es_version == 3.0:
@@ -148,14 +148,10 @@ class Parser(object):
 then the test will be skipped in the python layer.
 
 """
-if (_is_gles_version(version)
-and _HAS_GLES_BIN
-and not _FORCE_DESKTOP_VERSION):
+if _is_gles_version(version) and not _FORCE_DESKTOP_VERSION:
 return 'glslparsertest_gles2'
-elif _HAS_GL_BIN:
-return 'glslparsertest'
 else:
-return 'None'
+return 'glslparsertest'
 
 def get_command(self, filepath):
 """ Create the command argument to pass to super()
@@ -295,8 +291,12 @@ class GLSLParserTest(FastSkipMixin, PiglitBaseTest):
 glsl_es_version=parsed.glsl_es_version)
 
 def is_skip(self):
-if os.path.basename(self.command[0]) == 'None':
-raise TestIsSkip('Test is for desktop OpenGL, '
- 'but only an OpenGL ES binary has been built')
+if os.path.basename(self.command[0]) == 'glslparsertest' and not 
_HAS_GL_BIN:
+raise TestIsSkip('Test is for desktop OpenGL, but piglit was not '
+ 'built with OpenGL support.')
+elif (os.path.basename(self.command[0]) == 'glslparsertest_gles2'
+  and not _HAS_GLES_BIN):
+raise TestIsSkip('Test is for OpenGL ES, but piglit was not '
+ 'built with OpenGL ES support.')
 
 super(GLSLParserTest, self).is_skip()
diff --git a/unittests/framework/test/test_glsl_parser_test.py 
b/unittests/framework/test/test_glsl_parser_test.py
index b396fea..bf217b1 100644
--- a/unittests/framework/test/test_glsl_parser_test.py
+++ b/unittests/framework/test/test_glsl_parser_test.py
@@ -339,22 +339,21 @@ def test_valid_extensions(ext, tmpdir):
 
 
 @pytest.mark.parametrize(
-"version,has_bin,forced",
+"version,forced",
 itertools.product(
 ['1.00', '3.00', '3.10', '3.20', '3.00 es', '3.10 es', '3.20 es'],
-[True, False], [True, False]))
-def test_get_glslparsertest_gles2(version, has_bin, forced, tmpdir, mocker):
+[True, False]))
+def test_get_glslparsertest_gles2(version, forced, tmpdir, mocker):
 """Tests for assigning the correct binary for GLES tests.
 
 Tests with and without the gles binary and with and without the force
 desktop mode.
 """
-if not has_bin or forced:
+if forced:
 expected = 'glslparsertest'
 else:
 expected = 'glslparsertest_gles2'
 
-mocker.patch('framework.test.glsl_parser_test._HAS_GLES_BIN', has_bin)
 mocker.patch('framework.test.glsl_parser_test._FORCE_DESKTOP_VERSION',
  forced)
 
@@ -436,7 +435,7 @@ def test_add_compatibility_requirement_fastskip(version, 
extension, tmpdir,
 
 This test checks the fast skipping variable
 """
-mocker.patch('framework.test.glsl_parser_test._HAS_GLES_BIN', False)
+mocker.patch('framework.test.glsl_parser_test._FORCE_DESKTOP_VERSION', 
True)
 
 p = tmpdir.join('test.frag')
 p.write(textwrap.dedent("""\
@@ -466,7 +465,7 @@ def test_add_compatibility_requirement_binary(version, 
extension, tmpdir,
 
 This test checks the glslparsertest binary command line.
 """
-mocker.patch('framework.test.glsl_parser_test._HAS_GLES_BIN', False)
+mocker.patch('framework.test.glsl_parser_test._FORCE_DESKTOP_VERSION', 
True)
 
 p = tmpdir.join('test.frag')
 p.write(textwrap.dedent("""\
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 21/35] profile: allow forcing python or xml loading

[Dylan Baker]

This will be used during build time to allow profiles that modify other
profiles to load XML instead of rebuilding.
---
 framework/profile.py | 21 +++--
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/framework/profile.py b/framework/profile.py
index 7b1b41b..2395b22 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -429,7 +429,7 @@ class TestProfile(object):
 yield k, v
 
 
-def load_test_profile(filename):
+def load_test_profile(filename, python=None):
 """Load a python module and return it's profile attribute.
 
 All of the python test files provide a profile attribute which is a
@@ -449,15 +449,24 @@ def load_test_profile(filename):
 
 Arguments:
 filename -- the name of a python module to get a 'profile' from
+
+Keyword Arguments:
+python -- If this is None (the default) XML is tried, and then a python
+  module. If True, then only python is tried, if False then only
+  XML is tried.
 """
 name = os.path.splitext(os.path.basename(filename))[0]
-xml = os.path.join(ROOT_DIR, 'tests', name + '.xml')
-if os.path.exists(xml):
-return XMLProfile(xml)
+xml = os.path.join('tests', name + '.xml')
+if not python:
+xml = os.path.join(ROOT_DIR, 'tests', name + '.xml')
+if os.path.exists(xml):
+return XMLProfile(xml)
+
+if python is False:
+raise exceptions.PiglitFatalError('Cannot open 
"tests/{}.xml"'.format(name))
 
 try:
-mod = importlib.import_module('tests.{0}'.format(
-os.path.splitext(os.path.basename(filename))[0]))
+mod = importlib.import_module('tests.{0}'.format(name))
 except ImportError:
 raise exceptions.PiglitFatalError(
 'Failed to import "{}", there is either something wrong with the '
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 00/35] Serialize profiles into XML at build time

[Dylan Baker]

I don't expect everyone I've CC'd to give thorough review (or any
review), I've mostly CC'd people who I think would be interested in this
work, or who's work flow I might be altered by it.

Piglit has struggled to cope with the growing number of tests that it
contains, especially with startup time. Piglit has always calculated
tests at runtime, which was not a problem when there were only a few
hundred or even thousand tests. Piglit now has roughly 55,000
OpenGL/OpenGL ES tests, which is a lot to calculate at start up. It also
means that piglit needs to keep a python object for each of those tests
in memory, which has sent the resident memory usage soaring. We've also
moved to automatic test discovery for glslparser, asmparser, and shader
tests, which is very convenient and reduces typing, but further
increases the amount of time spent starting up. This has even made
features which decrease runtime, like fast skipping, hurt startup
performance, making it a less than desirable tradeoff in some cases.
Even on a relatively fast machine with an nvme disk 15-20 seconds is not
an unheard of startup time. That might be okay to run 55,000 tests, but
not if you only need a dozen, such as when bisecting.

This series is my proposal to fix that, mainly by moving much of that
cost to build time. This series creates the infrastructure build XML
base profiles at build time, which are installed with piglit instead of
the python profiles. These profiles are lazily iterated over to ease
memory usage, test objects are created as they are run, and python can
garbage collect them as soon as they are done running. Along with that
any filters applied to profiles (like removing 80% of the vs_in shader
tests in quick) are done before the profile is serialized, and all fast
skipping information is collected at build time as well, and encoded in
the XML. All this means that start times are vastly reduced.

For example:
XML profiles
quick: 0.5
shader: 0.5

master
quick: 11.6
shader: 7.3

This series also implements some optimizations for running without
filters or test-lists, if you add a filter quick would take 2.5
seconds, because that is necessary to calculate the total number of
tests before starting.

To keep classic profiles like all, quick, quick_cl, gpu, cpu, and
llvmpipe working this series adds meta profiles, small XML snippets that
list other profiles. These can contain other meta profiles, xml
profiles, or python profiles. This means that for most uses cases your
existing command line will still work, `./piglit run quick out -c` will
still do exactly the same thing as before, just faster.

The XML generated is dumb, there is no encoding of options or logic. An
early version of this series did contain logic and options, but the
result was pretty terrible. It was very hard to read, and the code to
handle it was very complicated. I've chosen not to go down that path.
There are drawbacks, some things that relied on run time generation have
cannot be handled the same way, among them the "multi shader" concept,
where shader_runner consumes a directory of shader_tests at a time. This
was previously handled via a --process-isolation=false flag, now its
encoded into profiles, "shader_multi" and "quick_shader_multi"; there
was also an option to use glslparsertest with ES shaders and
ARB_ES_compatibility, that is now "glslparser_arb_compat". I haven't
added metaprofiles for these cases, although we certainly could (or you
can write your own, the schema is dead simple), so `./piglit run quick
out --process-isolation=false` is now `./piglit run quick_gl glslparser
quick_shader_multi out`.

I've run this through our CI extensively, and gotten green results out
of it across the board.

I know this is a big series, but piglit makes a lot of assumptions about the
test profiles being created at runtime, and we've had to changes those
assumptions.


Dylan Baker (35):
  update git ignore for this series
  test/piglit_test: add ROOT_DIR variable
  framework/profile: Allow a group manager class to be overwritten
  framework/test: Use getter for altering PiglitBaseTest Command
  framework/test: expose required and excluded platforms
  framework/profile: Add a __len__ method to TestProfile
  framework: Use custom class for ASM parser tests
  framework/test: add a test class for built-in constants
  tests: use BuiltInConstantsClass
  framework: use a class method for building test via parsing
  framework: do the same for shader test
  framework/test: Split multishader too
  framework/test/piglit_test: make cl_concurrency always a boolean
  framework/test: Add class for cl-program-tester
  framework/test: Make shader paths relative
  framework/test: use relative paths for GLSLParser tests
  tests/all: Make asmparser tests path relative
  framework/test: make BuiltInConstantTest files relative
  framework/test: make CLProgramTester take relative paths
  profile: Add support for loa

[Piglit] [PATCH 19/35] framework/test: make CLProgramTester take relative paths

[Dylan Baker]

---
 framework/test/piglit_test.py | 2 +-
 tests/cl.py   | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index 4fd825a..6511e0f 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -226,5 +226,5 @@ class CLProgramTester(PiglitCLTest):
 @PiglitCLTest.command.getter
 def command(self):
 command = super(CLProgramTester, self).command
-command.insert(1, self.filename)
+command.insert(1, os.path.join(ROOT_DIR, self.filename))
 return command
diff --git a/tests/cl.py b/tests/cl.py
index 2ab3502..56b3fc1 100644
--- a/tests/cl.py
+++ b/tests/cl.py
@@ -14,7 +14,7 @@ from __future__ import (
 import os
 
 from framework.profile import TestProfile
-from framework.test.piglit_test import PiglitCLTest, CLProgramTester
+from framework.test.piglit_test import PiglitCLTest, CLProgramTester, ROOT_DIR
 from framework import grouptools
 from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
 
@@ -111,7 +111,7 @@ def add_program_test_dir(group, dirpath):
 continue
 
 profile.test_list[grouptools.join(group, testname)] = CLProgramTester(
-os.path.join(dirpath, filename))
+os.path.join(os.path.relpath(dirpath, ROOT_DIR), filename))
 
 
 add_program_test_dir(grouptools.join('program', 'build'),
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 18/35] framework/test: make BuiltInConstantTest files relative

[Dylan Baker]

---
 framework/test/piglit_test.py |  5 +
 tests/all.py  | 26 +-
 2 files changed, 18 insertions(+), 13 deletions(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index f5fd0c8..4fd825a 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -199,6 +199,11 @@ class BuiltInConstantsTest(PiglitBaseTest):
 
 """Test class for handling built in constants tests."""
 
+@PiglitBaseTest.command.getter
+def command(self):
+command = super(BuiltInConstantsTest, self).command
+command[1] = os.path.join(ROOT_DIR, 'tests', command[1])
+return command
 
 class PiglitCLTest(PiglitBaseTest):  # pylint: disable=too-few-public-methods
 """ OpenCL specific Test class.
diff --git a/tests/all.py b/tests/all.py
index 6aa45f0..33bca19 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -1231,7 +1231,7 @@ with profile.test_list.group_manager(
 BuiltInConstantsTest,
 grouptools.join('spec', 'glsl-es-1.00')) as g:
 g(['built-in-constants_gles2',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-es-1.00', 
'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-es-1.00', 'minimum-maximums.txt')],
   'built-in constants')
 
 # Group spec/glsl-1.10
@@ -1268,7 +1268,7 @@ with profile.test_list.group_manager(
 BuiltInConstantsTest,
 grouptools.join('spec', 'glsl-1.10')) as g:
 g(['built-in-constants',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-1.10', 'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-1.10', 'minimum-maximums.txt')],
   'built-in constants')
 
 with profile.test_list.group_manager(
@@ -1282,7 +1282,7 @@ with profile.test_list.group_manager(
 grouptools.join('spec', 'glsl-1.20')) as g:
 g(['glsl-1.20-getactiveuniform-constant'])
 g(['built-in-constants',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-1.20', 'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-1.20', 'minimum-maximums.txt')],
   'built-in constants',
   override_class=BuiltInConstantsTest)
 
@@ -1384,7 +1384,7 @@ with profile.test_list.group_manager(
 grouptools.join('spec', 'glsl-1.30')) as g:
 g(['glsl-1.30-texel-offset-limits'], 'texel-offset-limits')
 g(['built-in-constants',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-1.30', 'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-1.30', 'minimum-maximums.txt')],
   'built-in constants',
override_class=BuiltInConstantsTest)
 
@@ -1506,7 +1506,7 @@ with profile.test_list.group_manager(
 BuiltInConstantsTest,
 grouptools.join('spec', 'glsl-1.40')) as g:
 g(['built-in-constants',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-1.40', 'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-1.40', 'minimum-maximums.txt')],
   'built-in constants')
 
 with profile.test_list.group_manager(
@@ -1548,7 +1548,7 @@ with profile.test_list.group_manager(
 PiglitGLTest,
 grouptools.join('spec', 'glsl-1.50')) as g:
 g(['built-in-constants',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-1.50', 'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-1.50', 'minimum-maximums.txt')],
   'built-in constants',
   override_class=BuiltInConstantsTest)
 g(['glsl-1.50-gs-emits-too-few-verts'], 'gs-emits-too-few-verts')
@@ -1624,13 +1624,13 @@ with profile.test_list.group_manager(
 with profile.test_list.group_manager(
 BuiltInConstantsTest, grouptools.join('spec', 'glsl-3.30')) as g:
 g(['built-in-constants',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-3.30', 'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-3.30', 'minimum-maximums.txt')],
   'built-in constants')
 
 with profile.test_list.group_manager(
 BuiltInConstantsTest, grouptools.join('spec', 'glsl-es-3.00')) as g:
 g(['built-in-constants_gles3',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-es-3.00', 
'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-es-3.00', 'minimum-maximums.txt')],
   'built-in constants')
 
 with profile.test_list.group_manager(
@@ -1641,7 +1641,7 @@ with profile.test_list.group_manager(
 with profile.test_list.group_manager(
 BuiltInConstantsTest, grouptools.join('spec', 'glsl-es-3.10')) as g:
 g(['built-in-constants_gles3',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-es-3.10', 
'minimum-maximums.txt')],
+   os.path.join('spec', 'glsl-es-3.10', 'minimum-maximums.txt')],
   'built-in constants')
 
 # AMD_performance_monitor
@@ -1673,7 +1673,7 @@ with profile.test_list.group_manager(
 g(['arb_tessellation_shader-invalid-patch-vertices-range'])
 g(['arb_tessellation_shader-invalid-primitive'])
 g(['built-in-constants',
-   os.path.join(TESTS_DIR, 'spec', 'arb_tessellation_shader', 
'minimum-maximums.txt')],
+   os.path.join('spec', 'arb_tessellation_shader', 
'minimum-maximums.txt')],
   'built-in-constants',
   

[Piglit] [PATCH 05/35] framework/test: expose required and excluded platforms

[Dylan Baker]

These need to be exposed for serialization.
---
 framework/test/piglit_test.py | 12 ++--
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index 3d5b156..ef3bfdc 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -137,13 +137,13 @@ class PiglitGLTest(WindowResizeMixin, PiglitBaseTest):
 
 if not require_platforms or set(require_platforms).issubset(
 set(core.PLATFORMS)):
-self.__require_platforms = require_platforms or []
+self.require_platforms = require_platforms or []
 else:
 raise Exception("Error: require_platform is not valid")
 
 if (not exclude_platforms or
 set(exclude_platforms).issubset(set(core.PLATFORMS))):
-self.__exclude_platforms = exclude_platforms or []
+self.exclude_platforms = exclude_platforms or []
 else:
 raise Exception("Error: exclude_platforms is not valid")
 
@@ -156,16 +156,16 @@ class PiglitGLTest(WindowResizeMixin, PiglitBaseTest):
 
 """
 platform = options.OPTIONS.env['PIGLIT_PLATFORM']
-if self.__require_platforms and platform not in 
self.__require_platforms:
+if self.require_platforms and platform not in self.require_platforms:
 raise TestIsSkip(
 'Test requires one of the following platforms "{}" '
 'but the platform is "{}"'.format(
-self.__require_platforms, platform))
-elif self.__exclude_platforms and platform in self.__exclude_platforms:
+self.require_platforms, platform))
+elif self.exclude_platforms and platform in self.exclude_platforms:
 raise TestIsSkip(
 'Test cannot be run on any of the following platforms "{}" '
 'and the platform is "{}"'.format(
-self.__exclude_platforms, platform))
+self.exclude_platforms, platform))
 super(PiglitGLTest, self).is_skip()
 
 @PiglitBaseTest.command.getter
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 09/35] tests: use BuiltInConstantsClass

[Dylan Baker]

---
 tests/all.py | 55 ++---
 1 file changed, 27 insertions(+), 28 deletions(-)

diff --git a/tests/all.py b/tests/all.py
index fe4a14c..a50a4c3 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -21,7 +21,7 @@ from framework.driver_classifier import DriverClassifier
 from framework.test import (PiglitGLTest, PiglitBaseTest,
 GLSLParserTest, GLSLParserNoConfigError)
 from framework.test.shader_test import ShaderTest, MultiShaderTest
-from framework.test.piglit_test import ASMParserTest
+from framework.test.piglit_test import ASMParserTest, BuiltInConstantsTest
 from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
 
 __all__ = ['profile']
@@ -1224,11 +1224,10 @@ with profile.test_list.group_manager(
 
 # Group spec/glsl-es-1.00
 with profile.test_list.group_manager(
-PiglitGLTest,
+BuiltInConstantsTest,
 grouptools.join('spec', 'glsl-es-1.00')) as g:
 g(['built-in-constants_gles2',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-es-1.00',
-'minimum-maximums.txt')],
+   os.path.join(TESTS_DIR, 'spec', 'glsl-es-1.00', 
'minimum-maximums.txt')],
   'built-in constants')
 
 # Group spec/glsl-1.10
@@ -1262,7 +1261,7 @@ with profile.test_list.group_manager(
   'simple {} {}'.format(type_, arrayspec))
 
 with profile.test_list.group_manager(
-PiglitGLTest,
+BuiltInConstantsTest,
 grouptools.join('spec', 'glsl-1.10')) as g:
 g(['built-in-constants',
os.path.join(TESTS_DIR, 'spec', 'glsl-1.10', 'minimum-maximums.txt')],
@@ -1280,7 +1279,8 @@ with profile.test_list.group_manager(
 g(['glsl-1.20-getactiveuniform-constant'])
 g(['built-in-constants',
os.path.join(TESTS_DIR, 'spec', 'glsl-1.20', 'minimum-maximums.txt')],
-  'built-in constants')
+  'built-in constants',
+  override_class=BuiltInConstantsTest)
 
 with profile.test_list.group_manager(
 PiglitGLTest,
@@ -1381,7 +1381,8 @@ with profile.test_list.group_manager(
 g(['glsl-1.30-texel-offset-limits'], 'texel-offset-limits')
 g(['built-in-constants',
os.path.join(TESTS_DIR, 'spec', 'glsl-1.30', 'minimum-maximums.txt')],
-  'built-in constants')
+  'built-in constants',
+   override_class=BuiltInConstantsTest)
 
 with profile.test_list.group_manager(
 PiglitGLTest,
@@ -1498,7 +1499,7 @@ with profile.test_list.group_manager(
 
 # Group spec/glsl-1.40
 with profile.test_list.group_manager(
-PiglitGLTest,
+BuiltInConstantsTest,
 grouptools.join('spec', 'glsl-1.40')) as g:
 g(['built-in-constants',
os.path.join(TESTS_DIR, 'spec', 'glsl-1.40', 'minimum-maximums.txt')],
@@ -1544,7 +1545,8 @@ with profile.test_list.group_manager(
 grouptools.join('spec', 'glsl-1.50')) as g:
 g(['built-in-constants',
os.path.join(TESTS_DIR, 'spec', 'glsl-1.50', 'minimum-maximums.txt')],
-  'built-in constants')
+  'built-in constants',
+  override_class=BuiltInConstantsTest)
 g(['glsl-1.50-gs-emits-too-few-verts'], 'gs-emits-too-few-verts')
 g(['glsl-1.50-geometry-end-primitive-optional-with-points-out'],
   'gs-end-primitive-optional-with-points-out')
@@ -1616,16 +1618,15 @@ with profile.test_list.group_manager(
   prim_type, restart_index))
 
 with profile.test_list.group_manager(
-PiglitGLTest, grouptools.join('spec', 'glsl-3.30')) as g:
+BuiltInConstantsTest, grouptools.join('spec', 'glsl-3.30')) as g:
 g(['built-in-constants',
os.path.join(TESTS_DIR, 'spec', 'glsl-3.30', 'minimum-maximums.txt')],
   'built-in constants')
 
 with profile.test_list.group_manager(
-PiglitGLTest, grouptools.join('spec', 'glsl-es-3.00')) as g:
+BuiltInConstantsTest, grouptools.join('spec', 'glsl-es-3.00')) as g:
 g(['built-in-constants_gles3',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-es-3.00',
-'minimum-maximums.txt')],
+   os.path.join(TESTS_DIR, 'spec', 'glsl-es-3.00', 
'minimum-maximums.txt')],
   'built-in constants')
 
 with profile.test_list.group_manager(
@@ -1634,10 +1635,9 @@ with profile.test_list.group_manager(
 g(['varying-struct-centroid_gles3'])
 
 with profile.test_list.group_manager(
-PiglitGLTest, grouptools.join('spec', 'glsl-es-3.10')) as g:
+BuiltInConstantsTest, grouptools.join('spec', 'glsl-es-3.10')) as g:
 g(['built-in-constants_gles3',
-   os.path.join(TESTS_DIR, 'spec', 'glsl-es-3.10',
-'minimum-maximums.txt')],
+   os.path.join(TESTS_DIR, 'spec', 'glsl-es-3.10', 
'minimum-maximums.txt')],
   'built-in constants')
 
 # AMD_performance_monitor
@@ -1669,9 +1669,9 @@ with profile.test_list.group_manager(
 g(['arb_tessellation_shader-invalid-patch-vertices-range'])
 g(['arb_tessellation_shader-invalid-primitive'])
 g(['built-in-constants',
-   os.path.join(TESTS_DIR, 'spec', 

[Piglit] [PATCH 30/35] tests/cl.py: fix out of tree serialization

[Dylan Baker]

---
 tests/cl.py | 67 --
 1 file changed, 45 insertions(+), 22 deletions(-)

diff --git a/tests/cl.py b/tests/cl.py
index 56b3fc1..7d4e17d 100644
--- a/tests/cl.py
+++ b/tests/cl.py
@@ -104,47 +104,70 @@ with profile.test_list.group_manager(PiglitCLTest, 
'interop') as g:
 g(['cl-interop-egl_khr_cl_event2'], 'EGL_KHR_cl_event2')
 
 
-def add_program_test_dir(group, dirpath):
-for filename in os.listdir(dirpath):
+def add_program_test_dir(group, dirpath, buildbase, installbase):
+for filename in os.listdir(os.path.join(buildbase, dirpath)):
 testname, ext = os.path.splitext(filename)
 if ext not in ['.cl', '.program_test']:
 continue
 
 profile.test_list[grouptools.join(group, testname)] = CLProgramTester(
-os.path.join(os.path.relpath(dirpath, ROOT_DIR), filename))
+os.path.join(installbase, dirpath, os.path.basename(filename)))
 
 
+base_test_dir = os.path.basename(TESTS_DIR)
+base_gen_dir = os.path.basename(GENERATED_TESTS_DIR)
+
 add_program_test_dir(grouptools.join('program', 'build'),
- os.path.join(TESTS_DIR, 'cl', 'program', 'build'))
+ os.path.join('cl', 'program', 'build'),
+ TESTS_DIR,
+ base_test_dir)
 add_program_test_dir(grouptools.join('program', 'build', 'fail'),
- os.path.join(TESTS_DIR, 'cl', 'program', 'build', 'fail'))
+ os.path.join('cl', 'program', 'build', 'fail'),
+ TESTS_DIR,
+ base_test_dir)
 add_program_test_dir(grouptools.join('program', 'execute'),
- os.path.join(TESTS_DIR, 'cl', 'program', 'execute'))
+ os.path.join('cl', 'program', 'execute'),
+ TESTS_DIR,
+ base_test_dir)
 add_program_test_dir(grouptools.join('program', 'execute'),
- os.path.join(TESTS_DIR, 'cl', 'program', 'execute',
-  'builtin', 'atomic'))
+ os.path.join('cl', 'program', 'execute', 'builtin', 
'atomic'),
+ TESTS_DIR,
+ base_test_dir)
 add_program_test_dir(grouptools.join('program', 'execute'),
- os.path.join(TESTS_DIR, 'cl', 'program', 'execute',
-  'builtin', 'convert'))
+ os.path.join('cl', 'program', 'execute', 'builtin', 
'convert'),
+ TESTS_DIR,
+ base_test_dir)
 
 # Run generated built-in tests
 add_program_test_dir(grouptools.join('program', 'execute', 'builtin'),
- os.path.join(GENERATED_TESTS_DIR, 'cl', 'builtin', 'int'))
+ os.path.join('cl', 'builtin', 'int'),
+ GENERATED_TESTS_DIR,
+ base_gen_dir)
 add_program_test_dir(grouptools.join('program', 'execute', 'builtin'),
- os.path.join(GENERATED_TESTS_DIR, 'cl', 'builtin',
-  'math'))
+ os.path.join('cl', 'builtin', 'math'),
+ GENERATED_TESTS_DIR,
+ base_gen_dir)
 add_program_test_dir(grouptools.join('program', 'execute', 'builtin'),
- os.path.join(GENERATED_TESTS_DIR, 'cl', 'builtin',
-  'relational'))
+ os.path.join('cl', 'builtin', 'relational'),
+ GENERATED_TESTS_DIR,
+ base_gen_dir)
 add_program_test_dir(grouptools.join('program', 'execute', 'builtin'),
- os.path.join(GENERATED_TESTS_DIR, 'cl', 'builtin',
-  'common'))
+ os.path.join('cl', 'builtin', 'common'),
+ GENERATED_TESTS_DIR,
+ base_gen_dir)
 add_program_test_dir(grouptools.join('program', 'execute', 'builtin'),
- os.path.join(GENERATED_TESTS_DIR, 'cl', 'builtin',
-  'misc'))
+ os.path.join('cl', 'builtin', 'misc'),
+ GENERATED_TESTS_DIR,
+ base_gen_dir)
 add_program_test_dir(grouptools.join('program', 'execute', 'store'),
- os.path.join(GENERATED_TESTS_DIR, 'cl', 'store'))
+ os.path.join('cl', 'store'),
+ GENERATED_TESTS_DIR,
+ base_gen_dir)
 add_program_test_dir(grouptools.join('program', 'execute', 'vstore'),
- os.path.join(GENERATED_TESTS_DIR, 'cl', 'vstore'))
+ os.path.join('cl', 'vstore'),
+ GENERATED_TESTS_DIR,
+ base_gen_dir)
 add_program_test_dir(grouptools.join('program', 'execute', 'vload'),
- os.path.join(GENERATED_TESTS_DIR, 'cl', 'vload'))
+ os.path.join('cl', 'vload'),
+  

[Piglit] [PATCH 20/35] profile: Add support for loading xml based profiles

[Dylan Baker]

Which includes de-serialization on demand. This will vastly reduce the
amount of resident memory that is necessary to hold the tests since test
instances are made on demand, and can be garbage collected as soon as
the run in complete.
---
 framework/profile.py| 108 +-
 unittests/framework/test_profile.py |   5 +-
 2 files changed, 108 insertions(+), 5 deletions(-)

diff --git a/framework/profile.py b/framework/profile.py
index 925271e..7b1b41b 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -30,15 +30,18 @@ tests, and the Test instance.
 from __future__ import (
 absolute_import, division, print_function, unicode_literals
 )
+import ast
 import collections
 import contextlib
 import copy
 import importlib
+import io
 import itertools
 import multiprocessing
 import multiprocessing.dummy
 import os
 import re
+import xml.etree.cElementTree as et
 
 import six
 
@@ -47,6 +50,12 @@ from framework.dmesg import get_dmesg
 from framework.log import LogManager
 from framework.monitoring import Monitoring
 from framework.test.base import Test, DummyTest
+from framework.test.piglit_test import (
+PiglitCLTest, PiglitGLTest, ASMParserTest, BuiltInConstantsTest,
+CLProgramTester, ROOT_DIR,
+)
+from framework.test.shader_test import ShaderTest, MultiShaderTest
+from framework.test.glsl_parser_test import GLSLParserTest
 
 __all__ = [
 'RegexFilter',
@@ -256,6 +265,100 @@ class TestDict(collections.MutableMapping):
 self.__allow_reassignment -= 1
 
 
+def make_test(element):
+"""Rebuild a test instance from xml."""
+def process(elem, opt):
+k = elem.attrib['name']
+v = elem.attrib['value']
+try:
+opt[k] = ast.literal_eval(v)
+except ValueError:
+opt[k] = v
+
+type_ = element.attrib['type']
+options = {}
+for e in element.findall('./option'):
+process(e, options)
+
+if type_ == 'gl':
+return PiglitGLTest(**options)
+if type_ == 'gl_builtin':
+return BuiltInConstantsTest(**options)
+if type_ == 'cl':
+return PiglitCLTest(**options)
+if type_ == 'cl_prog':
+return CLProgramTester(**options)
+if type_ == 'shader':
+return ShaderTest(**options)
+if type_ == 'glsl_parser':
+return GLSLParserTest(**options)
+if type_ == 'asm_parser':
+return ASMParserTest(**options)
+if type_ == 'multi_shader':
+options['skips'] = []
+for e in element.findall('./Skips/Skip/option'):
+skips = {}
+process(e, skips)
+options['skips'].append(skips)
+return MultiShaderTest(**options)
+raise Exception('Unreachable')
+
+
+class XMLProfile(object):
+
+def __init__(self, filename):
+self.filename = filename
+self.forced_test_list = []
+self.filters = []
+self.options = {
+'dmesg': get_dmesg(False),
+'monitor': Monitoring(False),
+'ignore_missing': False,
+}
+
+def __len__(self):
+if not (self.filters or self.forced_test_list):
+with io.open(self.filename, 'rt') as f:
+iter_ = et.iterparse(f, events=(b'start', ))
+for _, elem in iter_:
+if elem.tag == 'PiglitTestList':
+return int(elem.attrib['count'])
+return sum(1 for _ in self.itertests())
+
+def setup(self):
+pass
+
+def teardown(self):
+pass
+
+def _itertests(self):
+"""Always iterates tests instead of using the forced test_list."""
+with io.open(self.filename, 'rt') as f:
+doc = et.iterparse(f, events=(b'end', ))
+_, root = next(doc)  # get the root so we can keep clearing it
+for _, e in doc:
+if e.tag != 'Test':
+continue
+k = e.attrib['name']
+v = make_test(e)
+if all(f(k, v) for f in self.filters):
+yield k, v
+root.clear()
+
+def itertests(self):
+if self.forced_test_list:
+alltests = dict(self._itertests())
+opts = collections.OrderedDict()
+for n in self.forced_test_list:
+if self.options['ignore_missing'] and n not in alltests:
+opts[n] = DummyTest(n, status.NOTRUN)
+else:
+opts[n] = alltests[n]
+return six.iteritems(opts)
+else:
+return iter(self._itertests())
+
+
 class TestProfile(object):
 """Class that holds a list of tests for execution.
 
@@ -347,6 +450,11 @@ def load_test_profile(filename):
 Arguments:
 filename -- the name of a python module to get a 'profile' from
 """
+name = os.path.splitext(os.path.basename(filename))[0]
+xml = os.path.join(ROOT_DIR, 'tests', name + '.xml')
+if os.path.exists(xml):
+ 

[Piglit] [PATCH 31/35] opengl.py: Remove exported gl extensions

[Dylan Baker]

This conflicts with the idea of serializing the tests at build time,
since we can't assume that the build machine has the same features set
as the machine that will run tests, we can't run such a filter.

Since startup time is much faster and fast skip calculations are done at
build time I hope that no one will miss this.
---
 tests/opengl.py | 335 +
 1 file changed, 122 insertions(+), 213 deletions(-)

diff --git a/tests/opengl.py b/tests/opengl.py
index ec80be1..3a203a1 100644
--- a/tests/opengl.py
+++ b/tests/opengl.py
@@ -11,9 +11,7 @@ import platform
 from six.moves import range
 
 from framework import grouptools
-from framework.test import opengl
 from framework import options
-from framework import wflinfo
 from framework.profile import TestProfile
 from framework.driver_classifier import DriverClassifier
 from framework.test.piglit_test import (
@@ -206,86 +204,6 @@ def power_set(s):
 # Collecting all tests
 profile = TestProfile()  # pylint: disable=invalid-name
 
-wfl_info = wflinfo.WflInfo()
-
-
-def gl_extension_supported(ext_name):
-"""Is the named OpenGL extension supported?"""
-if wfl_info.gl_extensions == set():
-# Don't know what extensions are supported.  Return true so we don't
-# skip anything.
-return True
-return ext_name in wfl_info.gl_extensions
-
-
-def is_feature_directory_supported(dir_name):
-"""Determine if dir_name specifies an OpenGL feature (extension or GL
-version) which is supported by the host.  If we return False, it means
-the extension/version is definitely not supported.  If we return True,
-it means the extension/version is possibly suppported.  We're a little
-fuzzy because we don't yet parse all the directory name possibilities
-(like ES tests).
-
-Also, if the 'wflinfo' program is not installed or fails (i.e. we can't
-determine the GL version/extension) we return True.
-"""
-if dir_name[:4] in {"amd_", "arb_", "ati_", "ext_", "khr_", "oes_"}:
-# The directory is a GL extension name, but of the format "arb_foo_bar"
-# instead of "GL_ARB_foo_bar".  We convert the former into the later
-# and check if the extension is supported.
-ext_name = "GL_" + dir_name[0:4].upper() + dir_name[4:]
-return gl_extension_supported(ext_name)
-elif dir_name[:5] == "gles-":
-# OpenGL ES test
-version = float(dir_name[5:])
-return wfl_info.gles_version is None or version <= 
wfl_info.gles_version
-elif dir_name[:8] == "glsl-es-":
-# OpenGL ES shader test
-version = float(dir_name[8:])
-return wfl_info.glsl_es_version is None or version <= 
wfl_info.glsl_es_version
-elif dir_name[:3] == "gl-":
-# The directory is a GL version
-version = float(dir_name[3:])
-return wfl_info.gl_version is None or version <= wfl_info.gl_version
-elif dir_name[:5] == "glsl-":
-# The directory is a GLSL version
-version = float(dir_name[5:])
-return wfl_info.glsl_version is None or version <= 
wfl_info.glsl_version
-else:
-# The directory is something else.  Don't skip it.
-return True
-
-
-def walk_filter_dir_tree(root):
-"""Recursively walk the directory tree rooted at 'root'.
-If we find a directory path of the form ".../spec/foo/" we'll check if
-'foo' is a supported extension/feature/version.  If not, we do not
-traverse foo/.  Otherwise, we add continue traversing.
-The return value is a list of (dirpath, filename) tuples.
-"""
-curdir = os.path.split(root)[1]
-files = []
-retval = []
-
-for entry in os.listdir(root):
-full_path = os.path.join(root, entry)
-if os.path.isdir(full_path):
-# Check if we're in a "spec/" direcotry
-if curdir == "spec" and not is_feature_directory_supported(entry):
-# The directory's tests aren't supported by the driver.
-print("Skipping spec/{}".format(entry))
-else:
-# recursively walk the subdirectory
-retval += walk_filter_dir_tree(full_path)
-elif os.path.isfile(full_path):
-# Add the file to the files list
-files += [entry]
-
-retval += [(root, [], files)]
-
-return retval
-
-
 # Find and add all apitrace tests.
 classifier = DriverClassifier()
 for basedir in [os.path.join(TESTS_DIR, 'apitrace', 'traces')]:
@@ -1680,28 +1598,25 @@ with profile.test_list.group_manager(
   '{}-textureSize-{}'.format(stage, sampler))
 
 # Group ARB_texture_gather
-if gl_extension_supported("GL_ARB_texture_gather"):
-with profile.test_list.group_manager(
-PiglitGLTest,
-grouptools.join('spec', 'ARB_texture_gather')) as g:
-stages = ['vs', 'fs']
-comps = ['r', 'rg', 'rgb', 'rgba']
-types = ['unorm', 'float', 'int', 'uint']
-samplers = 

[Piglit] [PATCH 02/35] test/piglit_test: add ROOT_DIR variable

[Dylan Baker]

This variable always points to the root of the piglit directory, which
contains the tests and generated_tests directories.
---
 framework/test/piglit_test.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index b6cec71..cc129ed 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -45,14 +45,16 @@ __all__ = [
 'PiglitGLTest',
 'PiglitBaseTest',
 'CL_CONCURRENT',
+'ROOT_DIR',
 'TEST_BIN_DIR',
 ]
 
 if 'PIGLIT_BUILD_DIR' in os.environ:
-TEST_BIN_DIR = os.path.join(os.environ['PIGLIT_BUILD_DIR'], 'bin')
+ROOT_DIR = os.environ['PIGLIT_BUILD_DIR']
 else:
-TEST_BIN_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__),
- '../../bin'))
+ROOT_DIR = os.path.normpath(os.path.join(os.path.dirname(__file__), 
'../..'))
+
+TEST_BIN_DIR = os.path.normpath(os.path.join(ROOT_DIR, 'bin'))
 
 CL_CONCURRENT = (not sys.platform.startswith('linux') or
  glob.glob('/dev/dri/render*'))
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 23/35] py_modules: Add support for out of tree builds

[Dylan Baker]

For out of tree builds we need to point at a TESTS_DIR that is in tree,
and a GENERATED_TESTS dir that is out of tree. At no other time do we
need to do this.
---
 tests/py_modules/constants.py | 8 +---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/py_modules/constants.py b/tests/py_modules/constants.py
index 94a41db..c9a7a41 100644
--- a/tests/py_modules/constants.py
+++ b/tests/py_modules/constants.py
@@ -32,9 +32,11 @@ TESTS_DIR = 
os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 # If the PIGLIT_BUILD_DIR env var is set, we want to look for the
 # generated tests in $PIGLIT_BUILD_DIR/generated_tests/.  Otherwise,
 # look in TESTS_DIR/../generated_tests.
-GENERATED_TESTS_DIR = os.path.abspath(os.path.join(
-os.environ.get('PIGLIT_BUILD_DIR', os.path.join(TESTS_DIR, '..')),
-'generated_tests'))
+build_dir = os.environ.get('PIGLIT_BUILD_TREE')
+if not build_dir:
+build_dir = os.environ.get('PIGLIT_BUILD_DIR', os.path.join(TESTS_DIR, 
'..'))
+
+GENERATED_TESTS_DIR = os.path.abspath(os.path.join(build_dir, 
'generated_tests'))
 
 # If on cygwin convert to a dos style path
 if sys.platform == 'cygwin':
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 10/35] framework: use a class method for building test via parsing

[Dylan Baker]

Since the serialized tests wont need this path, build a generic
constructor that this special constructor can leverage.
---
 framework/test/glsl_parser_test.py| 12 +++-
 tests/all.py  |  2 +-
 unittests/framework/test/test_glsl_parser_test.py | 48 
 3 files changed, 34 insertions(+), 28 deletions(-)

diff --git a/framework/test/glsl_parser_test.py 
b/framework/test/glsl_parser_test.py
index 6a06bc3..46344a8 100644
--- a/framework/test/glsl_parser_test.py
+++ b/framework/test/glsl_parser_test.py
@@ -272,11 +272,17 @@ class GLSLParserTest(FastSkipMixin, PiglitBaseTest):
 .tesc, .tese, .geom or .frag
 """
 
-def __init__(self, filepath):
-parsed = Parser(filepath)
+def __init__(self, command, gl_required=set(), glsl_version=None,
+ glsl_es_version=None, **kwargs):
 super(GLSLParserTest, self).__init__(
+command, run_concurrent=True, gl_required=gl_required,
+glsl_version=glsl_version, glsl_es_version=glsl_es_version)
+
+@classmethod
+def new(cls, filepath):
+parsed = Parser(filepath)
+return cls(
 parsed.command,
-run_concurrent=True,
 gl_required=parsed.gl_required,
 glsl_version=parsed.glsl_version,
 glsl_es_version=parsed.glsl_es_version)
diff --git a/tests/all.py b/tests/all.py
index a50a4c3..fa6b1c3 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -311,7 +311,7 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 continue
 elif ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
 try:
-test = GLSLParserTest(os.path.join(dirpath, filename))
+test = GLSLParserTest.new(os.path.join(dirpath, filename))
 except GLSLParserNoConfigError:
 # In the event that there is no config assume that it is a
 # legacy test, and continue
diff --git a/unittests/framework/test/test_glsl_parser_test.py 
b/unittests/framework/test/test_glsl_parser_test.py
index d252e27..b396fea 100644
--- a/unittests/framework/test/test_glsl_parser_test.py
+++ b/unittests/framework/test/test_glsl_parser_test.py
@@ -83,7 +83,7 @@ def test_no_config_start(tmpdir):
 // [end config]"""))
 
 with pytest.raises(glsl.GLSLParserNoConfigError):
-glsl.GLSLParserTest(six.text_type(p))
+glsl.GLSLParserTest.new(six.text_type(p))
 
 
 def test_find_config_start(tmpdir):
@@ -96,7 +96,7 @@ def test_find_config_start(tmpdir):
 // glsl_version: 1.10"""))
 
 with pytest.raises(exceptions.PiglitFatalError):
-glsl.GLSLParserTest(six.text_type(p))
+glsl.GLSLParserTest.new(six.text_type(p))
 
 
 def test_no_config_end(tmpdir):
@@ -106,7 +106,7 @@ def test_no_config_end(tmpdir):
 p.write('// [config]')
 
 with pytest.raises(exceptions.PiglitFatalError):
-glsl.GLSLParserTest(six.text_type(p))
+glsl.GLSLParserTest.new(six.text_type(p))
 
 
 def test_no_expect_result(tmpdir):
@@ -119,7 +119,7 @@ def test_no_expect_result(tmpdir):
 // [end config]"""))
 
 with pytest.raises(exceptions.PiglitFatalError):
-glsl.GLSLParserTest(six.text_type(p))
+glsl.GLSLParserTest.new(six.text_type(p))
 
 
 def test_no_glsl_version(tmpdir):
@@ -132,7 +132,7 @@ def test_no_glsl_version(tmpdir):
 // [end config]"""))
 
 with pytest.raises(exceptions.PiglitFatalError):
-glsl.GLSLParserTest(six.text_type(p))
+glsl.GLSLParserTest.new(six.text_type(p))
 
 
 def test_cpp_comments(tmpdir):
@@ -144,7 +144,7 @@ def test_cpp_comments(tmpdir):
 // expect_result: pass
 // glsl_version: 1.10
 // [end config]"""))
-test = glsl.GLSLParserTest(six.text_type(p))
+test = glsl.GLSLParserTest.new(six.text_type(p))
 
 assert test.command == [os.path.join(_TEST_BIN_DIR, 'glslparsertest'),
 six.text_type(p), 'pass', '1.10']
@@ -161,7 +161,7 @@ def test_c_comments(tmpdir):
  * [end config]
  */"""))
 
-test = glsl.GLSLParserTest(six.text_type(p))
+test = glsl.GLSLParserTest.new(six.text_type(p))
 
 assert test.command == [os.path.join(_TEST_BIN_DIR, 'glslparsertest'),
 six.text_type(p), 'pass', '1.10']
@@ -177,7 +177,7 @@ def test_blank_in_config_cpp(tmpdir):
 // expect_result: pass
 // glsl_version: 1.10
 // [end config]"""))
-test = glsl.GLSLParserTest(six.text_type(p))
+test = glsl.GLSLParserTest.new(six.text_type(p))
 
 assert test.command == [os.path.join(_TEST_BIN_DIR, 'glslparsertest'),
 six.text_type(p), 'pass', '1.10']
@@ -193,7 +193,7 @@ def test_empty_in_config_cpp(tmpdir):
 // expect_result: pass
 // glsl_version: 1.10
 // [end config]"""))
-test = 

[Piglit] [PATCH 17/35] tests/all: Make asmparser tests path relative

[Dylan Baker]

---
 tests/all.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/all.py b/tests/all.py
index e0c7311..6aa45f0 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -353,13 +353,14 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 'asmparsertest', os.path.relpath(dirpath, _basedir)))
 type_ = os.path.basename(dirpath)
 
+dirname = os.path.relpath(dirpath, ROOT_DIR)
 for filename in filenames:
 if not os.path.splitext(filename)[1] == '.txt':
 continue
 
 group = grouptools.join(base_group, filename)
 profile.test_list[group] = ASMParserTest(
-type_, os.path.join(dirpath, filename))
+type_, os.path.join(dirname, filename))
 
 # Find and add all apitrace tests.
 classifier = DriverClassifier()
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 26/35] shader_tests: correctly generate xml during out of tree builds

[Dylan Baker]

---
 framework/test/shader_test.py | 14 ++
 tests/shader.py   | 31 +++
 2 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/framework/test/shader_test.py b/framework/test/shader_test.py
index dbad16b..719b92f 100644
--- a/framework/test/shader_test.py
+++ b/framework/test/shader_test.py
@@ -167,12 +167,18 @@ class ShaderTest(FastSkipMixin, PiglitBaseTest):
 glsl_es_version=glsl_es_version)
 
 @classmethod
-def new(cls, filename):
+def new(cls, filename, installed_name=None):
+"""Parse an XML file and create a new instance.
+
+:param str filename: The name of the file to parse
+:param str installed_name: The relative path to the file when installed
+if not the same as the parsed name
+"""
 parser = Parser(filename)
 parser.parse()
 
 return cls(
-[parser.prog, parser.filename],
+[parser.prog, installed_name or filename],
 run_concurrent=True,
 gl_required=parser.gl_required,
 gl_version=parser.gl_version,
@@ -215,7 +221,7 @@ class MultiShaderTest(ReducedProcessMixin, PiglitBaseTest):
 self.skips = [FastSkip(**s) for s in skips]
 
 @classmethod
-def new(cls, filenames):
+def new(cls, filenames, installednames=None):
 # TODO
 assert filenames
 prog = None
@@ -257,7 +263,7 @@ class MultiShaderTest(ReducedProcessMixin, PiglitBaseTest):
 'glsl_es_version': parser.glsl_es_version,
 })
 
-return cls(prog, filenames, subtests, skips)
+return cls(prog, installednames or filenames, subtests, skips)
 
 def _process_skips(self):
 r_files = []
diff --git a/tests/shader.py b/tests/shader.py
index 0396c4c..3a43bcf 100644
--- a/tests/shader.py
+++ b/tests/shader.py
@@ -6,6 +6,7 @@ from __future__ import (
 import collections
 import os
 
+from six.moves import zip
 import six
 
 from framework.options import OPTIONS
@@ -22,17 +23,26 @@ shader_tests = collections.defaultdict(list)
 
 # Find and add all shader tests.
 basepath = os.path.normpath(os.path.join(TESTS_DIR, '..'))
+gen_basepath = os.path.relpath(os.path.join(GENERATED_TESTS_DIR, '..'), 
basepath)
+
 for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
+isgenerated = basedir == GENERATED_TESTS_DIR
 for dirpath, _, filenames in os.walk(basedir):
 groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
 for filename in filenames:
 testname, ext = os.path.splitext(filename)
 if ext == '.shader_test':
 dirname = os.path.relpath(dirpath, basepath)
+filepath = os.path.join(dirname, filename)
+if isgenerated:
+installpath = os.path.relpath(filepath, gen_basepath)
+else:
+installpath = None
+
 if OPTIONS.process_isolation:
-test = ShaderTest.new(os.path.join(dirname, filename))
+test = ShaderTest.new(filepath, installpath)
 else:
-shader_tests[groupname].append(os.path.join(dirname, 
filename))
+shader_tests[groupname].append((filepath, installpath))
 continue
 else:
 continue
@@ -46,11 +56,24 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 # this dictionary is constructed, then added to the actual test dictionary.
 for group, files in six.iteritems(shader_tests):
 assert group not in profile.test_list, 'duplicate group: {}'.format(group)
+
+# We'll end up with a list of tuples, split that into two lists
+files, installedfiles = list(zip(*files))
+files = list(files)
+installedfiles = list(installedfiles)
+
 # If there is only one file in the directory use a normal shader_test.
 # Otherwise use a MultiShaderTest
 if len(files) == 1:
 group = grouptools.join(
 group, os.path.basename(os.path.splitext(files[0])[0]))
-profile.test_list[group] = ShaderTest.new(files[0])
+profile.test_list[group] = ShaderTest.new(files[0], installedfiles[0])
 else:
-profile.test_list[group] = MultiShaderTest.new(files)
+if all(i is None for i in installedfiles):
+installedfiles = None
+else:
+for i, n in enumerate(installedfiles):
+if n is None:
+installedfiles[i] = files[i]
+
+profile.test_list[group] = MultiShaderTest.new(files, installedfiles)
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 13/35] framework/test/piglit_test: make cl_concurrency always a boolean

[Dylan Baker]

Just a nice cleanup.
---
 framework/test/piglit_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index 0986ab0..b5c9a53 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -56,8 +56,8 @@ else:
 
 TEST_BIN_DIR = os.path.normpath(os.path.join(ROOT_DIR, 'bin'))
 
-CL_CONCURRENT = (not sys.platform.startswith('linux') or
- glob.glob('/dev/dri/render*'))
+CL_CONCURRENT = bool(not sys.platform.startswith('linux') or
+ glob.glob('/dev/dri/render*'))
 
 
 class PiglitBaseTest(ValgrindMixin, Test):
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 29/35] fix asmparser test serialization for out of tree builds

[Dylan Baker]

---
 tests/glslparser.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/glslparser.py b/tests/glslparser.py
index b7de9f6..fe2b3c5 100644
--- a/tests/glslparser.py
+++ b/tests/glslparser.py
@@ -59,7 +59,7 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 'asmparsertest', os.path.relpath(dirpath, _basedir)))
 type_ = os.path.basename(dirpath)
 
-dirname = os.path.relpath(dirpath, ROOT_DIR)
+dirname = os.path.relpath(dirpath, os.path.join(basedir, '..'))
 for filename in filenames:
 if not os.path.splitext(filename)[1] == '.txt':
 continue
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 32/35] Add script to serialize profiles to XML

[Dylan Baker]

This script loads a python module and creates an XML profiles. It will
be used at build time to serialize the built-in profiles. It currently
cannot serialize integrated suites (such as dEQP or IGC), but it could
be extended to do so.
---
 tests/serializer.py | 149 +-
 1 file changed, 149 insertions(+)
 create mode 100644 tests/serializer.py

diff --git a/tests/serializer.py b/tests/serializer.py
new file mode 100644
index 000..58b3a14
--- /dev/null
+++ b/tests/serializer.py
@@ -0,0 +1,149 @@
+# encoding=utf-8
+# Copyright © 2018 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Script for taking profiles in python format and serializing them to XML."""
+
+import argparse
+import os
+import sys
+import xml.etree.cElementTree as et
+
+import six
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), 
'..'))
+
+from framework.test.piglit_test import (
+PiglitGLTest, PiglitCLTest, ASMParserTest, BuiltInConstantsTest,
+CLProgramTester,
+)
+from framework.test.shader_test import ShaderTest, MultiShaderTest
+from framework.test.glsl_parser_test import GLSLParserTest
+from framework.profile import load_test_profile
+from framework.options import OPTIONS
+
+
+def parser():
+"""Parse command line arguments."""
+parser = argparse.ArgumentParser()
+parser.add_argument('name')
+parser.add_argument('input')
+parser.add_argument('output')
+parser.add_argument('--no-process-isolation', action='store_true')
+parser.add_argument('--glsl-arb-compat', action='store_true')
+args = parser.parse_args()
+return args
+
+
+def _serialize_skips(test, elem):
+if getattr(test, 'gl_required', None):
+et.SubElement(elem, 'option', name='gl_required',
+  value=repr(test.gl_required))
+if getattr(test, 'gl_version', None):
+et.SubElement(elem, 'option', name='gl_version',
+  value=repr(test.gl_version))
+if getattr(test, 'gles_version', None):
+et.SubElement(elem, 'option', name='gles_version',
+  value=repr(test.gles_version))
+if getattr(test, 'glsl_version', None):
+et.SubElement(elem, 'option', name='glsl_version',
+  value=repr(test.glsl_version))
+if getattr(test, 'glsl_es_version', None):
+et.SubElement(elem, 'option', name='glsl_es_version',
+  value=repr(test.glsl_es_version))
+
+
+def serializer(name, profile, outfile):
+"""Take each test in the profile and write it out into the xml."""
+# TODO: This is going to take a lot of memory
+root = et.Element('PiglitTestList', count=six.text_type(len(profile)),
+  name=name)
+for name, test in profile.itertests():
+if isinstance(test, PiglitGLTest):
+elem = et.SubElement(root, 'Test', type='gl', name=name)
+if test.require_platforms:
+et.SubElement(elem, 'option', name='require_platforms',
+  value=repr(test.require_platforms))
+if test.exclude_platforms:
+et.SubElement(elem, 'option', name='exclude_platforms',
+  value=repr(test.exclude_platforms))
+_serialize_skips(test, elem)
+elif isinstance(test, BuiltInConstantsTest):
+elem = et.SubElement(root, 'Test', type='gl_builtin', name=name)
+elif isinstance(test, GLSLParserTest):
+elem = et.SubElement(root, 'Test', type='glsl_parser', name=name)
+_serialize_skips(test, elem)
+elif isinstance(test, ASMParserTest):
+elem = et.SubElement(root, 'Test', type='asm_parser', name=name)
+et.SubElement(elem, 'option', name='type_',
+  value=repr(test.command[1]))
+et.SubElement(elem, 'option', name='filename',
+  

[Piglit] [PATCH 16/35] framework/test: use relative paths for GLSLParser tests

[Dylan Baker]

---
 framework/test/glsl_parser_test.py | 13 ++---
 tests/all.py   |  4 ++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/framework/test/glsl_parser_test.py 
b/framework/test/glsl_parser_test.py
index 46344a8..35eadfe 100644
--- a/framework/test/glsl_parser_test.py
+++ b/framework/test/glsl_parser_test.py
@@ -33,7 +33,7 @@ import six
 from framework import exceptions
 from .base import TestIsSkip
 from .opengl import FastSkipMixin
-from .piglit_test import PiglitBaseTest, TEST_BIN_DIR
+from .piglit_test import PiglitBaseTest, TEST_BIN_DIR, ROOT_DIR
 
 __all__ = [
 'GLSLParserTest',
@@ -95,11 +95,12 @@ class Parser(object):
 self.gl_required = set()
 self.glsl_es_version = None
 self.glsl_version = None
+abs_filepath = os.path.join(ROOT_DIR, filepath)
 
 try:
-with io.open(filepath, mode='r', encoding='utf-8') as testfile:
+with io.open(abs_filepath, mode='r', encoding='utf-8') as testfile:
 testfile = testfile.read()
-self.config = self.parse(testfile, filepath)
+self.config = self.parse(testfile, abs_filepath)
 self.command = self.get_command(filepath)
 except GLSLParserInternalError as e:
 raise exceptions.PiglitFatalError(
@@ -278,6 +279,12 @@ class GLSLParserTest(FastSkipMixin, PiglitBaseTest):
 command, run_concurrent=True, gl_required=gl_required,
 glsl_version=glsl_version, glsl_es_version=glsl_es_version)
 
+@PiglitBaseTest.command.getter
+def command(self):
+command = super(GLSLParserTest, self).command
+glslfile = os.path.join(ROOT_DIR, command[1])
+return [command[0], glslfile] + command[2:]
+
 @classmethod
 def new(cls, filepath):
 parsed = Parser(filepath)
diff --git a/tests/all.py b/tests/all.py
index cf6ee5a..e0c7311 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -305,8 +305,8 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
 for filename in filenames:
 testname, ext = os.path.splitext(filename)
+dirname = os.path.relpath(dirpath, ROOT_DIR)
 if ext == '.shader_test':
-dirname = os.path.relpath(dirpath, ROOT_DIR)
 if PROCESS_ISOLATION:
 test = ShaderTest.new(os.path.join(dirname, filename))
 else:
@@ -314,7 +314,7 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 continue
 elif ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
 try:
-test = GLSLParserTest.new(os.path.join(dirpath, filename))
+test = GLSLParserTest.new(os.path.join(dirname, filename))
 except GLSLParserNoConfigError:
 # In the event that there is no config assume that it is a
 # legacy test, and continue
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 14/35] framework/test: Add class for cl-program-tester

[Dylan Baker]

These tests need some special handling for serializing, since they take
file paths as arguments.
---
 framework/test/piglit_test.py | 35 +--
 tests/cl.py   |  6 +++---
 2 files changed, 28 insertions(+), 13 deletions(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index b5c9a53..f5fd0c8 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -181,16 +181,6 @@ class PiglitGLTest(WindowResizeMixin, PiglitBaseTest):
 self._command = [n for n in new if n not in ['-auto', '-fbo']]
 
 
-class PiglitCLTest(PiglitBaseTest):  # pylint: disable=too-few-public-methods
-""" OpenCL specific Test class.
-
-Set concurrency based on CL requirements.
-
-"""
-def __init__(self, command, run_concurrent=CL_CONCURRENT, **kwargs):
-super(PiglitCLTest, self).__init__(command, run_concurrent, **kwargs)
-
-
 class ASMParserTest(PiglitBaseTest):
 
 """Test class for ASM parser tests."""
@@ -208,3 +198,28 @@ class ASMParserTest(PiglitBaseTest):
 class BuiltInConstantsTest(PiglitBaseTest):
 
 """Test class for handling built in constants tests."""
+
+
+class PiglitCLTest(PiglitBaseTest):  # pylint: disable=too-few-public-methods
+""" OpenCL specific Test class.
+
+Set concurrency based on CL requirements.
+
+"""
+def __init__(self, command, run_concurrent=CL_CONCURRENT, **kwargs):
+super(PiglitCLTest, self).__init__(command, run_concurrent, **kwargs)
+
+
+class CLProgramTester(PiglitCLTest):
+
+"""Class for cl-program-tester tests."""
+
+def __init__(self, filename):
+super(CLProgramTester, self).__init__(['cl-program-tester'])
+self.filename = filename
+
+@PiglitCLTest.command.getter
+def command(self):
+command = super(CLProgramTester, self).command
+command.insert(1, self.filename)
+return command
diff --git a/tests/cl.py b/tests/cl.py
index 24febe5..2ab3502 100644
--- a/tests/cl.py
+++ b/tests/cl.py
@@ -14,7 +14,7 @@ from __future__ import (
 import os
 
 from framework.profile import TestProfile
-from framework.test import PiglitCLTest
+from framework.test.piglit_test import PiglitCLTest, CLProgramTester
 from framework import grouptools
 from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
 
@@ -110,8 +110,8 @@ def add_program_test_dir(group, dirpath):
 if ext not in ['.cl', '.program_test']:
 continue
 
-profile.test_list[grouptools.join(group, testname)] = PiglitCLTest(
-['cl-program-tester', os.path.join(dirpath, filename)])
+profile.test_list[grouptools.join(group, testname)] = CLProgramTester(
+os.path.join(dirpath, filename))
 
 
 add_program_test_dir(grouptools.join('program', 'build'),
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 03/35] framework/profile: Allow a group manager class to be overwritten

[Dylan Baker]

This is useful for classes like the coming BuiltInConstantsTest class,
where just one or two tests need to change the test class.
---
 framework/profile.py | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/framework/profile.py b/framework/profile.py
index d77ac09..b6deed0 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -205,7 +205,7 @@ class TestDict(collections.MutableMapping):
 """
 assert isinstance(group, six.string_types), type(group)
 
-def adder(args, name=None, **kwargs):
+def adder(args, name=None, override_class=None, **kwargs):
 """Helper function that actually adds the tests.
 
 Arguments:
@@ -228,7 +228,9 @@ class TestDict(collections.MutableMapping):
 assert isinstance(name, six.string_types)
 lgroup = grouptools.join(group, name)
 
-self[lgroup] = test_class(
+class_ = override_class or test_class
+
+self[lgroup] = class_(
 args,
 **dict(itertools.chain(six.iteritems(default_args),
six.iteritems(kwargs
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 11/35] framework: do the same for shader test

[Dylan Baker]

---
 framework/test/shader_test.py| 46 +++--
 tests/all.py |  4 +-
 unittests/framework/test/test_shader_test.py | 20 -
 3 files changed, 38 insertions(+), 32 deletions(-)

diff --git a/framework/test/shader_test.py b/framework/test/shader_test.py
index a3fb8e2..64edb58 100644
--- a/framework/test/shader_test.py
+++ b/framework/test/shader_test.py
@@ -154,26 +154,31 @@ class ShaderTest(FastSkipMixin, PiglitBaseTest):
 
 """
 
-def __init__(self, filename):
-if bool(os.environ.get('PIGLIT_NO_FAST_SKIP', False)):
-# No need to parse the shader test file if we've disabled
-# the FastSkip feature.
-super(ShaderTest, self).__init__(
-['shader_runner', filename],
-run_concurrent=True,)
-else:
-parser = Parser(filename)
-parser.parse()
-
-super(ShaderTest, self).__init__(
-[parser.prog, parser.filename],
-run_concurrent=True,
-gl_required=parser.gl_required,
-gl_version=parser.gl_version,
-gles_version=parser.gles_version,
-glsl_version=parser.glsl_version,
-glsl_es_version=parser.glsl_es_version)
-
+def __init__(self, command, gl_required=set(), gl_version=None,
+ gles_version=None, glsl_version=None, glsl_es_version=None,
+ **kwargs):
+super(ShaderTest, self).__init__(
+command,
+run_concurrent=True,
+gl_required=gl_required,
+gl_version=gl_version,
+gles_version=gles_version,
+glsl_version=glsl_version,
+glsl_es_version=glsl_es_version)
+
+@classmethod
+def new(cls, filename):
+parser = Parser(filename)
+parser.parse()
+
+return cls(
+[parser.prog, parser.filename],
+run_concurrent=True,
+gl_required=parser.gl_required,
+gl_version=parser.gl_version,
+gles_version=parser.gles_version,
+glsl_version=parser.glsl_version,
+glsl_es_version=parser.glsl_es_version)
 
 @PiglitBaseTest.command.getter
 def command(self):
@@ -197,6 +202,7 @@ class MultiShaderTest(ReducedProcessMixin, PiglitBaseTest):
 """
 
 def __init__(self, filenames):
+# TODO
 assert filenames
 prog = None
 files = []
diff --git a/tests/all.py b/tests/all.py
index fa6b1c3..cbbe7d7 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -305,7 +305,7 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 testname, ext = os.path.splitext(filename)
 if ext == '.shader_test':
 if PROCESS_ISOLATION:
-test = ShaderTest(os.path.join(dirpath, filename))
+test = ShaderTest.new(os.path.join(dirpath, filename))
 else:
 shader_tests[groupname].append(os.path.join(dirpath, 
filename))
 continue
@@ -337,7 +337,7 @@ for group, files in six.iteritems(shader_tests):
 if len(files) == 1:
 group = grouptools.join(
 group, os.path.basename(os.path.splitext(files[0])[0]))
-profile.test_list[group] = ShaderTest(files[0])
+profile.test_list[group] = ShaderTest.new(files[0])
 else:
 profile.test_list[group] = MultiShaderTest(files)
 
diff --git a/unittests/framework/test/test_shader_test.py 
b/unittests/framework/test/test_shader_test.py
index 1637231..c7a8da2 100644
--- a/unittests/framework/test/test_shader_test.py
+++ b/unittests/framework/test/test_shader_test.py
@@ -93,7 +93,7 @@ class TestConfigParsing(object):
 
 [next section]
 """.format(operator, gles)))
-test = shader_test.ShaderTest(six.text_type(p))
+test = shader_test.ShaderTest.new(six.text_type(p))
 
 assert os.path.basename(test.command[0]) == expected
 
@@ -105,7 +105,7 @@ class TestConfigParsing(object):
 GL ES >= 3.0
 GLSL ES >= 3.00 es
 """))
-test = shader_test.ShaderTest(six.text_type(p))
+test = shader_test.ShaderTest.new(six.text_type(p))
 
 assert os.path.basename(test.command[0]) == "shader_runner_gles3"
 
@@ -117,7 +117,7 @@ class TestConfigParsing(object):
 GL >= 3.0
 GL_ARB_ham_sandwhich
 """))
-test = shader_test.ShaderTest(six.text_type(p))
+test = shader_test.ShaderTest.new(six.text_type(p))
 
 assert test.gl_required == {'GL_ARB_ham_sandwhich'}
 
@@ -129,7 +129,7 @@ class TestConfigParsing(object):
 GL >= 2.0
 GL_ARB_ham_sandwhich
 """))
-test = shader_test.ShaderTest(six.text_type(p))
+test = shader_test.ShaderTest.new(six.text_type(p))
 
 assert test.gl_version == 2.0
 
@@ -141,7 +141,7 @@ class 

[Piglit] [PATCH 07/35] framework: Use custom class for ASM parser tests

[Dylan Baker]

This is going to make serialization easier, since asmparser tests take a
file argument, which must be relative at build time, but absolute at run
time.
---
 framework/test/piglit_test.py | 14 ++
 tests/all.py  |  6 --
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index ef3bfdc..c53c3e5 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -189,3 +189,17 @@ class PiglitCLTest(PiglitBaseTest):  # pylint: 
disable=too-few-public-methods
 """
 def __init__(self, command, run_concurrent=CL_CONCURRENT, **kwargs):
 super(PiglitCLTest, self).__init__(command, run_concurrent, **kwargs)
+
+
+class ASMParserTest(PiglitBaseTest):
+
+"""Test class for ASM parser tests."""
+
+def __init__(self, type_, filename):
+super(ASMParserTest, self).__init__(['asmparsertest', type_])
+self.filename = filename
+
+@PiglitBaseTest.command.getter
+def command(self):
+command = super(ASMParserTest, self).command
+return command + [os.path.join(ROOT_DIR, self.filename)]
diff --git a/tests/all.py b/tests/all.py
index 7c8580e..fe4a14c 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -21,6 +21,7 @@ from framework.driver_classifier import DriverClassifier
 from framework.test import (PiglitGLTest, PiglitBaseTest,
 GLSLParserTest, GLSLParserNoConfigError)
 from framework.test.shader_test import ShaderTest, MultiShaderTest
+from framework.test.piglit_test import ASMParserTest
 from .py_modules.constants import TESTS_DIR, GENERATED_TESTS_DIR
 
 __all__ = ['profile']
@@ -340,6 +341,7 @@ for group, files in six.iteritems(shader_tests):
 else:
 profile.test_list[group] = MultiShaderTest(files)
 
+
 # Collect and add all asmparsertests
 for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 _basedir = os.path.join(basedir, 'asmparsertest', 'shaders')
@@ -353,8 +355,8 @@ for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
 continue
 
 group = grouptools.join(base_group, filename)
-profile.test_list[group] = PiglitGLTest(
-['asmparsertest', type_, os.path.join(dirpath, filename)])
+profile.test_list[group] = ASMParserTest(
+type_, os.path.join(dirpath, filename))
 
 # Find and add all apitrace tests.
 classifier = DriverClassifier()
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 35/35] profile: use gz to compress profiles

[Dylan Baker]

This results in substantially smaller profiles and doesn't seem to
affect runtime.
---
 CMakeLists.txt  | 4 ++--
 framework/profile.py| 8 
 tests/CMakeLists.no_api.txt | 6 +++---
 tests/serializer.py | 4 +++-
 4 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index db047b4..2e870d2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -533,14 +533,14 @@ install (
 install (
DIRECTORY tests
DESTINATION ${PIGLIT_INSTALL_LIBDIR}
-   FILES_MATCHING REGEX 
".*\\.(xml|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
+   FILES_MATCHING REGEX 
".*\\.(xml.gz|py|program_test|shader_test|frag|vert|geom|tesc|tese|comp|ktx|cl|txt|inc)$"
REGEX 
"CMakeFiles|CMakeLists|serializer.py|opengl.py|cl.py|quick_gl.py|glslparser.py|shader.py|quick_shader.py|no_error.py|llvmpipe_gl.py|sanity.py"
 EXCLUDE
 )
 
 install (
DIRECTORY ${CMAKE_BINARY_DIR}/tests
DESTINATION ${PIGLIT_INSTALL_LIBDIR}
-   FILES_MATCHING REGEX ".*\\.xml"
+   FILES_MATCHING REGEX ".*\\.xml.gz"
 )
 
 install (
diff --git a/framework/profile.py b/framework/profile.py
index 3975d06..7eefeeb 100644
--- a/framework/profile.py
+++ b/framework/profile.py
@@ -34,8 +34,8 @@ import ast
 import collections
 import contextlib
 import copy
+import gzip
 import importlib
-import io
 import itertools
 import multiprocessing
 import multiprocessing.dummy
@@ -318,7 +318,7 @@ class XMLProfile(object):
 
 def __len__(self):
 if not (self.filters or self.forced_test_list):
-with io.open(self.filename, 'rt') as f:
+with gzip.open(self.filename, 'rt') as f:
 iter_ = et.iterparse(f, events=(b'start', ))
 for _, elem in iter_:
 if elem.tag == 'PiglitTestList':
@@ -333,7 +333,7 @@ class XMLProfile(object):
 
 def _itertests(self):
 """Always iterates tests instead of using the forced test_list."""
-with io.open(self.filename, 'rt') as f:
+with gzip.open(self.filename, 'rt') as f:
 doc = et.iterparse(f, events=(b'end', ))
 _, root = next(doc)  # get the root so we can keep clearing it
 for _, e in doc:
@@ -517,7 +517,7 @@ def load_test_profile(filename, python=None):
 if os.path.exists(meta):
 return MetaProfile(meta)
 
-xml = os.path.join(ROOT_DIR, 'tests', name + '.xml')
+xml = os.path.join(ROOT_DIR, 'tests', name + '.xml.gz')
 if os.path.exists(xml):
 return XMLProfile(xml)
 
diff --git a/tests/CMakeLists.no_api.txt b/tests/CMakeLists.no_api.txt
index d7f95af..3ffba74 100644
--- a/tests/CMakeLists.no_api.txt
+++ b/tests/CMakeLists.no_api.txt
@@ -45,14 +45,14 @@ add_custom_target(
 
 function(piglit_generate_xml name profile meta_target extra_args)
add_custom_command(
-   OUTPUT ${CMAKE_BINARY_DIR}/tests/${name}.xml
-   COMMAND ${CMAKE_COMMAND} -E env 
PIGLIT_BUILD_TREE=${CMAKE_BINARY_DIR} ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${name} 
${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py ${CMAKE_BINARY_DIR}/tests/${name}.xml 
 ${extra_args}
+   OUTPUT ${CMAKE_BINARY_DIR}/tests/${name}.xml.gz
+   COMMAND ${CMAKE_COMMAND} -E env 
PIGLIT_BUILD_TREE=${CMAKE_BINARY_DIR} ${PYTHON_EXECUTABLE} 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${name} 
${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py 
${CMAKE_BINARY_DIR}/tests/${name}.xml.gz  ${extra_args}
DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${profile}.py 
${CMAKE_CURRENT_SOURCE_DIR}/serializer.py ${ARGN}
VERBATIM
)
add_custom_target(
generate-${name}-xml
-   DEPENDS ${CMAKE_BINARY_DIR}/tests/${name}.xml
+   DEPENDS ${CMAKE_BINARY_DIR}/tests/${name}.xml.gz
)
add_dependencies(${meta_target} generate-${name}-xml)
 endfunction()
diff --git a/tests/serializer.py b/tests/serializer.py
index 58b3a14..32beb42 100644
--- a/tests/serializer.py
+++ b/tests/serializer.py
@@ -22,6 +22,7 @@
 """Script for taking profiles in python format and serializing them to XML."""
 
 import argparse
+import gzip
 import os
 import sys
 import xml.etree.cElementTree as et
@@ -133,7 +134,8 @@ def serializer(name, profile, outfile):
 et.SubElement(env, 'env', name=k, value=v)
 
 tree = et.ElementTree(root)
-tree.write(outfile, encoding='utf-8', xml_declaration=True)
+with gzip.open(outfile, 'wb') as f:
+tree.write(f, encoding='utf-8', xml_declaration=True)
 
 
 def main():
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 33/35] tests: Add script to find all hand written test files

[Dylan Baker]

Including asmparser, glslparser, shader, and program tests, which will
be used by cmake to regenerate the appropriate xml whenever a new test
is added.
---
 tests/find_static_tests.py | 74 +++-
 1 file changed, 74 insertions(+)
 create mode 100644 tests/find_static_tests.py

diff --git a/tests/find_static_tests.py b/tests/find_static_tests.py
new file mode 100644
index 000..2152731
--- /dev/null
+++ b/tests/find_static_tests.py
@@ -0,0 +1,74 @@
+# encoding=utf-8
+# Copyright © 2018 Intel Corporation
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Script that finds all static tests of one kind or another."""
+
+from __future__ import (
+absolute_import, division, print_function, unicode_literals
+)
+import argparse
+import io
+import os
+
+
+def main():
+parser = argparse.ArgumentParser()
+parser.add_argument('sourcedir')
+parser.add_argument(
+'mode',
+choices=['shader', 'glslparser', 'asmparser', 'program'])
+parser.add_argument('output')
+args = parser.parse_args()
+
+if args.mode == 'asmparser':
+exts = ['.txt']
+directory = os.path.join(args.sourcedir, 'asmparsertest', 'shaders')
+elif args.mode == 'glslparser':
+exts = ['.frag', '.vert', '.goem', '.tess', 'tesc', '.comp']
+directory = args.sourcedir
+elif args.mode == 'shader':
+exts = ['.shader_test']
+directory = args.sourcedir
+elif args.mode == 'program':
+exts = ['.program_test']
+directory = args.sourcedir
+
+files = []
+for dirpath, _, filenames in os.walk(directory):
+for filename in filenames:
+if os.path.splitext(filename)[1] in exts:
+files.append(os.path.join(dirpath, filename))
+
+if os.path.exists(args.output):
+with io.open(args.output, 'rt', encoding='utf-8') as f:
+existing = f.read().rstrip().split('\n')
+else:
+existing = []
+
+if sorted(files) != sorted(existing):
+with io.open(args.output, 'wt', encoding='utf-8') as f:
+for filename in files:
+f.write(filename)
+f.write('\n')
+
+
+if __name__ == '__main__':
+main()
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 12/35] framework/test: Split multishader too

[Dylan Baker]

---
 framework/test/shader_test.py| 63 ++---
 tests/all.py |  2 +-
 unittests/framework/test/test_shader_test.py | 17 +-
 3 files changed, 59 insertions(+), 23 deletions(-)

diff --git a/framework/test/shader_test.py b/framework/test/shader_test.py
index 64edb58..3dc2c39 100644
--- a/framework/test/shader_test.py
+++ b/framework/test/shader_test.py
@@ -201,11 +201,22 @@ class MultiShaderTest(ReducedProcessMixin, 
PiglitBaseTest):
 filenames -- a list of absolute paths to shader test files
 """
 
-def __init__(self, filenames):
+def __init__(self, prog, files, subtests, skips):
+super(MultiShaderTest, self).__init__(
+[prog] + files,
+subtests=subtests,
+run_concurrent=True)
+
+self.prog = prog
+self.files = files
+self.subtests = subtests
+self.skips = [FastSkip(**s) for s in skips]
+
+@classmethod
+def new(cls, filenames):
 # TODO
 assert filenames
 prog = None
-files = []
 subtests = []
 skips = []
 
@@ -215,7 +226,7 @@ class MultiShaderTest(ReducedProcessMixin, PiglitBaseTest):
 for each in filenames:
 parser = Parser(each)
 parser.parse()
-subtest = os.path.basename(os.path.splitext(each)[0]).lower()
+
subtests.append(os.path.basename(os.path.splitext(each)[0]).lower())
 
 if prog is not None:
 # This allows mixing GLES2 and GLES3 shader test files
@@ -236,30 +247,42 @@ class MultiShaderTest(ReducedProcessMixin, 
PiglitBaseTest):
 else:
 prog = parser.prog
 
+skips.append({
+'gl_required': parser.gl_required,
+'gl_version': parser.gl_version,
+'glsl_version': parser.glsl_version,
+'gles_version': parser.gles_version,
+'glsl_es_version': parser.glsl_es_version,
+})
+
+return cls(prog, filenames, subtests, skips)
+
+def _process_skips(self):
+r_files = []
+r_subtests = []
+r_skips = []
+for f, s, k in zip(self.files, self.subtests, self.skips):
 try:
-skipper = FastSkip(gl_required=parser.gl_required,
-   gl_version=parser.gl_version,
-   gles_version=parser.gles_version,
-   glsl_version=parser.glsl_version,
-   glsl_es_version=parser.glsl_es_version)
-skipper.test()
+k.test()
 except TestIsSkip:
-skips.append(subtest)
-continue
-files.append(parser.filename)
-subtests.append(subtest)
+r_skips.append(s)
+else:
+r_files.append(f)
+r_subtests.append(s)
 
-assert len(subtests) + len(skips) == len(filenames), \
+assert len(r_subtests) + len(r_skips) == len(self.files), \
 'not all tests accounted for'
 
-super(MultiShaderTest, self).__init__(
-[prog] + files,
-subtests=subtests,
-run_concurrent=True)
-
-for name in skips:
+for name in r_skips:
 self.result.subtests[name] = status.SKIP
 
+self._expected = r_subtests
+self._command = [self._command[0]] + r_files
+
+def run(self):
+self._process_skips()
+super(MultiShaderTest, self).run()
+
 @PiglitBaseTest.command.getter  # pylint: disable=no-member
 def command(self):
 """Add -auto to the test command."""
diff --git a/tests/all.py b/tests/all.py
index cbbe7d7..6a2b990 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -339,7 +339,7 @@ for group, files in six.iteritems(shader_tests):
 group, os.path.basename(os.path.splitext(files[0])[0]))
 profile.test_list[group] = ShaderTest.new(files[0])
 else:
-profile.test_list[group] = MultiShaderTest(files)
+profile.test_list[group] = MultiShaderTest.new(files)
 
 
 # Collect and add all asmparsertests
diff --git a/unittests/framework/test/test_shader_test.py 
b/unittests/framework/test/test_shader_test.py
index c7a8da2..49caecb 100644
--- a/unittests/framework/test/test_shader_test.py
+++ b/unittests/framework/test/test_shader_test.py
@@ -33,6 +33,7 @@ except ImportError:
 import pytest
 import six
 
+from framework import status
 from framework.test import shader_test
 
 # pylint: disable=invalid-name,no-self-use,protected-access
@@ -242,7 +243,7 @@ class TestMultiShaderTest(object):
 
 [vertex shader]"""))
 
-return shader_test.MultiShaderTest(
+return shader_test.MultiShaderTest.new(
 [six.text_type(one), six.text_type(two)])
 
 def test_prog(self, inst):
@@ -268,10 +269,11 @@ class 

[Piglit] [PATCH 28/35] fix glslparser test for out of tree builds

[Dylan Baker]

---
 framework/test/glsl_parser_test.py | 19 +--
 tests/glslparser.py| 13 +++--
 2 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/framework/test/glsl_parser_test.py 
b/framework/test/glsl_parser_test.py
index 88646c9..8840c2d 100644
--- a/framework/test/glsl_parser_test.py
+++ b/framework/test/glsl_parser_test.py
@@ -89,7 +89,7 @@ class Parser(object):
 _CONFIG_KEYS = frozenset(['expect_result', 'glsl_version',
   'require_extensions', 'check_link'])
 
-def __init__(self, filepath):
+def __init__(self, filepath, installpath=None):
 # a set that stores a list of keys that have been found already
 self.__found_keys = set()
 self.gl_required = set()
@@ -101,7 +101,7 @@ class Parser(object):
 with io.open(abs_filepath, mode='r', encoding='utf-8') as testfile:
 testfile = testfile.read()
 self.config = self.parse(testfile, abs_filepath)
-self.command = self.get_command(filepath)
+self.command = self.get_command(filepath, installpath)
 except GLSLParserInternalError as e:
 raise exceptions.PiglitFatalError(
 'In file "{}":\n{}'.format(filepath, six.text_type(e)))
@@ -153,7 +153,7 @@ class Parser(object):
 else:
 return 'glslparsertest'
 
-def get_command(self, filepath):
+def get_command(self, filepath, installpath):
 """ Create the command argument to pass to super()
 
 This private helper creates a configparser object, then reads in the
@@ -172,7 +172,7 @@ class Parser(object):
 glsl = self.config['glsl_version']
 command = [
 self.pick_binary(glsl),
-filepath,
+installpath or filepath,
 self.config['expect_result'],
 self.config['glsl_version']
 ]
@@ -282,8 +282,15 @@ class GLSLParserTest(FastSkipMixin, PiglitBaseTest):
 return [command[0], glslfile] + command[2:]
 
 @classmethod
-def new(cls, filepath):
-parsed = Parser(filepath)
+def new(cls, filepath, installpath=None):
+"""Parse a file and create an instance.
+
+:param str filepath: the file to parse
+:param Optional[str] installpath:
+The relative path the file will be isntalled to if different than
+filepath
+"""
+parsed = Parser(filepath, installpath)
 return cls(
 parsed.command,
 gl_required=parsed.gl_required,
diff --git a/tests/glslparser.py b/tests/glslparser.py
index 9d50dab..b7de9f6 100644
--- a/tests/glslparser.py
+++ b/tests/glslparser.py
@@ -17,15 +17,24 @@ profile = TestProfile()
 
 # Find and add all shader tests.
 basepath = os.path.normpath(os.path.join(TESTS_DIR, '..'))
+gen_basepath = os.path.relpath(os.path.join(GENERATED_TESTS_DIR, '..'), 
basepath)
+
 for basedir in [TESTS_DIR, GENERATED_TESTS_DIR]:
+isgenerated = basedir == GENERATED_TESTS_DIR
 for dirpath, _, filenames in os.walk(basedir):
 groupname = grouptools.from_path(os.path.relpath(dirpath, basedir))
 for filename in filenames:
 testname, ext = os.path.splitext(filename)
 if ext in ['.vert', '.tesc', '.tese', '.geom', '.frag', '.comp']:
+dirname = os.path.relpath(dirpath, basepath)
+filepath = os.path.join(dirname, filename)
+if isgenerated:
+installpath = os.path.relpath(filepath, gen_basepath)
+else:
+installpath = None
+
 try:
-test = GLSLParserTest.new(
-os.path.join(os.path.relpath(dirpath, basepath), 
filename))
+test = GLSLParserTest.new(filepath, installpath)
 except GLSLParserNoConfigError:
 # In the event that there is no config assume that it is a
 # legacy test, and continue
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] tests: add nvidia conservative rasterization tests

[Dylan Baker]

Quoting Pending Chaos (2018-03-31 03:13:58)
> GL_NV_conservative_raster, GL_NV_conservative_raster_dilate and
> GL_NV_conservative_raster_pre_snap_triangles are three separate extensions.

Thanks for clarifying :)

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] tests: add nvidia conservative rasterization tests

[Dylan Baker]

Quoting Rhys Perry (2018-03-22 06:43:25)
> Adds tests for GL_NV_conservative_raster, GL_NV_conservative_raster_dilate
> and GL_NV_conservative_raster_pre_snap_triangles.
> ---
>  tests/all.py   |  20 ++
>  tests/spec/CMakeLists.txt  |   3 +
>  .../spec/nv_conservative_raster/CMakeLists.gl.txt  |  13 +
>  .../nv_conservative_raster/CMakeLists.gles2.txt|   3 +
>  tests/spec/nv_conservative_raster/CMakeLists.txt   |   1 +
>  tests/spec/nv_conservative_raster/attrib.c |  69 +
>  tests/spec/nv_conservative_raster/dlist.c  |  75 ++
>  tests/spec/nv_conservative_raster/draw.c   | 288 
> +
>  .../CMakeLists.gl.txt  |  11 +
>  .../CMakeLists.gles2.txt   |   3 +
>  .../nv_conservative_raster_dilate/CMakeLists.txt   |   1 +
>  tests/spec/nv_conservative_raster_dilate/draw.c| 128 +
>  .../CMakeLists.gl.txt  |  11 +
>  .../CMakeLists.gles2.txt   |   3 +
>  .../CMakeLists.txt |   1 +
>  .../draw.c | 117 +
>  16 files changed, 747 insertions(+)
>  create mode 100644 tests/spec/nv_conservative_raster/CMakeLists.gl.txt
>  create mode 100644 tests/spec/nv_conservative_raster/CMakeLists.gles2.txt
>  create mode 100644 tests/spec/nv_conservative_raster/CMakeLists.txt
>  create mode 100644 tests/spec/nv_conservative_raster/attrib.c
>  create mode 100644 tests/spec/nv_conservative_raster/dlist.c
>  create mode 100644 tests/spec/nv_conservative_raster/draw.c
>  create mode 100644 tests/spec/nv_conservative_raster_dilate/CMakeLists.gl.txt
>  create mode 100644 
> tests/spec/nv_conservative_raster_dilate/CMakeLists.gles2.txt
>  create mode 100644 tests/spec/nv_conservative_raster_dilate/CMakeLists.txt
>  create mode 100644 tests/spec/nv_conservative_raster_dilate/draw.c
>  create mode 100644 
> tests/spec/nv_conservative_raster_pre_snap_triangles/CMakeLists.gl.txt
>  create mode 100644 
> tests/spec/nv_conservative_raster_pre_snap_triangles/CMakeLists.gles2.txt
>  create mode 100644 
> tests/spec/nv_conservative_raster_pre_snap_triangles/CMakeLists.txt
>  create mode 100644 
> tests/spec/nv_conservative_raster_pre_snap_triangles/draw.c
> 
> diff --git a/tests/all.py b/tests/all.py
> index 4cd911fab..a5da80f14 100644
> --- a/tests/all.py
> +++ b/tests/all.py
> @@ -4984,5 +4984,25 @@ with profile.test_list.group_manager(
>  g(['arb_bindless_texture-uint64_attribs'], 'uint64_attribs')
>  g(['arb_bindless_texture-uniform'], 'uniform')
>  
> +# Group NV_conservative_raster
> +with profile.test_list.group_manager(
> +   PiglitGLTest,
> +   grouptools.join('spec', 'NV_conservative_raster')) as g:
> +g(['nv_conservative_raster-draw'], 'draw')
> +g(['nv_conservative_raster-dlist'], 'dlist')
> +g(['nv_conservative_raster-attrib'], 'attrib')
> +
> +# Group NV_conservative_raster_dilate
> +with profile.test_list.group_manager(
> +   PiglitGLTest,
> +   grouptools.join('spec', 'NV_conservative_raster_dilate')) as g:
> +g(['nv_conservative_raster_dilate-draw'], 'draw')
> +
> +# Group NV_conservative_raster_pre_snap_triangles
> +with profile.test_list.group_manager(
> +   PiglitGLTest,
> +   grouptools.join('spec', 'NV_conservative_raster_pre_snap_triangles')) 
> as g:
> +g(['nv_conservative_raster_pre_snap_triangles-draw'], 'draw')

I'm asking because I don't know the answer, but are these three separate
extensions? Usually we don't create separate groups for the same extension. We
might have sub-groups like `spec, arb_ham_sandwhich, gluten_free`.

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] Add support for Crucible

[Dylan Baker]

Quoting Juan A. Suarez Romero (2018-03-23 04:37:16)
> Allow to run Crucible testsuite from piglit
> 
> Signed-off-by: Juan A. Suarez Romero <jasua...@igalia.com>
> ---
>  tests/crucible.py | 84 
> +++
>  1 file changed, 84 insertions(+)
>  create mode 100644 tests/crucible.py
> 
> diff --git a/tests/crucible.py b/tests/crucible.py
> new file mode 100644
> index 0..01787d28c
> --- /dev/null
> +++ b/tests/crucible.py
> @@ -0,0 +1,84 @@
> +# Copyright 2014-2016 Intel Corporation
> +#
> +# Permission is hereby granted, free of charge, to any person obtaining a 
> copy
> +# of this software and associated documentation files (the "Software"), to 
> deal
> +# in the Software without restriction, including without limitation the 
> rights
> +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
> +# copies of the Software, and to permit persons to whom the Software is
> +# furnished to do so, subject to the following conditions:
> +#
> +# The above copyright notice and this permission notice shall be included in
> +# all copies or substantial portions of the Software.
> +#
> +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
> +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
> FROM,
> +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
> THE
> +# SOFTWARE.
> +
> +"""Piglit integrations for Crucible Test Suite.
> +
> +upstream: https://cgit.freedesktop.org/mesa/crucible/
> +
> +"""
> +
> +from __future__ import (
> +absolute_import, division, print_function, unicode_literals
> +)
> +import os
> +import subprocess
> +
> +from framework import grouptools, backends, exceptions
> +from framework.core import PIGLIT_CONFIG
> +from framework.profile import TestProfile, Test
> +
> +__all__ = ['profile']
> +
> +crucible_bin = os.environ.get('PIGLIT_CRUCIBLE_BIN', None)
> +if crucible_bin is None:
> +crucible_bin = PIGLIT_CONFIG.safe_get(('crucible', 'bin'), True)
> +
> +if crucible_bin is None:
> +raise exceptions.PiglitFatalError(
> +'Cannot get "PIGLIT_CRUCIBLE_BIN" or conf value "crucible:bin"')
> +
> +
> +class CrucibleTest(Test):
> +"""Test representation for Crucible"""
> +def __init__(self, case_name):
> +command = [crucible_bin, 'run', '--junit-xml=crucible.xml', 
> case_name]
> +self._case = case_name
> +super(CrucibleTest, self).__init__(command)
> +
> +def run(self):
> +super(CrucibleTest, self).run()

This isn't necessary, right?

> +
> +def interpret_result(self):
> +test = backends.junit.REGISTRY.load('crucible.xml', 'none')
> +result = test.get_result(next(iter(test.tests.keys(

I think this should be:
   result = test.get_result(next(six.iterkeys(test.tests)))

> +self.result.result = result.name
> +os.remove('crucible.xml')
> +super(CrucibleTest, self).interpret_result()
> +
> +def gen_caselist_txt(bin_):
> +with open('crucible.txt', 'w') as d:
> +subprocess.check_call(
> +[bin_, 'ls-tests'],
> +stdout=d, stderr=d)
> +assert os.path.exists('crucible.txt')
> +return 'crucible.txt'
> +
> +def _populate_profile():
> +profile = TestProfile()
> +case_file = gen_caselist_txt(crucible_bin)
> +with open(case_file, 'r') as caselist_file:
> +for i, line in enumerate(caselist_file):
> +case = line.rstrip()
> +piglit_name = case.replace('.', grouptools.SEPARATOR)
> +profile.test_list[piglit_name] = CrucibleTest(case)
> +os.remove('crucible.txt')
> +return profile
> +
> +profile = _populate_profile()
> -- 
> 2.14.3

Other than those small nits this seems fine,

Reviewed-by: Dylan Baker <dy...@pnwbakers.com>


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH v3 1/1] framework: change one missed core.checkDir call to core.check_dir

[Dylan Baker]

Quoting Petri Latvala (2018-03-26 02:51:08)
> 
> 
> On 03/23/2018 06:11 PM, Dylan Baker wrote:
> > Quoting Petri Latvala (2018-03-23 03:11:10)
> >> Commit 5e699e402a8b ("framework: rename core.checkDir to
> >> core.check_dir") missed one call to checkDir in summary/feature. Also
> >> convert the exception raised to PiglitFatalError.
> >>
> >> v2: Added exception changes
> >> v3: Fixed indentation
> >>
> >> Signed-off-by: Petri Latvala <petri.latv...@intel.com>
> >> Cc: Dylan Baker <dy...@pnwbakers.com>
> >> Cc: Jari Tahvanainen <jari.tahvanai...@intel.com>
> >> ---
> >>
> >> The indent was with spaces, but just incorrectly indented.
> >>
> >> framework/programs/summary.py | 8 +++-
> >>   1 file changed, 7 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/framework/programs/summary.py b/framework/programs/summary.py
> >> index c6f616dc8..ccfb24299 100644
> >> --- a/framework/programs/summary.py
> >> +++ b/framework/programs/summary.py
> >> @@ -307,6 +307,12 @@ def feature(input_):
> >>   shutil.rmtree(args.summaryDir)
> >>   
> >>   # If the requested directory doesn't exist, create it or throw an 
> >> error
> >> -core.checkDir(args.summaryDir, not args.overwrite)
> >> +try:
> >> +core.check_dir(args.summaryDir, not args.overwrite)
> >> +except exceptions.PiglitException:
> >> +    raise exceptions.PiglitFatalError(
> >> +'{} already exists.\n'
> >> +'use -o/--overwrite if you want to overwrite it.'.format(
> >> +args.summaryDir))
> >>   
> >>   summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)
> >> -- 
> >> 2.14.1
> >>
> > Reviewed-by: Dylan Baker <dy...@pnwbakers.com>
> 
> Thanks. I don't have commit rights, could you push this for me?
> 
> 
> Petri
> 

Done, thanks for fixing this.

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH v3 1/1] framework: change one missed core.checkDir call to core.check_dir

[Dylan Baker]

Quoting Petri Latvala (2018-03-23 03:11:10)
> Commit 5e699e402a8b ("framework: rename core.checkDir to
> core.check_dir") missed one call to checkDir in summary/feature. Also
> convert the exception raised to PiglitFatalError.
> 
> v2: Added exception changes
> v3: Fixed indentation
> 
> Signed-off-by: Petri Latvala <petri.latv...@intel.com>
> Cc: Dylan Baker <dy...@pnwbakers.com>
> Cc: Jari Tahvanainen <jari.tahvanai...@intel.com>
> ---
> 
> The indent was with spaces, but just incorrectly indented.
> 
> framework/programs/summary.py | 8 +++-
>  1 file changed, 7 insertions(+), 1 deletion(-)
> 
> diff --git a/framework/programs/summary.py b/framework/programs/summary.py
> index c6f616dc8..ccfb24299 100644
> --- a/framework/programs/summary.py
> +++ b/framework/programs/summary.py
> @@ -307,6 +307,12 @@ def feature(input_):
>  shutil.rmtree(args.summaryDir)
>  
>  # If the requested directory doesn't exist, create it or throw an error
> -core.checkDir(args.summaryDir, not args.overwrite)
> +try:
> +core.check_dir(args.summaryDir, not args.overwrite)
> +except exceptions.PiglitException:
> +raise exceptions.PiglitFatalError(
> +'{} already exists.\n'
> +'use -o/--overwrite if you want to overwrite it.'.format(
> +args.summaryDir))
>  
>  summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)
> -- 
> 2.14.1
> 

Reviewed-by: Dylan Baker <dy...@pnwbakers.com>


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH v2 1/1] framework: change one missed core.checkDir call to core.check_dir

[Dylan Baker]

Quoting Petri Latvala (2018-03-22 04:16:29)
> Commit 5e699e402a8b ("framework: rename core.checkDir to
> core.check_dir") missed one call to checkDir in summary/feature. Also
> convert the exception raised to PiglitFatalError.
> 
> v2: Added exception changes
> 
> Signed-off-by: Petri Latvala <petri.latv...@intel.com>
> Cc: Dylan Baker <dy...@pnwbakers.com>
> Cc: Jari Tahvanainen <jari.tahvanai...@intel.com>
> ---
>  framework/programs/summary.py | 8 +++-
>  1 file changed, 7 insertions(+), 1 deletion(-)
> 
> diff --git a/framework/programs/summary.py b/framework/programs/summary.py
> index c6f616dc8..3c21e5bf7 100644
> --- a/framework/programs/summary.py
> +++ b/framework/programs/summary.py
> @@ -307,6 +307,12 @@ def feature(input_):
>  shutil.rmtree(args.summaryDir)
>  
>  # If the requested directory doesn't exist, create it or throw an error
> -core.checkDir(args.summaryDir, not args.overwrite)
> +try:
> +core.check_dir(args.summaryDir, not args.overwrite)
> +except exceptions.PiglitException:
> +raise exceptions.PiglitFatalError(
> +'{} already exists.\n'
> +'use -o/--overwrite if you want to overwrite 
> it.'.format(
> +args.summaryDir))

This indent looks very strange, are you using tabs instead of spaces by chance?

>  
>  summary.feat(args.resultsFiles, args.summaryDir, args.featureFile)
> -- 
> 2.14.1
> 


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] cmake: Use WAYLAND_LDFLAGS instead of WAYLAND_LIBRARIES.

[Dylan Baker]

Quoting Rafael Antognolli (2018-03-09 10:42:50)
> If the wayland library is found in a prefix, or some non-default place,
> WAYLAND_LIBRARIES will not contain -L, only the
> -lwayland-client part. If we want the whole set of flags, as returned by
> "pkg-config --libs wayland-client", we need to use WAYLAND_LDFLAGS
> instead.
> 
> Signed-off-by: Rafael Antognolli <rafael.antogno...@intel.com>
> ---
>  tests/util/CMakeLists.txt | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/tests/util/CMakeLists.txt b/tests/util/CMakeLists.txt
> index a9fc33900..0508eb030 100644
> --- a/tests/util/CMakeLists.txt
> +++ b/tests/util/CMakeLists.txt
> @@ -155,7 +155,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
> endif()
>  
>  if(PIGLIT_HAS_WAYLAND)
> -   list(APPEND UTIL_GL_LIBS ${WAYLAND_LIBRARIES})
> +   list(APPEND UTIL_GL_LIBS ${WAYLAND_LDFLAGS})
>  endif()
>  
>  endif(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
> -- 
> 2.14.3
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit

Reviewed-by: Dylan Baker <dy...@pnwbakers.com>


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] tests: Added a new GLES test profile for the Khronos CTS runner

[Dylan Baker]

Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

Quoting Andres Gomez (2018-02-13 05:24:10)
> OpenGL GLES*-CTS case lists were renamed to KHR-GLES* in the upstream
> repository.
> 
> We want to keep the existing profiles so we are able to keep running
> the caselists from previous CTS releases and for branches in the open
> sourced repository created for API-specific release branches, as
> explained at:
> https://github.com/KhronosGroup/VK-GL-CTS/wiki/Contributing#branches
> 
> Therefore, we add this new test profile to be able to run the renamed
> tests in the master branch of the opensourced Khronos CTS tests at:
> https://github.com/KhronosGroup/VK-GL-CTS
> 
> Cc: Mark Janes <mark.a.ja...@intel.com>
> Cc: Dylan Baker <dylanx.c.ba...@intel.com>
> Cc: Juan A. Suarez Romero <jasua...@igalia.com>
> Signed-off-by: Andres Gomez <ago...@igalia.com>
> ---
>  tests/khr_gles.py | 88 
> +++
>  1 file changed, 88 insertions(+)
>  create mode 100644 tests/khr_gles.py
> 
> diff --git a/tests/khr_gles.py b/tests/khr_gles.py
> new file mode 100644
> index 0..59a0fe089
> --- /dev/null
> +++ b/tests/khr_gles.py
> @@ -0,0 +1,88 @@
> +# Copyright (c) 2017 Intel Corporation
> +
> +# Permission is hereby granted, free of charge, to any person obtaining a 
> copy
> +# of this software and associated documentation files (the "Software"), to 
> deal
> +# in the Software without restriction, including without limitation the 
> rights
> +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
> +# copies of the Software, and to permit persons to whom the Software is
> +# furnished to do so, subject to the following conditions:
> +
> +# The above copyright notice and this permission notice shall be included in
> +# all copies or substantial portions of the Software.
> +
> +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
> +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 
> FROM,
> +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 
> THE
> +# SOFTWARE.
> +
> +"""Piglit integration for the now open sourced Khronos CTS tests being
> +developed at https://github.com/KhronosGroup/VK-GL-CTS
> +
> +By default this will run GLES2, GLES3, GLES31, GLES32, and GLESEXT
> +test cases. Those desiring to run only a subset of them should
> +consider using the -t or -x options to include or exclude tests.
> +
> +For example:
> +./piglit run khr_gles -c foo -t ES3- would run only ES3 tests (note
> +the dash to exclude ES31 tests)
> +
> +This integration requires some configuration in piglit.conf, or the
> +use of environment variables.
> +
> +In piglit.conf one should set the following:
> +[khr_gles]:bin -- Path to the glcts binary
> +[khr_gles]:extra_args -- any extra arguments to be passed to cts
> +(optional)
> +
> +Alternatively (or in addition, since environment variables have
> +precedence), one could set:
> +PIGLIT_KHR_GLES_BIN -- environment equivalent of [khr_gles]:bin
> +PIGLIT_KHR_GLES_EXTRA_ARGS -- environment equivalent of
> +[khr_gles]:extra_args
> +
> +"""
> +
> +from __future__ import (
> +absolute_import, division, print_function, unicode_literals
> +)
> +import itertools
> +
> +from framework.test import deqp
> +
> +__all__ = ['profile']
> +
> +_KHR_BIN = deqp.get_option('PIGLIT_KHR_GLES_BIN', ('khr_gles', 'bin'),
> +   required=True)
> +
> +_EXTRA_ARGS = deqp.get_option('PIGLIT_KHR_GLES_EXTRA_ARGS', ('khr_gles', 
> 'extra_args'),
> +  default='').split()
> +
> +
> +class DEQPKHRTest(deqp.DEQPBaseTest):
> +deqp_bin = _KHR_BIN
> +
> +@property
> +def extra_args(self):
> +return super(DEQPKHRTest, self).extra_args + \
> +[x for x in _EXTRA_ARGS if not x.startswith('--deqp-case')]
> +
> +
> +# Add all of the suites by default, users can use filters to remove them.
> +profile = deqp.make_profile(  # pylint: disable=invalid-name
> +itertools.chain(
> +deqp.iter_deqp_test_cases(
> +deqp.gen_caselist_txt(_KHR_BIN, 'KHR-GLES2-cases.txt', 
> _EXTRA_ARGS)),
> +deqp.iter_deqp_test_cases(
> +deqp.gen_caselist_txt(_KHR_BIN, 'KHR-GLES3-cases.txt', 
> _EXTRA_ARGS)),
> +deqp.iter_deqp_tes

Re: [Piglit] [PATCH] degenerate-prims: fix incorrect return value from test_prim()

[Dylan Baker]

Oops,
Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

Quoting Brian Paul (2018-02-13 13:26:18)
> The function is supposed to return a piglit_result, not true/false.
> Fixes incorrect 'fail' results.
> ---
>  tests/general/degenerate-prims.c | 2 +-
>  1 file changed, 1 insertion(+), 1 deletion(-)
> 
> diff --git a/tests/general/degenerate-prims.c 
> b/tests/general/degenerate-prims.c
> index 0eed99c..3001d34 100644
> --- a/tests/general/degenerate-prims.c
> +++ b/tests/general/degenerate-prims.c
> @@ -72,7 +72,7 @@ test_prim(void *_data)
> piglit_report_subtest_result(pass ? PIGLIT_PASS : PIGLIT_FAIL,
>  "Primitive: %s", 
> piglit_get_prim_name(data->prim));
>  
> -   return pass;
> +   return pass ? PIGLIT_PASS : PIGLIT_FAIL;
>  }
>  
>  
> -- 
> 2.7.4
> 


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] AMD_performance_monitor: add VC4 specific tests

[Dylan Baker]

If there's no other changes required I don't need to see a v2 just to fix
all.py, I'm not really qualified to review the patch otherwise though.

Dylan

Quoting Boris Brezillon (2018-02-08 05:39:04)
> Hi Dylan,
> 
> On Wed, 07 Feb 2018 10:38:28 -0800
> Dylan Baker <dy...@pnwbakers.com> wrote:
> 
> > You'll need to add this to all.py.
> 
> Will fix that in the next version.
> 
> Thanks,
> 
> Boris
> -- 
> Boris Brezillon, Bootlin (formerly Free Electrons)
> Embedded Linux and Kernel engineering
> http://bootlin.com


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] AMD_performance_monitor: add VC4 specific tests

[Dylan Baker]

You'll need to add this to all.py.

Quoting Boris Brezillon (2018-02-07 00:19:34)
> From: Boris Brezillon 
> 
> This adds a specific test for the VC4 GPU.
> Right now, it only checks that FEP-valid-quads and
> QPU-total-clk-cycles-waiting-TMU are consistent after executing well
> know operations (draw a rectangle or a texture).
> 
> More tests will be added over time.
> 
> Signed-off-by: Boris Brezillon 
> ---
>  .../spec/amd_performance_monitor/CMakeLists.gl.txt |   1 +
>  tests/spec/amd_performance_monitor/vc4.c   | 309 
> +
>  2 files changed, 310 insertions(+)
>  create mode 100644 tests/spec/amd_performance_monitor/vc4.c
> 
> diff --git a/tests/spec/amd_performance_monitor/CMakeLists.gl.txt 
> b/tests/spec/amd_performance_monitor/CMakeLists.gl.txt
> index ac29de782f45..73dad38e9973 100644
> --- a/tests/spec/amd_performance_monitor/CMakeLists.gl.txt
> +++ b/tests/spec/amd_performance_monitor/CMakeLists.gl.txt
> @@ -10,3 +10,4 @@ link_libraries (
>  
>  piglit_add_executable (amd_performance_monitor_api api.c)
>  piglit_add_executable (amd_performance_monitor_measure measure.c)
> +piglit_add_executable (amd_performance_monitor_vc4 vc4.c)
> diff --git a/tests/spec/amd_performance_monitor/vc4.c 
> b/tests/spec/amd_performance_monitor/vc4.c
> new file mode 100644
> index ..40f9a0b93a52
> --- /dev/null
> +++ b/tests/spec/amd_performance_monitor/vc4.c
> @@ -0,0 +1,309 @@
> +/*
> + * Copyright © 2018 Broadcom
> + *
> + * Permission is hereby granted, free of charge, to any person obtaining a
> + * copy of this software and associated documentation files (the "Software"),
> + * to deal in the Software without restriction, including without limitation
> + * the rights to use, copy, modify, merge, publish, distribute, sublicense,
> + * and/or sell copies of the Software, and to permit persons to whom the
> + * Software is furnished to do so, subject to the following conditions:
> + *
> + * The above copyright notice and this permission notice (including the next
> + * paragraph) shall be included in all copies or substantial portions of the
> + * Software.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
> + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
> + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
> + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
> + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
> + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 
> DEALINGS
> + * IN THE SOFTWARE.
> + */
> +
> +/**
> + * \file vc4.c
> + *
> + * Check consistency of some of the VC4 perf counters.
> + */
> +
> +#define __STDC_FORMAT_MACROS
> +#include 
> +#include "piglit-util-gl.h"
> +
> +PIGLIT_GL_TEST_CONFIG_BEGIN
> +
> +   config.supports_gl_compat_version = 20;
> +   config.window_visual = PIGLIT_GL_VISUAL_RGB;
> +
> +PIGLIT_GL_TEST_CONFIG_END
> +
> +#define verify(x)
> \
> +   if (!(x)) {  \
> +   printf("%s:%i\n", __func__, __LINE__);   \
> +   piglit_report_subtest_result(PIGLIT_FAIL, "%s", test->name); \
> +   return;  \
> +   }
> +
> +/**/
> +
> +struct perfcounter_id {
> +   unsigned groupid;
> +   unsigned counterid;
> +};
> +
> +struct perfmon_counter {
> +   const char *name;
> +   unsigned id;
> +   GLenum type;
> +};
> +
> +struct perfmon_group {
> +   const char *name;
> +   unsigned id;
> +   int num_counters;
> +   int max_active_counters;
> +   struct perfmon_counter *counters;
> +};
> +
> +struct perfmon_info {
> +   int num_groups;
> +   struct perfmon_group *groups;
> +};
> +
> +static void
> +get_group_info(struct perfmon_group *group)
> +{
> +   unsigned *counterids;
> +   GLsizei length;
> +   char *name;
> +   int i;
> +
> +   glGetPerfMonitorGroupStringAMD(group->id, 0, , NULL);
> +   name = calloc(length + 1, sizeof(char));
> +   group->name = name;
> +   glGetPerfMonitorGroupStringAMD(group->id, length + 1, NULL, name);
> +
> +   glGetPerfMonitorCountersAMD(group->id, >num_counters,
> +   NULL, 0, NULL);
> +   group->counters = calloc(group->num_counters, 
> sizeof(*group->counters));
> +   counterids = calloc(group->num_counters, sizeof(*counterids));
> +   glGetPerfMonitorCountersAMD(group->id, NULL,
> +   >max_active_counters,
> +   group->num_counters, counterids);
> +
> +   for (i = 0; i < 

Re: [Piglit] [PATCH v4] travis: add docker based cmake build job

[Dylan Baker]

I haven't looked too closely at this, but I like the idea of using docker to
allow running the cmake based job.

Acked-by: Dylan Baker <dy...@pnwbakers.com>

Quoting Andres Gomez (2018-01-31 13:11:52)
> Until now we were only running the python unit tests.
> 
> It seems desirable to also check that the CMake based build compiles
> successfully. We do that now using docker.
> 
> The docker build can be tweaked with some environment variables and,
> also, be stored in the docker hub if desired. Check the changes for
> extra details regarding these variables.
> 
> v2: Removed other build possibilities other than just from inside
> Travis-CI, as suggested by Juan.
> v3: Replaced the "RELEASE" parameter to create the docker image with
> "PARENT" and removed some unneeded documentation after v2, as
> suggested by Juan.
> v4:
>   - Use DOCKER_PARENT, DOCKER_IMAGE and DOCKER_TAG variables from the
> custom Travis-CI setup to define the PARENT, IMAGE and TAG
> parameters during the docker image creation.
>   - Upload the image only if DOCKER_IMAGE and DOCKER_TAG are set.
> 
> Cc: Dylan Baker <dy...@pnwbakers.com>
> Cc: Juan A. Suarez <jasua...@igalia.com>
> Signed-off-by: Andres Gomez <ago...@igalia.com>
> Reviewed-by: Juan A. Suarez <jasua...@igalia.com>
> ---
>  .travis.yml  | 47 --
>  docker/Rockerfile.piglit | 74 
> 
>  2 files changed, 118 insertions(+), 3 deletions(-)
>  create mode 100644 docker/Rockerfile.piglit
> 
> diff --git a/.travis.yml b/.travis.yml
> index b47829ff9..b4cbcca49 100644
> --- a/.travis.yml
> +++ b/.travis.yml
> @@ -1,5 +1,16 @@
> +sudo: false
> +os: linux
>  language: python
> -cache: pip
> +cache:
> +  - ccache
> +  - pip
> +services:
> +  - docker
> +
> +env:
> +  global:
> +- BUILD=pytest
> +
>  matrix:
>include:
>  - python: 2.7
> @@ -12,7 +23,37 @@ matrix:
>env: TOX_ENV="py35-{generator,noaccel,accel-nix,streams}"
>  - python: 3.6
>env: TOX_ENV="py36-{generator,noaccel,accel-nix,streams}"
> +- env: BUILD=cmake
> +
>  install:
> -pip install tox
> +  - |
> +if [[ $BUILD == pytest ]]; then
> +  pip install tox
> +else
> +  wget 
> https://github.com/grammarly/rocker/releases/download/1.3.1/rocker-1.3.1-linux_amd64.tar.gz
> +  tar xvf rocker-1.3.1-linux_amd64.tar.gz
> +  rm rocker-1.3.1-linux_amd64.tar.gz
> +fi
> +
> +before_script:
> +  - |
> +if [[ $BUILD != pytest ]]; then
> +  mkdir -p -m777 ~/.ccache
> +fi
> +
>  script:
> -- tox -e $TOX_ENV
> +  - |
> +if [[ $BUILD == pytest ]]; then
> +  tox -e $TOX_ENV
> +else
> +  ./rocker build -f docker/Rockerfile.piglit ${DOCKER_TAG:+--var 
> TAG=}${DOCKER_TAG} ${DOCKER_IMAGE:+--var IMAGE=}${DOCKER_IMAGE} 
> ${DOCKER_PARENT:+--var PARENT=}${DOCKER_PARENT} .
> +fi
> +
> +after_success:
> +  - |
> +if [[ $BUILD != pytest ]]; then
> +  if [[ -n $DOCKER_IMAGE && -n $DOCKER_TAG && -n $DOCKER_USERNAME && 
> $TRAVIS_BRANCH == master ]]; then
> +docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}"
> +docker push "${DOCKER_IMAGE}":"${DOCKER_TAG}"
> +  fi
> +fi
> diff --git a/docker/Rockerfile.piglit b/docker/Rockerfile.piglit
> new file mode 100644
> index 0..0d8f8bb45
> --- /dev/null
> +++ b/docker/Rockerfile.piglit
> @@ -0,0 +1,74 @@
> +#
> +# This builds Piglit.
> +#
> +# ~~~
> +#  rocker build -f Rockerfile.piglit [--attach]  
>\
> +#[--var IMAGE=freedesktop/mesa] # freedesktop/mesa, myrepo/myproyect, 
> ...   \
> +#[--var TAG=piglit] # piglit-17.0, piglit-13.0, ...  
>\
> +#[--var PARENT=ubuntu:xenial]   # ubuntu:xenial, freedesktop/mesa:17.3, 
> ...
> +# ~~~
> +#
> +# Environment variables that are used in the build:
> +#  - MAKEFLAGS: flags to pass to make (e.g., "-j8")
> +#  - CCACHE_DIR: ccache directory (default: ~/.ccache)
> +#
> +# To run
> +#
> +# ~~~
> +#   mkdir -p -m777 ~/my_results_dir
> +#   docker run --privileged --rm -t -v ~/my_results_dir:/results:Z  \
> +#  -v /tmp/.X11-unix:/tmp/.X11-unix freedesktop/mesa:piglit
> +# ~~~
> +#
> +
> +{{ $image := (or .IMAGE "freedesktop/mesa") }}
> +{{ $parent_image := (or .PARENT "ubuntu:xenial") }}
> +{{ $ccachedir := (or .Env.CCACHE_DIR "~/.ccache") }}
> +
> +FROM {{ $parent_image 

Re: [Piglit] [PATCH] all.py: fix extension/version testing when wflinfo fails

[Dylan Baker]

Quoting Brian Paul (2018-02-02 11:46:22)
> On 02/02/2018 11:57 AM, Dylan Baker wrote:
> > Quoting Brian Paul (2018-02-02 09:35:21)
> >> If wflinfo is not installed or fails, the wfl_info.gl_extensions value
> >> will be the empty set.  Similarly, the gl_version, glsl_version, etc.
> >> will be None.
> >>
> >> In those cases, return True from the gl_extension_supported() and
> >> is_feature_directory_supported() functions so that we don't skip any
> >> subdirs.
> >>
> >> This is only relevant when PIGLIT_FILTER_DIRECTORIES is set.
> >> ---
> >>   tests/all.py | 15 +++
> >>   1 file changed, 11 insertions(+), 4 deletions(-)
> >>
> >> diff --git a/tests/all.py b/tests/all.py
> >> index 6d4e887..eece906 100644
> >> --- a/tests/all.py
> >> +++ b/tests/all.py
> >> @@ -217,6 +217,10 @@ wfl_info = wflinfo.WflInfo()
> >>   
> >>   def gl_extension_supported(ext_name):
> >>   """Is the named OpenGL extension supported?"""
> >> +if wfl_info.gl_extensions == set():
> > 
> > I think we should just do 'if not wfl_info.gl_extensions' here.
> 
> That doesn't work since we set return set() in WflInfo.gl_extensions().

Okay

> 
> 
> > 
> >> +# Don't know what extensions are supported.  Return true so we 
> >> don't
> >> +# skip anything.
> >> +return True
> >>   return ext_name in wfl_info.gl_extensions
> >>   
> >>   
> >> @@ -227,6 +231,9 @@ def is_feature_directory_supported(dir_name):
> >>   it means the extension/version is possibly suppported.  We're a 
> >> little
> >>   fuzzy because we don't yet parse all the directory name possibilities
> >>   (like ES tests).
> >> +
> >> +Also, if the 'wflinfo' program is not installed or fails (i.e. we 
> >> can't
> >> +determine the GL version/extension) we return True.
> >>   """
> >>   if dir_name[:4] in {"amd_", "arb_", "ati_", "ext_", "khr_", "oes_"}:
> >>   # The directory is a GL extension name, but of the format 
> >> "arb_foo_bar"
> >> @@ -237,19 +244,19 @@ def is_feature_directory_supported(dir_name):
> >>   elif dir_name[:5] == "gles-":
> >>   # OpenGL ES test
> >>   version = float(dir_name[5:])
> >> -return wfl_info.gles_version != None and version <= 
> >> wfl_info.gles_version
> >> +return wfl_info.gles_version == None or version <= 
> >> wfl_info.gles_version
> > 
> > I know this uses != with None, but we really should be using `is` and `is 
> > not`
> > with None.
> 
> OK.
> 
> 
> > 
> >>   elif dir_name[:8] == "glsl-es-":
> >>   # OpenGL ES shader test
> >>   version = float(dir_name[8:])
> >> -return wfl_info.glsl_es_version != None and version <= 
> >> wfl_info.glsl_es_version
> >> +return wfl_info.glsl_es_version == None or version <= 
> >> wfl_info.glsl_es_version
> >>   elif dir_name[:3] == "gl-":
> >>   # The directory is a GL version
> >>   version = float(dir_name[3:])
> >> -return version <= wfl_info.gl_version
> >> +return wfl_info.gl_version == None or version <= 
> >> wfl_info.gl_version
> >>   elif dir_name[:5] == "glsl-":
> >>   # The directory is a GLSL version
> >>   version = float(dir_name[5:])
> >> -return version <= wfl_info.glsl_version
> >> +return wfl_info.glsl_version == None or version <= 
> >> wfl_info.glsl_version
> >>   else:
> >>   # The directory is something else.  Don't skip it.
> >>   return True
> > 
> > With  the nits above fixed,
> > Reviewed-by: Dylan Baker <dy...@pnwbakers.com>
> > 
> 
> Thanks!
> 
> -Brian
> 
> 


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] Log crashed subtest V2

[Dylan Baker]

Quoting Dylan Baker (2018-01-30 09:55:06)
> for the series:
> Reviewed-by: Dylan Baker <dy...@pnwbakers.com>
> 
> Quoting Fabian Bieler (2018-01-28 16:22:10)
> > The idea is to convert tests with subtests to either use 
> > piglit_run_selected_subtests or call piglit_register_subtests at the start 
> > of the test. Either approach enumerates all all subtests to stdout in JSON 
> > format before running them.
> > 
> > The python framework can use this to deduce which subtest crashed.
> > ___
> > Piglit mailing list
> > Piglit@lists.freedesktop.org
> > https://lists.freedesktop.org/mailman/listinfo/piglit
> 
> 
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit

I hate merging things on Friday, but I'd like to get the framework bits merged
so we can start fixing more tests. So, if no one objects before then I suggest
merging this on Monday.

Dylan


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] all.py: fix extension/version testing when wflinfo fails

[Dylan Baker]

Quoting Brian Paul (2018-02-02 09:35:21)
> If wflinfo is not installed or fails, the wfl_info.gl_extensions value
> will be the empty set.  Similarly, the gl_version, glsl_version, etc.
> will be None.
> 
> In those cases, return True from the gl_extension_supported() and
> is_feature_directory_supported() functions so that we don't skip any
> subdirs.
> 
> This is only relevant when PIGLIT_FILTER_DIRECTORIES is set.
> ---
>  tests/all.py | 15 +++
>  1 file changed, 11 insertions(+), 4 deletions(-)
> 
> diff --git a/tests/all.py b/tests/all.py
> index 6d4e887..eece906 100644
> --- a/tests/all.py
> +++ b/tests/all.py
> @@ -217,6 +217,10 @@ wfl_info = wflinfo.WflInfo()
>  
>  def gl_extension_supported(ext_name):
>  """Is the named OpenGL extension supported?"""
> +if wfl_info.gl_extensions == set():

I think we should just do 'if not wfl_info.gl_extensions' here.

> +# Don't know what extensions are supported.  Return true so we don't
> +# skip anything.
> +return True
>  return ext_name in wfl_info.gl_extensions
>  
>  
> @@ -227,6 +231,9 @@ def is_feature_directory_supported(dir_name):
>  it means the extension/version is possibly suppported.  We're a little
>  fuzzy because we don't yet parse all the directory name possibilities
>  (like ES tests).
> +
> +Also, if the 'wflinfo' program is not installed or fails (i.e. we can't
> +determine the GL version/extension) we return True.
>  """
>  if dir_name[:4] in {"amd_", "arb_", "ati_", "ext_", "khr_", "oes_"}:
>  # The directory is a GL extension name, but of the format 
> "arb_foo_bar"
> @@ -237,19 +244,19 @@ def is_feature_directory_supported(dir_name):
>  elif dir_name[:5] == "gles-":
>  # OpenGL ES test
>  version = float(dir_name[5:])
> -return wfl_info.gles_version != None and version <= 
> wfl_info.gles_version
> +return wfl_info.gles_version == None or version <= 
> wfl_info.gles_version

I know this uses != with None, but we really should be using `is` and `is not`
with None.

>  elif dir_name[:8] == "glsl-es-":
>  # OpenGL ES shader test
>  version = float(dir_name[8:])
> -return wfl_info.glsl_es_version != None and version <= 
> wfl_info.glsl_es_version
> +return wfl_info.glsl_es_version == None or version <= 
> wfl_info.glsl_es_version
>  elif dir_name[:3] == "gl-":
>  # The directory is a GL version
>  version = float(dir_name[3:])
> -return version <= wfl_info.gl_version
> +return wfl_info.gl_version == None or version <= wfl_info.gl_version
>  elif dir_name[:5] == "glsl-":
>      # The directory is a GLSL version
>  version = float(dir_name[5:])
> -return version <= wfl_info.glsl_version
> +return wfl_info.glsl_version == None or version <= 
> wfl_info.glsl_version
>  else:
>  # The directory is something else.  Don't skip it.
>  return True

With  the nits above fixed,
Reviewed-by: Dylan Baker <dy...@pnwbakers.com>


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] Log crashed subtest V2

[Dylan Baker]

for the series:
Reviewed-by: Dylan Baker <dy...@pnwbakers.com>

Quoting Fabian Bieler (2018-01-28 16:22:10)
> The idea is to convert tests with subtests to either use 
> piglit_run_selected_subtests or call piglit_register_subtests at the start of 
> the test. Either approach enumerates all all subtests to stdout in JSON 
> format before running them.
> 
> The python framework can use this to deduce which subtest crashed.
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 01/15] tests/util: Add config value to enumerate subtests

[Dylan Baker]

I can go either way, if you like 2 better then lets go with that.

Quoting Fabian Bieler (2018-01-26 11:33:04)
> I can resend to the list.
> Did we reach an agreement on how piglit_run_selected_subtests should behave?
> 
> Should it
> enumerate all tests and mark unselected tests PIGLIT_SKIP
> or
> enumerate only selected tests implicitly marking unselected tests as "not 
> run" when compared to another test run?
> 
> I favor the second option as understand PIGLIT_SKIP means "tests ought to be 
> run, but for some reason (missing extension, limited resources, ...) aren't" 
> and not "test shouldn't be run to begin with (e.g. because it's not included 
> in the current test suite)" for which I thought "not run" is the appropriate 
> status.
> 
> Fabian
> 
> On Fri, Jan 26, 2018, at 6:53 PM, Dylan Baker wrote:
> > Fabian,
> > 
> > When you get a chance can you send your updated version of this series to 
> > the
> > list? I think this is probably a good candidate for piecemeal fixes, we can
> > land the framework bits and the initial tests that are ported, then land the
> > changes to the rest of the tests in manageable chunks.
> > 
> > Dylan
> > 
> > Quoting Fabian Bieler (2018-01-23 15:35:32)
> > > Hello!
> > > 
> > > So after some consideration I think I prefer your first approach
> > > (explicitly calling the enumeration function) after all.
> > > 
> > > First of all: Sorry for the back and forth on my part and the excess
> > > work you put into this series because of it. Please don't hate me! 
> > > 
> > > However, I think this approach will be more DRY and less WET.
> > > In particular when used with the existing struct piglit_subtest and
> > > piglit_run_selected_subtests.
> > > 
> > > I updated the series accordingly (and fixed a type-o in a docstring in
> > > 5/15 and the commit message of 6/15 and a small arithmetical error in
> > > 11/15).
> > > 
> > > The result can be found here:
> > > https://github.com/fabianbieler/piglit/tree/enumerate_subtests
> > > If this isn't acceptable for intermediate review, I'll post to the list,
> > > too.
> > > 
> > > Note that I leveraged piglit_run_selected_subtests for gl-2.1-pbo,
> > > fbo-incomplete, degenerate-prims and linestipple and used
> > > piglit_register_subtests in fbo-storage-formats.
> > > 
> > > Regards (and sorry again)
> > > Fabian
> > > 
> > > On 2018-01-23 02:22, Dylan Baker wrote:
> > > > This adds a new member to the GL config struct for informing the python
> > > > framework the number and order of subtests that will be run (if any). If
> > > > this value is unset then no subtests are expected.
> > > > ---
> > > >  tests/util/piglit-framework-gl.c | 13 +
> > > >  tests/util/piglit-framework-gl.h | 11 +++
> > > >  2 files changed, 24 insertions(+)
> > > > 
> > > > diff --git a/tests/util/piglit-framework-gl.c 
> > > > b/tests/util/piglit-framework-gl.c
> > > > index 1b2078d..1ce5474 100644
> > > > --- a/tests/util/piglit-framework-gl.c
> > > > +++ b/tests/util/piglit-framework-gl.c
> > > > @@ -318,3 +318,16 @@ piglit_get_selected_tests(const char 
> > > > ***selected_subtests)
> > > >   *selected_subtests = gl_fw->test_config->selected_subtests;
> > > >   return gl_fw->test_config->num_selected_subtests;
> > > >  }
> > > > +
> > > > +void
> > > > +piglit_gl_enumerate_subtests(const struct piglit_gl_test_config 
> > > > *config)
> > > > +{
> > > > + if (config->all_subtests) {
> > > > + printf("PIGLIT: {\"enumerate subtests\": [\"%s\"", 
> > > > config->all_subtests[0]);
> > > > + for (int i = 1; config->all_subtests[i]; i++) {
> > > > + printf(", \"%s\"", config->all_subtests[i]);
> > > > + }
> > > > + printf("]}\n");
> > > > + fflush(stdout);
> > > > + }
> > > > +}
> > > > diff --git a/tests/util/piglit-framework-gl.h 
> > > > b/tests/util/piglit-framework-gl.h
> > > > index 970fd55..d36c4f0 100644
> > > > --- a/tests/util/piglit-framework-gl.h
> > > > +++ b/tests/util/pig

Re: [Piglit] [PATCH 01/15] tests/util: Add config value to enumerate subtests

[Dylan Baker]

Fabian,

When you get a chance can you send your updated version of this series to the
list? I think this is probably a good candidate for piecemeal fixes, we can
land the framework bits and the initial tests that are ported, then land the
changes to the rest of the tests in manageable chunks.

Dylan

Quoting Fabian Bieler (2018-01-23 15:35:32)
> Hello!
> 
> So after some consideration I think I prefer your first approach
> (explicitly calling the enumeration function) after all.
> 
> First of all: Sorry for the back and forth on my part and the excess
> work you put into this series because of it. Please don't hate me! 
> 
> However, I think this approach will be more DRY and less WET.
> In particular when used with the existing struct piglit_subtest and
> piglit_run_selected_subtests.
> 
> I updated the series accordingly (and fixed a type-o in a docstring in
> 5/15 and the commit message of 6/15 and a small arithmetical error in
> 11/15).
> 
> The result can be found here:
> https://github.com/fabianbieler/piglit/tree/enumerate_subtests
> If this isn't acceptable for intermediate review, I'll post to the list,
> too.
> 
> Note that I leveraged piglit_run_selected_subtests for gl-2.1-pbo,
> fbo-incomplete, degenerate-prims and linestipple and used
> piglit_register_subtests in fbo-storage-formats.
> 
> Regards (and sorry again)
> Fabian
> 
> On 2018-01-23 02:22, Dylan Baker wrote:
> > This adds a new member to the GL config struct for informing the python
> > framework the number and order of subtests that will be run (if any). If
> > this value is unset then no subtests are expected.
> > ---
> >  tests/util/piglit-framework-gl.c | 13 +
> >  tests/util/piglit-framework-gl.h | 11 +++
> >  2 files changed, 24 insertions(+)
> > 
> > diff --git a/tests/util/piglit-framework-gl.c 
> > b/tests/util/piglit-framework-gl.c
> > index 1b2078d..1ce5474 100644
> > --- a/tests/util/piglit-framework-gl.c
> > +++ b/tests/util/piglit-framework-gl.c
> > @@ -318,3 +318,16 @@ piglit_get_selected_tests(const char 
> > ***selected_subtests)
> >   *selected_subtests = gl_fw->test_config->selected_subtests;
> >   return gl_fw->test_config->num_selected_subtests;
> >  }
> > +
> > +void
> > +piglit_gl_enumerate_subtests(const struct piglit_gl_test_config *config)
> > +{
> > + if (config->all_subtests) {
> > + printf("PIGLIT: {\"enumerate subtests\": [\"%s\"", 
> > config->all_subtests[0]);
> > + for (int i = 1; config->all_subtests[i]; i++) {
> > + printf(", \"%s\"", config->all_subtests[i]);
> > + }
> > + printf("]}\n");
> > + fflush(stdout);
> > + }
> > +}
> > diff --git a/tests/util/piglit-framework-gl.h 
> > b/tests/util/piglit-framework-gl.h
> > index 970fd55..d36c4f0 100644
> > --- a/tests/util/piglit-framework-gl.h
> > +++ b/tests/util/piglit-framework-gl.h
> > @@ -213,6 +213,13 @@ struct piglit_gl_test_config {
> >* enum used for markin test as supporting KHR_no_error or not.
> >*/
> >   enum piglit_khr_no_error_support khr_no_error_support;
> > +
> > + /**
> > +  * Null terminated list of subtests to be enumerated.
> > +  *
> > +  * Each subtest *must* be run in the order reported by this list.
> > +  */
> > + const char **all_subtests;
> >  };
> >  
> >  /**
> > @@ -246,6 +253,9 @@ void
> >  piglit_gl_test_run(int argc, char *argv[],
> >  const struct piglit_gl_test_config *config);
> >  
> > +void
> > +piglit_gl_enumerate_subtests(const struct piglit_gl_test_config *config);
> > +
> >  #ifdef __cplusplus
> >  #  define PIGLIT_EXTERN_C_BEGIN extern "C" {
> >  #  define PIGLIT_EXTERN_C_END   }
> > @@ -287,6 +297,7 @@ piglit_gl_test_run(int argc, char *argv[],
> >  }  
> >   \
> > 
> >   \
> >  piglit_gl_process_args(, argv, );  
> >   \
> > +piglit_gl_enumerate_subtests(); 
> >   \
> >  piglit_gl_test_run(argc, argv, );   
> >   \
> > 
> >   \
> >  assert(false); 
> >   \
> > 
> > base-commit: 736496667329bf73a706aebec6f8287078df79ae
> > 


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH 01/15] tests/util: Add config value to enumerate subtests

[Dylan Baker]

No worries. I'd rather get this right the first time before we convert 100+
tests than afterwards :)

I also appreciate that you've converted some of the tests to use
piglit_run_selected_subtests. I've lamented with others that it's unfortunate
that we've never really used that feature, since it would be nice for debugging.

I've left you a few comments on github, but we should send it to the list before
we land it so that others have a chance to look at it.

Dylan

Quoting Fabian Bieler (2018-01-23 15:35:32)
> Hello!
> 
> So after some consideration I think I prefer your first approach
> (explicitly calling the enumeration function) after all.
> 
> First of all: Sorry for the back and forth on my part and the excess
> work you put into this series because of it. Please don't hate me! 
> 
> However, I think this approach will be more DRY and less WET.
> In particular when used with the existing struct piglit_subtest and
> piglit_run_selected_subtests.
> 
> I updated the series accordingly (and fixed a type-o in a docstring in
> 5/15 and the commit message of 6/15 and a small arithmetical error in
> 11/15).
> 
> The result can be found here:
> https://github.com/fabianbieler/piglit/tree/enumerate_subtests
> If this isn't acceptable for intermediate review, I'll post to the list,
> too.
> 
> Note that I leveraged piglit_run_selected_subtests for gl-2.1-pbo,
> fbo-incomplete, degenerate-prims and linestipple and used
> piglit_register_subtests in fbo-storage-formats.
> 
> Regards (and sorry again)
> Fabian
> 


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 08/15] tests/fbo-storage-formats: Always print the same number of subtests

[Dylan Baker]

!skip -> skip won't show up in the regressions/fixes/etc lists anyway,
and this means that the output will always be the same.
---
 tests/fbo/fbo-storage-formats.c | 6 ++
 1 file changed, 6 insertions(+)

diff --git a/tests/fbo/fbo-storage-formats.c b/tests/fbo/fbo-storage-formats.c
index 3ecd07a..91b80e0 100644
--- a/tests/fbo/fbo-storage-formats.c
+++ b/tests/fbo/fbo-storage-formats.c
@@ -223,6 +223,12 @@ test(void)
piglit_report_subtest_result(PIGLIT_PASS, "%s", 
name);
}
}
+   } else {
+   printf("Skipping error tests because KHR_NO_ERROR is 
enabled\n");
+   for (i = 0; i < ARRAY_SIZE(invalid_formats); i++) {
+   const char *name = 
piglit_get_gl_enum_name(invalid_formats[i]);
+   piglit_report_subtest_result(PIGLIT_SKIP, "%s", name);
+   }
}
 
return pass ? PIGLIT_PASS : PIGLIT_FAIL;
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 12/15] tests: always report subtest results for degenerate-prims

[Dylan Baker]

---
 tests/general/degenerate-prims.c | 6 ++
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/general/degenerate-prims.c b/tests/general/degenerate-prims.c
index 7ed6093..d2bb7d6 100644
--- a/tests/general/degenerate-prims.c
+++ b/tests/general/degenerate-prims.c
@@ -63,10 +63,8 @@ test_prim(GLenum prim, unsigned numVerts, const void *verts)
 
piglit_present_results();
 
-   if (!pass) {
-   piglit_report_subtest_result(PIGLIT_FAIL, "Primitive: %s",
-piglit_get_prim_name(prim));
-   }
+   piglit_report_subtest_result(pass ? PIGLIT_PASS : PIGLIT_FAIL,
+"Primitive: %s", 
piglit_get_prim_name(prim));
 
return pass;
 }
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 15/15] Add series todo

[Dylan Baker]

---
 todo | 108 -
 1 file changed, 108 insertions(+)
 create mode 100644 todo

diff --git a/todo b/todo
new file mode 100644
index 000..c869582
--- /dev/null
+++ b/todo
@@ -0,0 +1,108 @@
+tests/cl/api/set-kernel-arg.c
+tests/cl/api/get-mem-object-info.c
+tests/cl/custom/buffer-flags.c
+tests/cl/program/predefined-macros.c
+tests/cl/program/program-tester.c
+tests/fbo/fbo-alphatest-formats.c
+tests/fbo/fbo-blending-formats.c
+tests/fbo/fbo-blending-snorm.c
+tests/fbo/fbo-clear-formats.c
+tests/fbo/fbo-colormask-formats.c
+tests/fbo/fbo-generatemipmap-formats.c
+tests/fbo/fbo-readpixels-depth-formats.c
+tests/general/polygon-mode-offset.c
+tests/general/clipflat.c
+tests/shaders/built-in-constants.c
+tests/shaders/shader_runner.c
+tests/spec/amd_performance_monitor/api.c
+tests/spec/amd_performance_monitor/measure.c
+tests/spec/arb_arrays_of_arrays/max-binding.c
+tests/spec/arb_copy_buffer/subdata-sync.c
+tests/spec/arb_copy_buffer/data-sync.c
+tests/spec/arb_copy_image/formats.c
+tests/spec/arb_direct_state_access/create-programpipelines.c
+tests/spec/arb_direct_state_access/texture-errors.c
+tests/spec/arb_direct_state_access/texturesubimage.c
+tests/spec/arb_direct_state_access/getcompressedtextureimage.c
+tests/spec/arb_direct_state_access/texture-storage-multisample.c
+tests/spec/arb_direct_state_access/texture-storage.c
+tests/spec/arb_draw_indirect/api-errors.c
+tests/spec/arb_enhanced_layouts/transform-feedback-layout-query-api.c
+tests/spec/arb_framebuffer_no_attachments/atomic.c
+tests/spec/arb_framebuffer_no_attachments/minmax.c
+tests/spec/arb_framebuffer_no_attachments/params.c
+tests/spec/arb_framebuffer_no_attachments/query.c
+tests/spec/arb_framebuffer_srgb/srgb_conformance.c
+tests/spec/arb_gpu_shader5/execution/xfb-streams.c
+tests/spec/arb_internalformat_query2/color-encoding.c
+tests/spec/arb_internalformat_query2/filter.c
+tests/spec/arb_internalformat_query2/format-components.c
+tests/spec/arb_internalformat_query2/generic-pname-checks.c
+tests/spec/arb_internalformat_query2/image-texture.c
+tests/spec/arb_internalformat_query2/internalformat-size-checks.c
+tests/spec/arb_internalformat_query2/internalformat-type-checks.c
+tests/spec/arb_internalformat_query2/max-dimensions.c
+tests/spec/arb_internalformat_query2/samples-pnames.c
+tests/spec/arb_internalformat_query2/texture-compressed-block.c
+tests/spec/arb_internalformat_query2/image-format-compatibility-type.c
+tests/spec/arb_point_parameters/point-attenuation.c
+tests/spec/arb_program_interface_query/compare-with-shader-subroutine.c
+tests/spec/arb_program_interface_query/getprogramresourceiv.c
+tests/spec/arb_program_interface_query/getprogramresourcename.c
+tests/spec/arb_program_interface_query/resource-location.c
+tests/spec/arb_program_interface_query/resource-query.c
+tests/spec/arb_program_interface_query/getprograminterfaceiv.c
+tests/spec/arb_program_interface_query/getprogramresourceindex.c
+tests/spec/arb_query_buffer_object/coherency.c
+tests/spec/arb_query_buffer_object/qbo.c
+tests/spec/arb_sample_shading/execution/samplemask.cpp
+tests/spec/arb_separate_shader_objects/ProgramUniform-coverage.c
+tests/spec/arb_separate_shader_objects/ValidateProgramPipeline.c
+tests/spec/arb_separate_shader_objects/api-errors.c
+tests/spec/arb_separate_shader_objects/mixed_explicit_and_non_explicit_locations.c
+tests/spec/arb_separate_shader_objects/rendezvous_by_name.c
+tests/spec/arb_shader_atomic_counters/common.h
+tests/spec/arb_shader_atomic_counters/max-counters.c
+tests/spec/arb_shader_image_load_store/common.h
+tests/spec/arb_shader_texture_image_samples/builtin-image.c
+tests/spec/arb_sync/ClientWaitSync-timeout.c
+tests/spec/arb_tessellation_shader/large-uniforms.c
+tests/spec/arb_texture_buffer_object/formats.c
+tests/spec/arb_texture_multisample/fb-completeness.c
+tests/spec/arb_texture_multisample/negative-max-samples.c
+tests/spec/arb_texture_storage/texture-storage.c
+tests/spec/arb_texture_storage_multisample/tex-storage.c
+tests/spec/arb_texture_storage_multisample/tex-param.c
+tests/spec/arb_texture_view/rendering_target.c
+tests/spec/arb_texture_view/formats.c
+tests/spec/arb_texture_view/params.c
+tests/spec/arb_texture_view/rendering-formats.c
+tests/spec/arb_texture_view/rendering_layers.c
+tests/spec/arb_texture_view/rendering_levels.c
+tests/spec/arb_texture_view/sampling-2d-array-as-2d-layer.c
+tests/spec/arb_texture_view/targets.c
+tests/spec/arb_texture_view/lifetime_format.c
+tests/spec/arb_vertex_type_10f_11f_11f_rev/api-errors.c
+tests/spec/arb_viewport_array/bounds.c
+tests/spec/arb_viewport_array/render_scissor.c
+tests/spec/ext_packed_float/getteximage-invalid-format-for-packed-type.c
+tests/spec/ext_polygon_offset_clamp/dlist.c
+tests/spec/ext_polygon_offset_clamp/draw.c
+tests/spec/ext_transform_feedback/max-varyings.c
+tests/spec/ext_window_rectangles/dlist.c
+tests/spec/gl-1.0/beginend-coverage.c
+tests/spec/gl-1.0/dlist-beginend.c

[Piglit] [PATCH 01/15] tests/util: Add config value to enumerate subtests

[Dylan Baker]

This adds a new member to the GL config struct for informing the python
framework the number and order of subtests that will be run (if any). If
this value is unset then no subtests are expected.
---
 tests/util/piglit-framework-gl.c | 13 +
 tests/util/piglit-framework-gl.h | 11 +++
 2 files changed, 24 insertions(+)

diff --git a/tests/util/piglit-framework-gl.c b/tests/util/piglit-framework-gl.c
index 1b2078d..1ce5474 100644
--- a/tests/util/piglit-framework-gl.c
+++ b/tests/util/piglit-framework-gl.c
@@ -318,3 +318,16 @@ piglit_get_selected_tests(const char ***selected_subtests)
*selected_subtests = gl_fw->test_config->selected_subtests;
return gl_fw->test_config->num_selected_subtests;
 }
+
+void
+piglit_gl_enumerate_subtests(const struct piglit_gl_test_config *config)
+{
+   if (config->all_subtests) {
+   printf("PIGLIT: {\"enumerate subtests\": [\"%s\"", 
config->all_subtests[0]);
+   for (int i = 1; config->all_subtests[i]; i++) {
+   printf(", \"%s\"", config->all_subtests[i]);
+   }
+   printf("]}\n");
+   fflush(stdout);
+   }
+}
diff --git a/tests/util/piglit-framework-gl.h b/tests/util/piglit-framework-gl.h
index 970fd55..d36c4f0 100644
--- a/tests/util/piglit-framework-gl.h
+++ b/tests/util/piglit-framework-gl.h
@@ -213,6 +213,13 @@ struct piglit_gl_test_config {
 * enum used for markin test as supporting KHR_no_error or not.
 */
enum piglit_khr_no_error_support khr_no_error_support;
+
+   /**
+* Null terminated list of subtests to be enumerated.
+*
+* Each subtest *must* be run in the order reported by this list.
+*/
+   const char **all_subtests;
 };
 
 /**
@@ -246,6 +253,9 @@ void
 piglit_gl_test_run(int argc, char *argv[],
   const struct piglit_gl_test_config *config);
 
+void
+piglit_gl_enumerate_subtests(const struct piglit_gl_test_config *config);
+
 #ifdef __cplusplus
 #  define PIGLIT_EXTERN_C_BEGIN extern "C" {
 #  define PIGLIT_EXTERN_C_END   }
@@ -287,6 +297,7 @@ piglit_gl_test_run(int argc, char *argv[],
 }\
  \
 piglit_gl_process_args(, argv, );\
+piglit_gl_enumerate_subtests();   \
 piglit_gl_test_run(argc, argv, ); \
  \
 assert(false);   \

base-commit: 736496667329bf73a706aebec6f8287078df79ae
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 06/15] tests: enumerate subtests in gl-2.1-gbo test

[Dylan Baker]

This gives us something to test the python part against.

Signed-off-by: Dylan Baker <dylan.c.ba...@intel.com>
---
 tests/spec/gl-2.1/pbo.c | 12 
 1 file changed, 12 insertions(+)

diff --git a/tests/spec/gl-2.1/pbo.c b/tests/spec/gl-2.1/pbo.c
index 83dc1c4..81d4f64 100644
--- a/tests/spec/gl-2.1/pbo.c
+++ b/tests/spec/gl-2.1/pbo.c
@@ -50,6 +50,18 @@ PIGLIT_GL_TEST_CONFIG_BEGIN
PIGLIT_GL_VISUAL_DOUBLE;
 
config.khr_no_error_support = PIGLIT_NO_ERRORS;
+   const char *tests[] = {
+   "test_sanity",
+   "test_draw_pixels",
+   "test_pixel_map",
+   "test_bitmap",
+   "test_tex_image",
+   "test_tex_sub_image",
+   "test_polygon_stip",
+   "test_error_handling",
+   NULL,
+   };
+   config.all_subtests = tests;
 
 PIGLIT_GL_TEST_CONFIG_END
 
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 13/15] tests: enumerate subtests in degenerate-prims

[Dylan Baker]

---
 tests/general/degenerate-prims.c | 59 ++---
 1 file changed, 41 insertions(+), 18 deletions(-)

diff --git a/tests/general/degenerate-prims.c b/tests/general/degenerate-prims.c
index d2bb7d6..6a73b65 100644
--- a/tests/general/degenerate-prims.c
+++ b/tests/general/degenerate-prims.c
@@ -35,10 +35,44 @@
 
 #include "piglit-util-gl.h"
 
+static const float verts2[2][2] = { {-1, -1}, {1, 1} };
+static const float verts3[3][2] = { {-1, -1}, {1, -1}, {0, 1} };
+static const float verts4[4][2] = { {-1, -1}, {1, -1}, {1, 1}, {-1, 1} };
+
+struct test_config
+{
+   GLenum prim;
+   unsigned numVerts;
+   const void *verts;
+};
+
+static const struct test_config tests[] = {
+   { GL_POINTS, 0, verts2 },
+   { GL_LINES, 1, verts2 },
+   { GL_LINE_STRIP, 1, verts2 },
+   { GL_LINE_LOOP, 1, verts2 },
+   { GL_TRIANGLES, 2, verts3 },
+   { GL_TRIANGLE_STRIP, 2, verts3 },
+   { GL_TRIANGLE_FAN, 2, verts3 },
+   { GL_QUADS, 3, verts4 },
+   { GL_QUAD_STRIP, 3, verts4 },
+   { GL_POLYGON, 2, verts4 },
+   { 0 }
+};
+
+
 PIGLIT_GL_TEST_CONFIG_BEGIN
config.supports_gl_compat_version = 10;
config.window_visual = PIGLIT_GL_VISUAL_RGB | PIGLIT_GL_VISUAL_DOUBLE;
config.khr_no_error_support = PIGLIT_NO_ERRORS;
+
+   const char *subtests[ARRAY_SIZE(tests) - 1];
+   for (int i = 0; tests[i].verts; i++) {
+   subtests[i] = piglit_get_prim_name(tests[i].prim);
+   }
+   subtests[ARRAY_SIZE(tests) - 1] = NULL;
+   config.all_subtests = subtests;
+
 PIGLIT_GL_TEST_CONFIG_END
 
 
@@ -47,16 +81,16 @@ PIGLIT_GL_TEST_CONFIG_END
  * The expected outcome is that nothing will be drawn.
  */
 static bool
-test_prim(GLenum prim, unsigned numVerts, const void *verts)
+test_prim(const struct test_config *config)
 {
static const float black[] = {0, 0, 0, 0};
bool pass;
 
glClear(GL_COLOR_BUFFER_BIT);
 
-   glVertexPointer(2, GL_FLOAT, 0, verts);
+   glVertexPointer(2, GL_FLOAT, 0, config->verts);
glEnable(GL_VERTEX_ARRAY);
-   glDrawArrays(prim, 0, numVerts);
+   glDrawArrays(config->prim, 0, config->numVerts);
 
/* Nothing should have been drawn / look for all black */
pass = piglit_probe_rect_rgb(0, 0, piglit_width, piglit_height, black);
@@ -64,7 +98,7 @@ test_prim(GLenum prim, unsigned numVerts, const void *verts)
piglit_present_results();
 
piglit_report_subtest_result(pass ? PIGLIT_PASS : PIGLIT_FAIL,
-"Primitive: %s", 
piglit_get_prim_name(prim));
+"Primitive: %s", 
piglit_get_prim_name(config->prim));
 
return pass;
 }
@@ -73,10 +107,6 @@ test_prim(GLenum prim, unsigned numVerts, const void *verts)
 enum piglit_result
 piglit_display(void)
 {
-   static const float
-   verts2[2][2] = { {-1, -1}, {1, 1} },
-   verts3[3][2] = { {-1, -1}, {1, -1}, {0, 1} },
-   verts4[4][2] = { {-1, -1}, {1, -1}, {1, 1}, {-1, 1} };
bool pass = true;
 
glMatrixMode(GL_PROJECTION);
@@ -85,16 +115,9 @@ piglit_display(void)
 
glColor3f(1, 1, 1);
 
-   pass = test_prim(GL_POINTS, 0, verts2) && pass;
-   pass = test_prim(GL_LINES, 1, verts2) && pass;
-   pass = test_prim(GL_LINE_STRIP, 1, verts2) && pass;
-   pass = test_prim(GL_LINE_LOOP, 1, verts2) && pass;
-   pass = test_prim(GL_TRIANGLES, 2, verts3) && pass;
-   pass = test_prim(GL_TRIANGLE_STRIP, 2, verts3) && pass;
-   pass = test_prim(GL_TRIANGLE_FAN, 2, verts3) && pass;
-   pass = test_prim(GL_QUADS, 3, verts4) && pass;
-   pass = test_prim(GL_QUAD_STRIP, 3, verts4) && pass;
-   pass = test_prim(GL_POLYGON, 2, verts4) && pass;
+   for (int i = 0; tests[i].verts; i++) {
+   pass = test_prim([i]) && pass;
+   }
 
return pass ? PIGLIT_PASS : PIGLIT_FAIL;
 }
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 07/15] tests: enumerate subtests in fbo-incomplete

[Dylan Baker]

Signed-off-by: Dylan Baker <dylan.c.ba...@intel.com>
---
 tests/fbo/fbo-incomplete.cpp | 12 
 1 file changed, 12 insertions(+)

diff --git a/tests/fbo/fbo-incomplete.cpp b/tests/fbo/fbo-incomplete.cpp
index 8cde6d2..8d16ec1 100644
--- a/tests/fbo/fbo-incomplete.cpp
+++ b/tests/fbo/fbo-incomplete.cpp
@@ -34,6 +34,18 @@ PIGLIT_GL_TEST_CONFIG_BEGIN
 
config.window_visual = PIGLIT_GL_VISUAL_RGB | PIGLIT_GL_VISUAL_DOUBLE;
config.khr_no_error_support = PIGLIT_NO_ERRORS;
+   const char *tests[] = {
+   "0x0 texture",
+   "0x0 renderbuffer",
+   "invalid slice of 3D texture",
+   "invalid layer of a 1D-array texture",
+   "invalid layer of a 2D-array texture",
+   "invalid layer of a cube-array texture",
+   "delete texture of bound FBO",
+   "delete renderbuffer of bound FBO",
+   NULL
+   };
+   config.all_subtests = tests;
 
 PIGLIT_GL_TEST_CONFIG_END
 
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 05/15] framework: if a test with subtests crashes mark the offending subtest

[Dylan Baker]

This relies on the fact that subtests are guaranteed to be ordered to
mark the crashing subtest as such. This ensures that the correct status
will be propagated up the totals tree.
---
 framework/test/base.py|  8 ++-
 unittests/framework/test/test_base.py | 36 -
 2 files changed, 44 insertions(+)

diff --git a/framework/test/base.py b/framework/test/base.py
index 2d761ef..134b872 100644
--- a/framework/test/base.py
+++ b/framework/test/base.py
@@ -236,6 +236,14 @@ class Test(object):
 """
 if is_crash_returncode(self.result.returncode):
 self.result.result = status.CRASH
+if self.result.subtests:
+# We know because subtests are ordered that the first test with
+# a status of NOTRUN is the subtest that crashed, mark that
+# test and move on.
+for k, v in six.iteritems(self.result.subtests):
+if v == status.NOTRUN:
+self.result.subtests[k] = status.CRASH
+break
 elif self.result.returncode != 0:
 if self.result.result == status.PASS:
 self.result.result = status.WARN
diff --git a/unittests/framework/test/test_base.py 
b/unittests/framework/test/test_base.py
index 656d839..b1b2cdb 100644
--- a/unittests/framework/test/test_base.py
+++ b/unittests/framework/test/test_base.py
@@ -33,6 +33,8 @@ except ImportError:
 import pytest
 import six
 
+from six.moves import range
+
 from framework import dmesg
 from framework import log
 from framework import monitoring
@@ -283,6 +285,40 @@ class TestTest(object):
 
 assert test.result.result is status.FAIL
 
+def test_crash_subtest_before_start(self):
+"""A test for a a test with a subtest, that crashes in the middle
+of the run.
+"""
+test = _Test(['foobar'])
+test.result.returncode = -1
+for x in (str(y) for y in range(5)):
+test.result.subtests[x] = status.NOTRUN
+test.interpret_result()
+
+assert test.result.result is status.CRASH
+assert test.result.subtests['0'] is status.CRASH
+for x in (str(y) for y in range(1, 5)):
+assert test.result.subtests[x] is status.NOTRUN
+
+def test_crash_subtest_mid(self):
+"""A test for a a test with a subtest, that crashes in the middle
+of the run.
+"""
+test = _Test(['foobar'])
+test.result.returncode = -1
+for x in (str(y) for y in range(2)):
+test.result.subtests[x] = status.PASS
+for x in (str(y) for y in range(2, 5)):
+test.result.subtests[x] = status.NOTRUN
+test.interpret_result()
+
+assert test.result.result is status.CRASH
+for x in (str(y) for y in range(2)):
+assert test.result.subtests[x] is status.PASS
+assert test.result.subtests['2'] is status.CRASH
+for x in (str(y) for y in range(3, 5)):
+assert test.result.subtests[x] is status.NOTRUN
+
 
 class TestWindowResizeMixin(object):
 """Tests for the WindowResizeMixin class."""
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 03/15] framework: use statuses instead of relying on string conversions

[Dylan Baker]

This is a small cleanup, but we need to touch this code further for
subtest enumeration work, and we'll want to use statuses there too.
---
 framework/test/base.py | 8 
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/framework/test/base.py b/framework/test/base.py
index 0b7ebab..2d761ef 100644
--- a/framework/test/base.py
+++ b/framework/test/base.py
@@ -235,12 +235,12 @@ class Test(object):
 """Convert the raw output of the test into a form piglit understands.
 """
 if is_crash_returncode(self.result.returncode):
-self.result.result = 'crash'
+self.result.result = status.CRASH
 elif self.result.returncode != 0:
-if self.result.result == 'pass':
-self.result.result = 'warn'
+if self.result.result == status.PASS:
+self.result.result = status.WARN
 else:
-self.result.result = 'fail'
+self.result.result = status.FAIL
 
 def run(self):
 """
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 09/15] tests/fbo-storage-formats: Make subtest names predictable

[Dylan Baker]

Currently the name of the subtest relies on checking the GL state of the
command to know whether the framebuffer is completely or incomplete.
That is a problem for enumerating subtests, we would need to know ahead
of time whether the framebuffer is complete or not.

Instead print the completeness of the test apart from the subtest name,
and name the subtest name just the format name. This also makes the name
the same when the test fails and when in passes.

Signed-off-by: Dylan Baker <dylan.c.ba...@intel.com>
---
 tests/fbo/fbo-storage-formats.c |  9 -
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/tests/fbo/fbo-storage-formats.c b/tests/fbo/fbo-storage-formats.c
index 91b80e0..9cf8c67 100644
--- a/tests/fbo/fbo-storage-formats.c
+++ b/tests/fbo/fbo-storage-formats.c
@@ -200,11 +200,10 @@ test(void)
pass = GL_FALSE;
} else {
GLenum status = 
glCheckFramebufferStatus(GL_FRAMEBUFFER);
-   piglit_report_subtest_result(PIGLIT_PASS,
-"%s (%s)",
-name,
-(status == 
GL_FRAMEBUFFER_COMPLETE ?
- "complete" : 
"incomplete"));
+   printf("%s is %s\n",
+  name,
+  (status == GL_FRAMEBUFFER_COMPLETE ?  "complete" 
: "incomplete"));
+   piglit_report_subtest_result(PIGLIT_PASS, "%s", name);
}
}
 
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 11/15] tests/fbo-storage-formats: enumerate subtests

[Dylan Baker]

---
 tests/fbo/fbo-storage-formats.c | 41 ++
 1 file changed, 27 insertions(+), 14 deletions(-)

diff --git a/tests/fbo/fbo-storage-formats.c b/tests/fbo/fbo-storage-formats.c
index 7f13684..ee22883 100644
--- a/tests/fbo/fbo-storage-formats.c
+++ b/tests/fbo/fbo-storage-formats.c
@@ -32,31 +32,17 @@
 
 #include "piglit-util-gl.h"
 
-PIGLIT_GL_TEST_CONFIG_BEGIN
-
-   config.supports_gl_compat_version = 10;
-
-   config.window_visual = PIGLIT_GL_VISUAL_RGB;
-   config.khr_no_error_support = PIGLIT_NO_ERRORS;
-
-PIGLIT_GL_TEST_CONFIG_END
-
 #define EXT_packed_depth_stencil 1
 #define ARB_framebuffer_object 2
 #define ARB_texture_rg 3
 #define MAX_EXT 4
 
-static GLboolean have_extension[MAX_EXT];
-
-
-
 struct format_info
 {
GLenum format;
GLuint extension;
 };
 
-
 static const struct format_info formats[] = {
{ GL_RGB, 0 },
{ GL_R3_G3_B2, 0 },
@@ -158,6 +144,33 @@ static const GLenum invalid_formats[] = {
 };
 
 
+PIGLIT_GL_TEST_CONFIG_BEGIN
+
+   config.supports_gl_compat_version = 10;
+
+   config.window_visual = PIGLIT_GL_VISUAL_RGB;
+   config.khr_no_error_support = PIGLIT_NO_ERRORS;
+
+   const char *names[64];
+
+   int t = 0;
+   for (int i = 0; i < ARRAY_SIZE(formats); i++) {
+   names[t] = piglit_get_gl_enum_name(formats[i].format);
+   ++t;
+   }
+   for (int i = 0; i < ARRAY_SIZE(invalid_formats); i++) {
+   names[t] = piglit_get_gl_enum_name(invalid_formats[i]);
+   ++t;
+   }
+   names[++t] = NULL;
+
+   config.all_subtests = names;
+
+PIGLIT_GL_TEST_CONFIG_END
+
+static GLboolean have_extension[MAX_EXT];
+
+
 static enum piglit_result
 test(void)
 {
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 10/15] tests/fbo-storage-formats: print subtest result for skip too

[Dylan Baker]

Because we always want to have the same number of subtests printed, or
they'll show up as "NOTRUN" instead of "SKIP" in the summary.

Signed-off-by: Dylan Baker <dylan.c.ba...@intel.com>
---
 tests/fbo/fbo-storage-formats.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/fbo/fbo-storage-formats.c b/tests/fbo/fbo-storage-formats.c
index 9cf8c67..7f13684 100644
--- a/tests/fbo/fbo-storage-formats.c
+++ b/tests/fbo/fbo-storage-formats.c
@@ -190,8 +190,10 @@ test(void)
for (i = 0; i < ARRAY_SIZE(formats); i++) {
const char *name = piglit_get_gl_enum_name(formats[i].format);
 
-   if (!have_extension[formats[i].extension])
+   if (!have_extension[formats[i].extension]) {
+   piglit_report_subtest_result(PIGLIT_SKIP, "%s", name);
continue;
+   }
 
glRenderbufferStorageEXT(GL_RENDERBUFFER_EXT, formats[i].format,
 piglit_width, piglit_height);
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 14/15] tests: enumerate subtests in linestipple

[Dylan Baker]

---
 tests/general/linestipple.c | 24 
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/tests/general/linestipple.c b/tests/general/linestipple.c
index ba0924e..6626b3c 100644
--- a/tests/general/linestipple.c
+++ b/tests/general/linestipple.c
@@ -28,14 +28,6 @@
 
 #include "piglit-util-gl.h"
 
-PIGLIT_GL_TEST_CONFIG_BEGIN
-
-   config.supports_gl_compat_version = 10;
-
-   config.window_visual = PIGLIT_GL_VISUAL_RGB | PIGLIT_GL_VISUAL_DOUBLE;
-
-PIGLIT_GL_TEST_CONFIG_END
-
 struct vertex {
GLuint x;
GLuint y;
@@ -198,6 +190,22 @@ static struct stipple_line Lines[] = {
}
 };
 
+PIGLIT_GL_TEST_CONFIG_BEGIN
+
+   config.supports_gl_compat_version = 10;
+
+   config.window_visual = PIGLIT_GL_VISUAL_RGB | PIGLIT_GL_VISUAL_DOUBLE;
+
+   const int numLines = ARRAY_SIZE(Lines);
+   const char *subtests[ARRAY_SIZE(Lines)];
+   for (int i = 0; i < numLines; i++) {
+   subtests[i] = Lines[i].name;
+   }
+   subtests[numLines] = NULL;
+   config.all_subtests = subtests;
+
+PIGLIT_GL_TEST_CONFIG_END
+
 enum piglit_result
 piglit_display(void)
 {
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 02/15] framework: use an OrderedDict for subtest storage

[Dylan Baker]

This makes the order of the subtests deterministic. Since piglit test
binaries are expected to enumerate subtests in the same order that
they'll run them this will allow us to know which subtest crashed when
there is a crash.
---
 framework/results.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/framework/results.py b/framework/results.py
index 99dd373..2a3e0ba 100644
--- a/framework/results.py
+++ b/framework/results.py
@@ -41,7 +41,7 @@ __all__ = [
 class Subtests(collections.MutableMapping):
 """A dict-like object that stores Statuses as values."""
 def __init__(self, dict_=None):
-self.__container = {}
+self.__container = collections.OrderedDict()
 
 if dict_ is not None:
 self.update(dict_)
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 04/15] framework: add support for parsing subtest enumeration

[Dylan Baker]

This adds support for enumerating subtests to the python layer. When it
sees this it sets each subtest to notrun. This allows the python
framework to report that tests didn't run when they were expected to.

Signed-off-by: Dylan Baker <dylan.c.ba...@intel.com>
---
 framework/test/piglit_test.py | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index 491f3d3..950a762 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -34,6 +34,7 @@ except ImportError:
 import json
 
 from framework import core, options
+from framework import status
 from .base import Test, WindowResizeMixin, ValgrindMixin, TestIsSkip
 
 
@@ -73,7 +74,12 @@ class PiglitBaseTest(ValgrindMixin, Test):
 
 for each in self.result.out.split('\n'):
 if each.startswith('PIGLIT:'):
-self.result.update(json.loads(each[8:]))
+deserial = json.loads(each[8:])
+if 'enumerate subtests' in deserial:
+for n in deserial['enumerate subtests']:
+self.result.subtests[n] = status.NOTRUN
+else:
+self.result.update(deserial)
 else:
 out.append(each)
 
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [PATCH] framework: Propagate crash to subtest.

[Dylan Baker]

Quoting Fabian Bieler (2018-01-20 05:38:45)
> If a subtest crashed it and all following subtests have the result notrun.
> Only the test as a whole has the result crash.
> This commit propagates the crash to the correct subtest.
> 
> Note that if the crash happened after the last subtest it is silently ignored.
> ---
> I like this approach. Personally, I would have added a NULL terminated array 
> of char* to config that defaults to NULL and emitted the JSON structure in 
> PIGLIT_GL_TEST_CONFIG_END, but that's just details.

I'm not a great C/C++ programmer, so if you have a better suggestion on how to
enumerate the subtests I'd be glad to do that instead.

> 
> Also this needs a little more work in the python framework: As is crashes 
> don't show up in the result summary (regardless of whether the crash happens 
> before or after the first call to piglit_report_subtest_result).
> 
> If we use an OrderedDict instead of a regular dict in patch [3/9] and as the 
> container of Subtests (framework/results.py:44) we could deduce the crashing 
> subtest (if the order is preserved between piglit_enumerate_subtests and test 
> execution) with this patch.

You're right. I had a versions of this I was working on at one point, but
apparently lost when migrating machines or deleted :/

I think using an ordered dict would be better, in that case we can mark the
correct subtest as crashed during the test run and not need to mess with any
other code.

> 
> Note that this would differ from the previous behavior in that if a test 
> crashes before a call to piglit_report_subtest_result (as is common in CL 
> tests, I am told) the first subtest is marked as crashed and the whole test 
> doesn't show up in the summary.
> 
> 
> I'll gladly help with the adaption of the remaining tests if needed.
> 
>  framework/results.py | 18 ++
>  1 file changed, 18 insertions(+)
> 
> diff --git a/framework/results.py b/framework/results.py
> index 99dd3735b..2e1b832fa 100644
> --- a/framework/results.py
> +++ b/framework/results.py
> @@ -69,6 +69,22 @@ class Subtests(collections.MutableMapping):
>  res['__type__'] = 'Subtests'
>  return res
>  
> +def set_crash(self):
> +"""Find the first subtest marked as notrun and mark it as crashed.
> +
> +If a subtest crashed it and all following subtests have the result
> +notrun. Only the test as a whole has the result crash.
> +This function propagates the crash to the correct subtest.
> +Note that if the crash happened after the last subtest it is silently
> +ignored.
> +"""
> +if status.CRASH in six.itervalues(self):
> +return
> +for key, result in six.iteritems(self):
> +if result == status.NOTRUN:
> +self[key] = status.CRASH
> +break
> +
>  @classmethod
>  def from_dict(cls, dict_):
>  if '__type__' in dict_:
> @@ -232,6 +248,8 @@ class TestResult(object):
>  # Set special instances
>  if 'subtests' in dict_:
>  inst.subtests = Subtests.from_dict(dict_['subtests'])
> +if inst.result == status.CRASH:
> +inst.subtests.set_crash()
>  if 'time' in dict_:
>  inst.time = TimeAttribute.from_dict(dict_['time'])
>  
> -- 
> 2.15.1
> 


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

Re: [Piglit] [RFC 1/9] util: Add a function for enumerating subtests

[Dylan Baker]

I meant to add a cover letter to this, also, there's only 8 patches.

This is an RFC, I'm looking for input into fixing the subtest reporting problem.
This approach is going to be more complete, but it's going to be pretty
involved. If anyone has a better approach I'd be happy to hear it.

Quoting Dylan Baker (2018-01-19 16:25:56)
> This function takes one or more subtests as strings and returns a JSON
> structure that the python framework can consume.
> 
> Signed-off-by: Dylan Baker <dylan.c.ba...@intel.com>
> ---
>  tests/util/piglit-util.c | 29 +
>  tests/util/piglit-util.h |  2 ++
>  2 files changed, 31 insertions(+)
> 
> diff --git a/tests/util/piglit-util.c b/tests/util/piglit-util.c
> index e33d055..46b4b75 100644
> --- a/tests/util/piglit-util.c
> +++ b/tests/util/piglit-util.c
> @@ -290,6 +290,35 @@ piglit_report_subtest_result(enum piglit_result result, 
> const char *format, ...)
> va_end(ap);
>  }
>  
> +void
> +piglit_enumerate_subtests(int num_args, const char *name, ...)
> +{
> +   va_list ap;
> +
> +   va_start(ap, name);
> +
> +   printf("PIGLIT: {\"enumerate subtests\": [\"%s\"", name);
> +   for (int i = 1; i < num_args; i++) {
> +   vprintf(", \"%s\"", ap);
> +   }
> +   printf("]}\n");
> +   fflush(stdout);
> +
> +   va_end(ap);
> +}
> +
> +void
> +piglit_enumerate_subtest_list(int length, const char *names[])
> +{
> +   assert(length > 0);
> +   printf("PIGLIT: {\"enumerate subtests\": [\"%s\"", names[0]);
> +   for (int i = 1; i < length; i++) {
> +   printf(", \"%s\"", names[i]);
> +   }
> +   printf("]}\n");
> +   fflush(stdout);
> +}
> +
>  
>  static void
>  piglit_disable_error_message_boxes(void)
> diff --git a/tests/util/piglit-util.h b/tests/util/piglit-util.h
> index 3757f86..c5adb11 100644
> --- a/tests/util/piglit-util.h
> +++ b/tests/util/piglit-util.h
> @@ -360,6 +360,8 @@ NORETURN void piglit_report_result(enum piglit_result 
> result);
>  void piglit_set_timeout(double seconds, enum piglit_result timeout_result);
>  void piglit_report_subtest_result(enum piglit_result result,
>   const char *format, ...) PRINTFLIKE(2, 3);
> +void piglit_enumerate_subtests(int num_args, const char *name, ...);
> +void piglit_enumerate_subtest_list(int length, const char *names[]);
>  
>  void piglit_general_init(void);
>  
> 
> base-commit: 736496667329bf73a706aebec6f8287078df79ae
> -- 
> git-series 0.9.1
> ___
> Piglit mailing list
> Piglit@lists.freedesktop.org
> https://lists.freedesktop.org/mailman/listinfo/piglit


signature.asc
Description: signature
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 5/9] tests/fbo-storage-formats: Always print the same number of subtests

[Dylan Baker]

!skip -> skip won't show up in the regressions/fixes/etc lists anyway,
and this means that the output will always be the same.
---
 tests/fbo/fbo-storage-formats.c | 6 ++
 1 file changed, 6 insertions(+)

diff --git a/tests/fbo/fbo-storage-formats.c b/tests/fbo/fbo-storage-formats.c
index 3ecd07a..91b80e0 100644
--- a/tests/fbo/fbo-storage-formats.c
+++ b/tests/fbo/fbo-storage-formats.c
@@ -223,6 +223,12 @@ test(void)
piglit_report_subtest_result(PIGLIT_PASS, "%s", 
name);
}
}
+   } else {
+   printf("Skipping error tests because KHR_NO_ERROR is 
enabled\n");
+   for (i = 0; i < ARRAY_SIZE(invalid_formats); i++) {
+   const char *name = 
piglit_get_gl_enum_name(invalid_formats[i]);
+   piglit_report_subtest_result(PIGLIT_SKIP, "%s", name);
+   }
}
 
return pass ? PIGLIT_PASS : PIGLIT_FAIL;
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 3/9] framework: add support for parsing subtest enumeration

[Dylan Baker]

This adds support for enumerating subtests to the python layer. When it
sees this it sets each subtest to notrun. This allows the python
framework to report that tests didn't run when they were expected to.

Signed-off-by: Dylan Baker <dylan.c.ba...@intel.com>
---
 framework/test/piglit_test.py | 8 +++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/framework/test/piglit_test.py b/framework/test/piglit_test.py
index 491f3d3..a7406c1 100644
--- a/framework/test/piglit_test.py
+++ b/framework/test/piglit_test.py
@@ -34,6 +34,7 @@ except ImportError:
 import json
 
 from framework import core, options
+from framework import status
 from .base import Test, WindowResizeMixin, ValgrindMixin, TestIsSkip
 
 
@@ -73,7 +74,12 @@ class PiglitBaseTest(ValgrindMixin, Test):
 
 for each in self.result.out.split('\n'):
 if each.startswith('PIGLIT:'):
-self.result.update(json.loads(each[8:]))
+deserial = json.loads(each[8:])
+if 'enumerate subtests' in deserial:
+self.result.subtests.update(
+{n: status.NOTRUN for n in deserial['enumerate 
subtests']})
+else:
+self.result.update(deserial)
 else:
 out.append(each)
 
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit

[Piglit] [PATCH 8/9] tests/fbo-storage-formats: enumerate subtests

[Dylan Baker]

---
 tests/fbo/fbo-storage-formats.c | 22 ++
 1 file changed, 22 insertions(+)

diff --git a/tests/fbo/fbo-storage-formats.c b/tests/fbo/fbo-storage-formats.c
index 4db990d..a11335b 100644
--- a/tests/fbo/fbo-storage-formats.c
+++ b/tests/fbo/fbo-storage-formats.c
@@ -243,9 +243,31 @@ piglit_display(void)
 }
 
 
+static void
+enumerate_subtests(void)
+{
+   static int len = ARRAY_SIZE(formats) + ARRAY_SIZE(invalid_formats);
+   const char* names[64];
+
+   int t = 0;
+   for (int i = 0; i < ARRAY_SIZE(formats); i++) {
+   names[t] = piglit_get_gl_enum_name(formats[i].format);
+   ++t;
+   }
+   for (int i = 0; i < ARRAY_SIZE(invalid_formats); i++) {
+   names[t] = piglit_get_gl_enum_name(invalid_formats[i]);
+   ++t;
+   }
+
+   piglit_enumerate_subtest_list(len, names);
+}
+
+
 void
 piglit_init(int argc, char**argv)
 {
+   enumerate_subtests();
+
piglit_require_extension("GL_EXT_framebuffer_object");
 
have_extension[0] = GL_TRUE;
-- 
git-series 0.9.1
___
Piglit mailing list
Piglit@lists.freedesktop.org
https://lists.freedesktop.org/mailman/listinfo/piglit