Re: [PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Eric Fiselier via cfe-commits]

Fixed in r288863.

On Tue, Dec 6, 2016 at 4:00 PM, Eric Fiselier  wrote:

> Investigating right now. Sorry for the breakage.
>
> On Dec 6, 2016 3:55 PM, "Matthias Braun via Phabricator" <
> revi...@reviews.llvm.org> wrote:
>
> MatzeB added a comment.
>
> This breaks the test-suite lit scripts:
>
> Exception during script execution:
> Traceback (most recent call last):
>
>   File "/Users/mbraun/dev/public_llvm/utils/lit/lit/run.py", line 183, in
> execute_test
> result = test.config.test_format.execute(test, self.lit_config)
>   File "/Users/mbraun/dev/test-suite/litsupport/test.py", line 73, in
> execute
> testfile.parse(context, test.getSourcePath())
>   File "/Users/mbraun/dev/test-suite/litsupport/testfile.py", line 45, in
> parse
> command_type,))
>
> ValueError: unknown script command type: 'RUN:'
>
>
> https://reviews.llvm.org/D27005
>
>
>
>
>
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Eric Fiselier via cfe-commits]

Investigating right now. Sorry for the breakage.

On Dec 6, 2016 3:55 PM, "Matthias Braun via Phabricator" <
revi...@reviews.llvm.org> wrote:

MatzeB added a comment.

This breaks the test-suite lit scripts:

Exception during script execution:
Traceback (most recent call last):

  File "/Users/mbraun/dev/public_llvm/utils/lit/lit/run.py", line 183, in
execute_test
result = test.config.test_format.execute(test, self.lit_config)
  File "/Users/mbraun/dev/test-suite/litsupport/test.py", line 73, in
execute
testfile.parse(context, test.getSourcePath())
  File "/Users/mbraun/dev/test-suite/litsupport/testfile.py", line 45, in
parse
command_type,))

ValueError: unknown script command type: 'RUN:'


https://reviews.llvm.org/D27005
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Matthias Braun via Phabricator via cfe-commits]

MatzeB added a comment.

This breaks the test-suite lit scripts:

Exception during script execution:
Traceback (most recent call last):

  File "/Users/mbraun/dev/public_llvm/utils/lit/lit/run.py", line 183, in 
execute_test
result = test.config.test_format.execute(test, self.lit_config)
  File "/Users/mbraun/dev/test-suite/litsupport/test.py", line 73, in execute
testfile.parse(context, test.getSourcePath())
  File "/Users/mbraun/dev/test-suite/litsupport/testfile.py", line 45, in parse
command_type,))

ValueError: unknown script command type: 'RUN:'


https://reviews.llvm.org/D27005



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Jonathan Roelofs via Phabricator via cfe-commits]

jroelofs added a comment.

LGTM


https://reviews.llvm.org/D27005



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Eric Fiselier via Phabricator via cfe-commits]

EricWF updated this revision to Diff 80168.
EricWF added a comment.

- Add unit tests as requested.

This patch is ready to go. If there are no objections in the next day or two 
I'll commit it.


https://reviews.llvm.org/D27005

Files:
  utils/lit/lit/TestRunner.py
  utils/lit/tests/Inputs/testrunner-custom-parsers/lit.cfg
  utils/lit/tests/Inputs/testrunner-custom-parsers/test.txt
  utils/lit/tests/unit/TestRunner.py

Index: utils/lit/tests/unit/TestRunner.py
===
--- /dev/null
+++ utils/lit/tests/unit/TestRunner.py
@@ -0,0 +1,114 @@
+# RUN: %{python} %s
+#
+# END.
+
+
+import unittest
+import platform
+import os.path
+import tempfile
+
+import lit
+from lit.TestRunner import ParserKind, IntegratedTestKeywordParser, \
+   parseIntegratedTestScript
+
+
+class TestIntegratedTestKeywordParser(unittest.TestCase):
+inputTestCase = None
+
+@staticmethod
+def load_keyword_parser_lit_tests():
+"""
+Create and load the LIT test suite and test objects used by
+TestIntegratedTestKeywordParser
+"""
+# Create the global config object.
+lit_config = lit.LitConfig.LitConfig(progname='lit',
+ path=[],
+ quiet=False,
+ useValgrind=False,
+ valgrindLeakCheck=False,
+ valgrindArgs=[],
+ noExecute=False,
+ debug=False,
+ isWindows=(
+   platform.system() == 'Windows'),
+ params={})
+TestIntegratedTestKeywordParser.litConfig = lit_config
+# Perform test discovery.
+test_path = os.path.dirname(os.path.dirname(__file__))
+inputs = [os.path.join(test_path, 'Inputs/testrunner-custom-parsers/')]
+assert os.path.isdir(inputs[0])
+run = lit.run.Run(lit_config,
+  lit.discovery.find_tests_for_inputs(lit_config, inputs))
+assert len(run.tests) == 1 and "there should only be one test"
+TestIntegratedTestKeywordParser.inputTestCase = run.tests[0]
+
+@staticmethod
+def make_parsers():
+def custom_parse(line_number, line, output):
+if output is None:
+output = []
+output += [part for part in line.split(' ') if part.strip()]
+return output
+
+return [
+IntegratedTestKeywordParser("MY_TAG.", ParserKind.TAG),
+IntegratedTestKeywordParser("MY_DNE_TAG.", ParserKind.TAG),
+IntegratedTestKeywordParser("MY_LIST:", ParserKind.LIST),
+IntegratedTestKeywordParser("MY_RUN:", ParserKind.COMMAND),
+IntegratedTestKeywordParser("MY_CUSTOM:", ParserKind.CUSTOM,
+custom_parse)
+]
+
+@staticmethod
+def get_parser(parser_list, keyword):
+for p in parser_list:
+if p.keyword == keyword:
+return p
+assert False and "parser not found"
+
+@staticmethod
+def parse_test(parser_list):
+script = parseIntegratedTestScript(
+TestIntegratedTestKeywordParser.inputTestCase,
+additional_parsers=parser_list, require_script=False)
+assert not isinstance(script, lit.Test.Result)
+assert isinstance(script, list)
+assert len(script) == 0
+
+def test_tags(self):
+parsers = self.make_parsers()
+self.parse_test(parsers)
+tag_parser = self.get_parser(parsers, 'MY_TAG.')
+dne_tag_parser = self.get_parser(parsers, 'MY_DNE_TAG.')
+self.assertTrue(tag_parser.getValue())
+self.assertFalse(dne_tag_parser.getValue())
+
+def test_lists(self):
+parsers = self.make_parsers()
+self.parse_test(parsers)
+list_parser = self.get_parser(parsers, 'MY_LIST:')
+self.assertItemsEqual(list_parser.getValue(),
+  ['one', 'two', 'three', 'four'])
+
+def test_commands(self):
+parsers = self.make_parsers()
+self.parse_test(parsers)
+cmd_parser = self.get_parser(parsers, 'MY_RUN:')
+value = cmd_parser.getValue()
+self.assertEqual(len(value), 2)  # there are only two run lines
+self.assertEqual(value[0].strip(), 'baz')
+self.assertEqual(value[1].strip(), 'foo  bar')
+
+def test_custom(self):
+parsers = self.make_parsers()
+self.parse_test(parsers)
+custom_parser = self.get_parser(parsers, 'MY_CUSTOM:')
+value = custom_parser.getValue()
+self.assertItemsEqual(value, ['a', 'b', 'c'])
+
+
+if __name__ == '__main__':
+

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Jonathan Roelofs via cfe-commits]

jroelofs added a comment.

In https://reviews.llvm.org/D27005#603979, @EricWF wrote:

> In https://reviews.llvm.org/D27005#603692, @jroelofs wrote:
>
> > Should probably add a testcase in lit/tests that exercises the new CUSTOM 
> > parser stuff, so people working on LIT don't have to build/test libc++ in 
> > order to know whether they've broken its testsuite.
>
>
> I plan to implement tests. I just wanted to ensure the approach was agreeable 
> first.


Looks like a good approach to me.


https://reviews.llvm.org/D27005



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Eric Fiselier via cfe-commits]

EricWF added a comment.

In https://reviews.llvm.org/D27005#603692, @jroelofs wrote:

> Should probably add a testcase in lit/tests that exercises the new CUSTOM 
> parser stuff, so people working on LIT don't have to build/test libc++ in 
> order to know whether they've broken its testsuite.


I plan to implement tests. I just wanted to ensure the approach was agreeable 
first.


https://reviews.llvm.org/D27005



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Jonathan Roelofs via cfe-commits]

jroelofs added a comment.

Should probably add a testcase in lit/tests that exercises the new CUSTOM 
parser stuff, so people working on LIT don't have to build/test libc++ in order 
to know whether they've broken its testsuite.


https://reviews.llvm.org/D27005



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Eric Fiselier via cfe-commits]

EricWF updated this revision to Diff 78988.
EricWF added a comment.

Pepify the changed lines.


https://reviews.llvm.org/D27005

Files:
  utils/lit/lit/TestRunner.py

Index: utils/lit/lit/TestRunner.py
===
--- utils/lit/lit/TestRunner.py
+++ utils/lit/lit/TestRunner.py
@@ -630,7 +630,7 @@
 # version.
 
 keywords_re = re.compile(
-to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
+to_bytes("(%s)(.*)\n" % ("|".join(re.escape(k) for k in keywords),)))
 
 f = open(source_path, 'rb')
 try:
@@ -657,7 +657,7 @@
 # Python 2, to avoid other code having to differentiate between the
 # str and unicode types.
 keyword,ln = match.groups()
-yield (line_number, to_string(keyword[:-1].decode('utf-8')),
+yield (line_number, to_string(keyword.decode('utf-8')),
to_string(ln.decode('utf-8')))
 finally:
 f.close()
@@ -739,10 +739,119 @@
 # convert to list before returning.
 return list(map(processLine, script))
 
-def parseIntegratedTestScript(test, require_script=True):
+
+class ParserKind(object):
+"""
+An enumeration representing the style of an integrated test keyword or
+command.
+
+TAG: A keyword taking no value. Ex 'END.'
+COMMAND: A Keyword taking a list of shell commands. Ex 'RUN:'
+LIST: A keyword taking a comma separated list of value. Ex 'XFAIL:'
+CUSTOM: A keyword with custom parsing semantics.
+"""
+TAG = 0
+COMMAND = 1
+LIST = 2
+CUSTOM = 3
+
+
+class IntegratedTestKeywordParser(object):
+"""A parser for LLVM/Clang style integrated test scripts.
+
+keyword: The keyword to parse for. It must end in either '.' or ':'.
+kind: An value of ParserKind.
+parser: A custom parser. This value may only be specified with
+ParserKind.CUSTOM.
+"""
+def __init__(self, keyword, kind, parser=None, initial_value=None):
+if not keyword.endswith('.') and not keyword.endswith(':'):
+raise ValueError("keyword '%s' must end with either '.' or ':' "
+ % keyword)
+if keyword.endswith('.') and kind in \
+[ParserKind.LIST, ParserKind.COMMAND]:
+raise ValueError("Keyword '%s' should end in ':'" % keyword)
+
+elif keyword.endswith(':') and kind in [ParserKind.TAG]:
+raise ValueError("Keyword '%s' should end in '.'" % keyword)
+if parser is not None and kind != ParserKind.CUSTOM:
+raise ValueError("custom parsers can only be specified with "
+ "ParserKind.CUSTOM")
+self.keyword = keyword
+self.kind = kind
+self.parsed_lines = []
+self.value = initial_value
+self.parser = parser
+
+if kind == ParserKind.COMMAND:
+self.parser = self._handleCommand
+elif kind == ParserKind.LIST:
+self.parser = self._handleList
+elif kind == ParserKind.TAG:
+if not keyword.endswith('.'):
+raise ValueError("keyword '%s' should end with '.'" % keyword)
+self.parser = self._handleTag
+elif kind == ParserKind.CUSTOM:
+if parser is None:
+raise ValueError("ParserKind.CUSTOM requires a custom parser")
+self.parser = parser
+else:
+raise ValueError("Unknown kind '%s'" % kind)
+
+def parseLine(self, line_number, line):
+self.parsed_lines += [(line_number, line)]
+self.value = self.parser(line_number, line, self.value)
+
+def getValue(self):
+return self.value
+
+@staticmethod
+def _handleTag(line_number, line, output):
+"""A helper for parsing TAG type keywords"""
+return (not line.strip() or output)
+
+@staticmethod
+def _handleCommand(line_number, line, output):
+"""A helper for parsing COMMAND type keywords"""
+# Trim trailing whitespace.
+line = line.rstrip()
+# Substitute line number expressions
+line = re.sub('%\(line\)', str(line_number), line)
+
+def replace_line_number(match):
+if match.group(1) == '+':
+return str(line_number + int(match.group(2)))
+if match.group(1) == '-':
+return str(line_number - int(match.group(2)))
+line = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, line)
+# Collapse lines with trailing '\\'.
+if output and output[-1][-1] == '\\':
+output[-1] = output[-1][:-1] + line
+else:
+if output is None:
+output = []
+output.append(line)
+return output
+
+@staticmethod
+def _handleList(line_number, line, output):
+"""A parser for LIST type keywords"""
+if output is None:
+output = []
+output.extend([s.strip() 

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Mandeep Singh Grang via cfe-commits]

mgrang added a comment.

I ran pep8 on this script:

line 742: E302 expected 2 blank lines, found 1
line 757: E302 expected 2 blank lines, found 1
line 819: E301 expected 1 blank line, found 0

Can you pep-ify this script?


https://reviews.llvm.org/D27005



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[PATCH] D27005: [lit] Support custom parsers in parseIntegratedTestScript

[Eric Fiselier via cfe-commits]

EricWF created this revision.
EricWF added reviewers: ddunbar, modocache, rnk, danalbert, jroelofs.
EricWF added subscribers: cfe-commits, llvm-commits.

Libc++ frequently has the need to parse more than just the builtin *test 
keywords* (`RUN`, `REQUIRES`, `XFAIL`, ect). For example libc++ currently needs 
a new keyword `MODULES-DEFINES: macro list...`. Instead of re-implementing the 
script parsing in libc++ this patch allows `parseIntegratedTestScript` to take 
custom parsers.

This patch introduces a new class `IntegratedTestKeywordParser` which 
implements the logic to parse/process a test keyword. Parsing of various 
keyword "kinds" are supported out of the box, including 'TAG', 'COMMAND', and 
'LIST', which parse keywords such as `END.`, `RUN:` and `XFAIL:` respectively.

As an example after this change libc++ can implement the `MODULES-DEFINES` 
simply using:

  mparser = IntegratedTestKeywordParser('MODULES-DEFINES:', ParserKind.LIST)
  parseIntegratedTestScript(test, additional_parsers=[mparser])
  macro_list = mparser.getValue()


https://reviews.llvm.org/D27005

Files:
  utils/lit/lit/TestRunner.py

Index: utils/lit/lit/TestRunner.py
===
--- utils/lit/lit/TestRunner.py
+++ utils/lit/lit/TestRunner.py
@@ -630,7 +630,7 @@
 # version.
 
 keywords_re = re.compile(
-to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
+to_bytes("(%s)(.*)\n" % ("|".join(re.escape(k) for k in keywords),)))
 
 f = open(source_path, 'rb')
 try:
@@ -657,7 +657,7 @@
 # Python 2, to avoid other code having to differentiate between the
 # str and unicode types.
 keyword,ln = match.groups()
-yield (line_number, to_string(keyword[:-1].decode('utf-8')),
+yield (line_number, to_string(keyword.decode('utf-8')),
to_string(ln.decode('utf-8')))
 finally:
 f.close()
@@ -739,10 +739,116 @@
 # convert to list before returning.
 return list(map(processLine, script))
 
-def parseIntegratedTestScript(test, require_script=True):
+class ParserKind(object):
+"""
+An enumeration representing the style of an integrated test keyword or
+command.
+
+TAG: A keyword taking no value. Ex 'END.'
+COMMAND: A Keyword taking a list of shell commands. Ex 'RUN:'
+LIST: A keyword taking a comma separated list of value. Ex 'XFAIL:'
+CUSTOM: A keyword with custom parsing semantics.
+"""
+TAG = 0
+COMMAND = 1
+LIST = 2
+CUSTOM = 3
+
+class IntegratedTestKeywordParser(object):
+"""A parser for LLVM/Clang style integrated test scripts.
+
+keyword: The keyword to parse for. It must end in either '.' or ':'.
+kind: An value of ParserKind.
+parser: A custom parser. This value may only be specified with
+ParserKind.CUSTOM.
+"""
+def __init__(self, keyword, kind, parser=None, initial_value=None):
+if not keyword.endswith('.') and not keyword.endswith(':'):
+raise ValueError("keyword '%s' must end with either '.' or ':' "
+  % keyword)
+if keyword.endswith('.') and kind in \
+[ParserKind.LIST, ParserKind.COMMAND]:
+raise ValueError("Keyword '%s' should end in ':'" % keyword)
+
+elif keyword.endswith(':') and kind in [ParserKind.TAG]:
+raise ValueError("Keyword '%s' should end in '.'" % keyword)
+if parser is not None and kind != ParserKind.CUSTOM:
+raise ValueError("custom parsers can only be specified with "
+ "ParserKind.CUSTOM")
+self.keyword = keyword
+self.kind = kind
+self.parsed_lines = []
+self.value = initial_value
+self.parser = parser
+
+if kind == ParserKind.COMMAND:
+self.parser = self._handleCommand
+elif kind == ParserKind.LIST:
+self.parser = self._handleList
+elif kind == ParserKind.TAG:
+if not keyword.endswith('.'):
+raise ValueError("keyword '%s' should end with '.'" % keyword)
+self.parser = self._handleTag
+elif kind == ParserKind.CUSTOM:
+if parser is None:
+raise ValueError("ParserKind.CUSTOM requires a custom parser")
+self.parser = parser
+else:
+raise ValueError("Unknown kind '%s'" % kind)
+
+def parseLine(self, line_number, line):
+self.parsed_lines += [(line_number, line)]
+self.value = self.parser(line_number, line, self.value)
+
+def getValue(self):
+return self.value
+
+@staticmethod
+def _handleTag(line_number, line, output):
+"""A helper for parsing TAG type keywords"""
+return (not line.strip() or output)
+
+@staticmethod
+def _handleCommand(line_number, line, output):
+"""A helper for parsing COMMAND type keywords"""
+# Trim