zturner created this revision.
zturner added reviewers: labath, stella.stamenova, davide, rnk, aprantl, friss.
Herald added a subscriber: delcypher.
Currently lit supports running shell commands through the use of the `RUN: `
prefix. This patch allows individual test suites to install their own run
handlers that can do things other than run shell commands. `RUN` commands
still work as they always do, just that now if a different kind of command
appears it will be appropriately sequenced along with the run command.
The commands the user installs can execute arbitrary Python code. As such,
they can in theory write directly to stdout or stderr, but a well-behaved
command should return its stdout and stderr from the function so that this can
be reported to the user in a manner consistent with output from `RUN` lines.
The motivating use case for this is being able to provide a richer and more
powerful syntax by which to compile test programs in LLDB tests. Currently
everything is based off of substitutions and explicitly shell commands, but
this is problematic when you get into interesting compilation scenarios.
For example, one could imagine wanting to write a test that tested the behavior
of the debugger with optimized code. Each driver has different sets of flags
that control the optimization behavior.
Another example is in cross-compilation scenarios. Certain types of PDB tests
don't need to run a process, so the tests can be run anywhere, but they need to
be linked with special flags to avoid pulling in system libraries.
We can try to make substitutions for all of these cases, but it will quickly
become unwieldy and you will end up with a command line like: `RUN: %cxx %debug
%opt %norun`, and this still isn't as flexible as you'd like.
With this patch, we could (in theory) do the compilation directly from Python.
Instead of a shell command like above, we could write something like:
COMPILE: mode=debug \
opt=none \
link=no
output=%t.o
clean=yes
source=%p/Inputs/foo.cpp
and let the function figure out how best to do this for each platform. This is
similar in spirit to how lldb's `dotest.py` already works with its platform
specific builders, but the mechanism here is general enough that it can be used
for anything a test suite wants, not just compiling.
https://reviews.llvm.org/D54731
Files:
llvm/utils/lit/lit/LitConfig.py
llvm/utils/lit/lit/ShCommands.py
llvm/utils/lit/lit/TestRunner.py
llvm/utils/lit/tests/Inputs/shtest-keyword-command/keyword-command.txt
llvm/utils/lit/tests/Inputs/shtest-keyword-command/keyword_helper.py
llvm/utils/lit/tests/Inputs/shtest-keyword-command/lit.cfg
llvm/utils/lit/tests/shtest-keyword-command.py
Index: llvm/utils/lit/tests/shtest-keyword-command.py
===================================================================
--- /dev/null
+++ llvm/utils/lit/tests/shtest-keyword-command.py
@@ -0,0 +1,22 @@
+# Check the that keyword commands work as expected.
+#
+# RUN: %{lit} -j 1 -sav %{inputs}/shtest-keyword-command > %t.out
+# RUN: FileCheck --input-file %t.out %s
+#
+# END.
+
+# CHECK: $ MYCOMMAND: at line 1 Command1
+# CHECK: command output:
+# CHECK: STDOUT: Command1
+# CHECK: command stderr:
+# CHECK: STDERR: Command1
+# CHECK: $ MYCOMMAND: at line 2 Command2
+# CHECK: command output:
+# CHECK: STDOUT: Command2
+# CHECK: command stderr:
+# CHECK: STDERR: Command2
+# CHECK: $ MYCOMMAND: at line 3 Multi-line Command
+# CHECK: command output:
+# CHECK: STDOUT: Multi-line Command
+# CHECK: command stderr:
+# CHECK: STDERR: Multi-line Command
\ No newline at end of file
Index: llvm/utils/lit/tests/Inputs/shtest-keyword-command/lit.cfg
===================================================================
--- /dev/null
+++ llvm/utils/lit/tests/Inputs/shtest-keyword-command/lit.cfg
@@ -0,0 +1,18 @@
+
+import os
+import site
+
+site.addsitedir(os.path.dirname(__file__))
+
+
+import lit.formats
+import keyword_helper
+
+config.name = 'shtest-keyword-command'
+config.suffixes = ['.txt']
+config.test_format = lit.formats.ShTest()
+config.test_source_root = None
+config.test_exec_root = None
+config.substitutions.append(('%{python}', '"%s"' % (sys.executable)))
+
+lit_config.installKeywordCommand('MYCOMMAND:', keyword_helper.customCommand)
Index: llvm/utils/lit/tests/Inputs/shtest-keyword-command/keyword_helper.py
===================================================================
--- /dev/null
+++ llvm/utils/lit/tests/Inputs/shtest-keyword-command/keyword_helper.py
@@ -0,0 +1,3 @@
+
+def customCommand(line):
+ return ('STDOUT: ' + line, 'STDERR: ' + line)
Index: llvm/utils/lit/tests/Inputs/shtest-keyword-command/keyword-command.txt
===================================================================
--- /dev/null
+++ llvm/utils/lit/tests/Inputs/shtest-keyword-command/keyword-command.txt
@@ -0,0 +1,4 @@
+MYCOMMAND: Command1
+MYCOMMAND: Command2
+MYCOMMAND: Multi-line \
+MYCOMMAND: Command
Index: llvm/utils/lit/lit/TestRunner.py
===================================================================
--- llvm/utils/lit/lit/TestRunner.py
+++ llvm/utils/lit/lit/TestRunner.py
@@ -20,6 +20,7 @@
from io import StringIO
from lit.ShCommands import GlobItem
+from lit.ShCommands import CustomCommand
import lit.ShUtil as ShUtil
import lit.Test as Test
import lit.util
@@ -1000,34 +1001,51 @@
def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
cmds = []
- for i, ln in enumerate(commands):
- ln = commands[i] = re.sub(kPdbgRegex, ": '\\1'; ", ln)
- try:
- cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
- test.config.pipefail).parse())
- except:
- return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln)
-
- cmd = cmds[0]
- for c in cmds[1:]:
- cmd = ShUtil.Seq(cmd, '&&', c)
+ for i, (keyword, ln) in enumerate(commands):
+ if keyword == 'RUN:':
+ c = None
+ ln = re.sub(kPdbgRegex, ": '\\1'; ", ln)
+ try:
+ parser = ShUtil.ShParser(ln, litConfig.isWindows, test.config.pipefail)
+ c = parser.parse()
+ except:
+ return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln)
+ if cmds and not isinstance(cmds[-1], CustomCommand):
+ cmds[-1] = ShUtil.Seq(cmds[-1], '&&', c)
+ else:
+ cmds.append(c)
+ else:
+ match = re.match('%dbg\((.* at line \\d+)\)(.*)', ln)
+ command = next(iter(filter(lambda x : x[0] == keyword, litConfig.additionalCommands)))
+ cmds.append(CustomCommand(command, match.group(1), match.group(2)))
results = []
timeoutInfo = None
- try:
- shenv = ShellEnvironment(cwd, test.config.environment)
- exitCode, timeoutInfo = executeShCmd(cmd, shenv, results, timeout=litConfig.maxIndividualTestTime)
- except InternalShellError:
- e = sys.exc_info()[1]
- exitCode = 127
- results.append(
- ShellCommandResult(e.command, '', e.message, exitCode, False))
+ for cmd in cmds:
+ try:
+ if isinstance(cmd, CustomCommand):
+ fn = cmd.command[1]
+ stdout, stderr = fn(cmd.parameter)
+ result = ShellCommandResult(command=cmd, stdout=stdout, stderr=stderr, exitCode=0, timeoutReached=False)
+ results.append(result)
+ exitCode = 0
+ else:
+ shenv = ShellEnvironment(cwd, test.config.environment)
+ exitCode, timeoutInfo = executeShCmd(cmd, shenv, results, timeout=litConfig.maxIndividualTestTime)
+ except InternalShellError:
+ e = sys.exc_info()[1]
+ exitCode = 127
+ results.append(
+ ShellCommandResult(e.command, '', e.message, exitCode, False))
out = err = ''
for i,result in enumerate(results):
# Write the command line run.
- out += '$ %s\n' % (' '.join('"%s"' % s
- for s in result.command.args),)
+ if isinstance(result.command, CustomCommand):
+ temp = result.command.location + result.command.parameter
+ else:
+ temp = ' '.join('"%s"' % s for s in result.command.args)
+ out += '$ %s\n' % (temp,)
# If nothing interesting happened, move on.
if litConfig.maxIndividualTestTime == 0 and \
@@ -1246,14 +1264,15 @@
Replace each matching occurrence of regular expression pattern a with
substitution b in line ln."""
def processLine(ln):
+ keyword, ln = ln
# Apply substitutions
for a,b in substitutions:
if kIsWindows:
b = b.replace("\\","\\\\")
ln = re.sub(a, b, ln)
# Strip the trailing newline and any extra whitespace.
- return ln.strip()
+ return (keyword, ln.strip())
# Note Python 3 map() gives an iterator rather than a list so explicitly
# convert to list before returning.
return list(map(processLine, script))
@@ -1327,9 +1346,7 @@
self.parser = parser
if kind == ParserKind.COMMAND:
- self.parser = lambda line_number, line, output: \
- self._handleCommand(line_number, line, output,
- self.keyword)
+ self.parser = self._handleCommand
elif kind == ParserKind.LIST:
self.parser = self._handleList
elif kind == ParserKind.BOOLEAN_EXPR:
@@ -1343,19 +1360,19 @@
else:
raise ValueError("Unknown kind '%s'" % kind)
- def parseLine(self, line_number, line):
+ def parseLine(self, line_number, line, keyword):
try:
self.parsed_lines += [(line_number, line)]
- self.value = self.parser(line_number, line, self.value)
+ self.value = self.parser(line_number, line, self.value, keyword)
except ValueError as e:
raise ValueError(str(e) + ("\nin %s directive on test line %d" %
(self.keyword, line_number)))
def getValue(self):
return self.value
@staticmethod
- def _handleTag(line_number, line, output):
+ def _handleTag(line_number, line, output, keyword):
"""A helper for parsing TAG type keywords"""
return (not line.strip() or output)
@@ -1374,8 +1391,8 @@
return str(line_number - int(match.group(2)))
line = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, line)
# Collapse lines with trailing '\\'.
- if output and output[-1][-1] == '\\':
- output[-1] = output[-1][:-1] + line
+ if output and output[-1][1][-1] == '\\':
+ output[-1] = (keyword, output[-1][1][:-1] + line)
else:
if output is None:
output = []
@@ -1387,19 +1404,19 @@
line = "{pdbg} {real_command}".format(
pdbg=pdbg,
real_command=line)
- output.append(line)
+ output.append((keyword, line))
return output
@staticmethod
- def _handleList(line_number, line, output):
+ def _handleList(line_number, line, output, keyword):
"""A parser for LIST type keywords"""
if output is None:
output = []
output.extend([s.strip() for s in line.split(',')])
return output
@staticmethod
- def _handleBooleanExpr(line_number, line, output):
+ def _handleBooleanExpr(line_number, line, output, keyword):
"""A parser for BOOLEAN_EXPR type keywords"""
if output is None:
output = []
@@ -1412,17 +1429,17 @@
return output
@staticmethod
- def _handleRequiresAny(line_number, line, output):
+ def _handleRequiresAny(line_number, line, output, keyword):
"""A custom parser to transform REQUIRES-ANY: into REQUIRES:"""
# Extract the conditions specified in REQUIRES-ANY: as written.
conditions = []
- IntegratedTestKeywordParser._handleList(line_number, line, conditions)
+ IntegratedTestKeywordParser._handleList(line_number, line, conditions, keyword)
# Output a `REQUIRES: a || b || c` expression in its place.
expression = ' || '.join(conditions)
IntegratedTestKeywordParser._handleBooleanExpr(line_number,
- expression, output)
+ expression, output, keyword)
return output
def parseIntegratedTestScript(test, additional_parsers=[],
@@ -1465,24 +1482,27 @@
if parser.keyword in keyword_parsers:
raise ValueError("Parser for keyword '%s' already exists"
% parser.keyword)
+ parser.value = script
keyword_parsers[parser.keyword] = parser
# Collect the test lines from the script.
sourcepath = test.getSourcePath()
for line_number, command_type, ln in \
parseIntegratedTestScriptCommands(sourcepath,
keyword_parsers.keys()):
parser = keyword_parsers[command_type]
- parser.parseLine(line_number, ln)
+ parser.parseLine(line_number, ln, parser.keyword)
+ if command_type != 'RUN:' and parser.kind == ParserKind.COMMAND:
+ has_custom_command_lines = True
if command_type == 'END.' and parser.getValue() is True:
break
# Verify the script contains a run line.
if require_script and not script:
return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!")
# Check for unterminated run lines.
- if script and script[-1][-1] == '\\':
+ if script and script[-1][-1][-1] == '\\':
return lit.Test.Result(Test.UNRESOLVED,
"Test has unterminated run lines (with '\\')")
@@ -1512,14 +1532,18 @@
return script
-
def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
# Create the output directory if it does not already exist.
lit.util.mkdir_p(os.path.dirname(tmpBase))
+ disableExternalSh = any(x[0] != 'RUN:' for x in script)
+ if useExternalSh and disableExternalSh:
+ litConfig.note("External shell disabled since custom command was encountered.")
+
execdir = os.path.dirname(test.getExecPath())
- if useExternalSh:
- res = executeScript(test, litConfig, tmpBase, script, execdir)
+ scriptCommands = [x[1] for x in script]
+ if useExternalSh and not disableExternalSh:
+ res = executeScript(test, litConfig, tmpBase, scriptCommands, execdir)
else:
res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
if isinstance(res, lit.Test.Result):
@@ -1536,7 +1560,7 @@
# Form the output log.
output = """Script:\n--\n%s\n--\nExit Code: %d\n""" % (
- '\n'.join(script), exitCode)
+ '\n'.join(scriptCommands), exitCode)
if timeoutInfo is not None:
output += """Timeout: %s\n""" % (timeoutInfo,)
@@ -1556,7 +1580,11 @@
if test.config.unsupported:
return lit.Test.Result(Test.UNSUPPORTED, 'Test is unsupported')
- script = parseIntegratedTestScript(test)
+ additional_parsers = [IntegratedTestKeywordParser(x[0], ParserKind.COMMAND) for x in litConfig.additionalCommands]
+ script = parseIntegratedTestScript(test, additional_parsers=additional_parsers)
+
+ # If there are custom command lines we can't use an external shell because
+ # it won't understand them.
if isinstance(script, lit.Test.Result):
return script
if litConfig.noExecute:
Index: llvm/utils/lit/lit/ShCommands.py
===================================================================
--- llvm/utils/lit/lit/ShCommands.py
+++ llvm/utils/lit/lit/ShCommands.py
@@ -35,6 +35,12 @@
else:
file.write("%s%s '%s'" % (r[0][1], r[0][0], r[1]))
+class CustomCommand:
+ def __init__(self, command, location, parameter):
+ self.command = command
+ self.location = location
+ self.parameter = parameter
+
class GlobItem:
def __init__(self, pattern):
self.pattern = pattern
Index: llvm/utils/lit/lit/LitConfig.py
===================================================================
--- llvm/utils/lit/lit/LitConfig.py
+++ llvm/utils/lit/lit/LitConfig.py
@@ -6,6 +6,7 @@
import lit.Test
import lit.formats
import lit.TestingConfig
+import lit.TestRunner
import lit.util
# LitConfig must be a new style class for properties to work
@@ -41,6 +42,7 @@
self.isWindows = bool(isWindows)
self.params = dict(params)
self.bashPath = None
+ self.additionalCommands = []
# Configuration files to look for when discovering test suites.
self.config_prefix = config_prefix or 'lit'
@@ -108,6 +110,9 @@
config.load_from_path(path, self)
return config
+ def installKeywordCommand(self, keyword, command):
+ self.additionalCommands.append((keyword, command))
+
def getBashPath(self):
"""getBashPath - Get the path to 'bash'"""
if self.bashPath is not None:
_______________________________________________
lldb-commits mailing list
lldb-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/lldb-commits
