varungandhi-apple updated this revision to Diff 274242.
varungandhi-apple added a comment.
- [docs] [lit] Add a more helpful description for lit.py's -s flag.
- [NFC] [lit] Separate verbose and showOutput.
- [lit] Improve lit's output with default settings and --verbose.
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D82791/new/
https://reviews.llvm.org/D82791
Files:
llvm/docs/CommandGuide/lit.rst
llvm/utils/lit/lit/LitConfig.py
llvm/utils/lit/lit/OutputSettings.py
llvm/utils/lit/lit/TestRunner.py
llvm/utils/lit/lit/cl_arguments.py
llvm/utils/lit/lit/display.py
llvm/utils/lit/lit/main.py
llvm/utils/lit/tests/shtest-format.py
llvm/utils/lit/tests/shtest-run-at-line.py
llvm/utils/lit/tests/unittest-failing-locator.py
Index: llvm/utils/lit/tests/unittest-failing-locator.py
===================================================================
--- /dev/null
+++ llvm/utils/lit/tests/unittest-failing-locator.py
@@ -0,0 +1,122 @@
+# Check that the locate_last_failing_run_line function works as expected.
+
+# RUN: %{python} %s
+# END.
+
+import unittest
+
+from lit.TestRunner import locate_last_run_line
+
+class TestRunLineLocatorHeuristics(unittest.TestCase):
+
+ def test_basic(self):
+ # from a script like
+ # RUN: echo Hello 1>&2
+ basic = (
+ "+ : 'RUN: at line 1'\n"
+ "+ echo Hello 1>&2\n"
+ "+ Hello\n"
+ )
+ line_start, substr = locate_last_run_line(basic)
+ self.assertEqual(line_start, 0)
+ self.assertEqual(substr, "RUN: at line 1")
+
+ def test_multiple(self):
+ # from a script like
+ # RUN: echo Hello 1>&2
+ # RUN: false
+ multiple = (
+ "+ : 'RUN: at line 1'\n"
+ "+ echo Hello 1>&2\n"
+ "+ Hello\n"
+ "+ : 'RUN: at line 2'\n"
+ "+ false\n"
+ )
+ line_start, substr = locate_last_run_line(multiple)
+ self.assertEqual(line_start, multiple.rfind("+ :"))
+ self.assertEqual(substr, "RUN: at line 2")
+
+ def test_varying_prefix(self):
+ # from a script like
+ # RUN: echo Hello 1>&2
+ # RUN: false
+ #
+ # in a hypothetical shell which prints line-numbers on 'set +x'
+ # (as an example of something that varies)
+ varying_prefix = (
+ "+ 1 : 'RUN: at line 1'\n"
+ "+ 2 echo Hello 1>&2\n"
+ "+ Hello\n"
+ "+ 3 : 'RUN: at line 2'\n"
+ "+ 4 false\n"
+ )
+ line_start, substr = locate_last_run_line(varying_prefix)
+ self.assertEqual(line_start, varying_prefix.rfind("+ 3"))
+ self.assertEqual(substr, "RUN: at line 2")
+
+ def test_confusing_basic(self):
+ # from a script like
+ # RUN: echo 'RUN: at line 10' 1>&2
+ confusing_basic = (
+ "+ : 'RUN: at line 1'\n"
+ "+ echo 'RUN: at line 10'\n"
+ "RUN: at line 10\n"
+ )
+ line_start, substr = locate_last_run_line(confusing_basic)
+ # FIXME: These should both be equal ideally.
+ self.assertNotEqual(line_start, 0)
+ self.assertNotEqual(substr, "RUN: at line 1")
+
+ def test_confusing_multiple(self):
+ # from a script like
+ # RUN: echo 'RUN: at line 10' 1>&2
+ # RUN: false
+ confusing_multiple = (
+ "+ : 'RUN: at line 1'\n"
+ "+ echo 'RUN: at line 10'\n"
+ "RUN: at line 10\n"
+ "+ : 'RUN: at line 2'\n"
+ "+ false\n"
+ )
+ line_start, substr = locate_last_run_line(confusing_multiple)
+ self.assertEqual(line_start, confusing_multiple.rfind("+ :"))
+ self.assertEqual(substr, "RUN: at line 2")
+
+ def test_confusing_varying_prefix_1(self):
+ # from a script like
+ # RUN: echo 'RUN: at line 10' 1>&2
+ # RUN: false
+ #
+ # in a hypothetical shell which prints line-numbers on 'set +x'
+ # (as an example of something that varies)
+ confusing_varying_prefix = (
+ "+ 1 : 'RUN: at line 1'\n"
+ "+ 2 echo 'RUN: at line 10'\n"
+ "RUN: at line 10\n"
+ "+ 3 : 'RUN: at line 2'\n"
+ "+ 4 false\n"
+ )
+ line_start, substr = locate_last_run_line(confusing_varying_prefix)
+ self.assertEqual(line_start, confusing_varying_prefix.rfind("+ 3"))
+ self.assertEqual(substr, "RUN: at line 2")
+
+ def test_confusing_varying_prefix_2(self):
+ # from a script like
+ # RUN: true
+ # RUN: not echo 'RUN: at line 100'
+ #
+ # in a hypothetical shell which prints line-numbers on 'set +x'
+ # (as an example of something that varies)
+ confusing_varying_prefix = (
+ "+ 1 : 'RUN: at line 1'\n"
+ "+ 2 true\n"
+ "+ 3 : 'RUN: at line 2'\n"
+ "+ 4 not echo 'RUN: at line 100'\n"
+ "+ RUN: at line 100\n"
+ )
+ line_start, substr = locate_last_run_line(confusing_varying_prefix)
+ # FIXME: These should both be equal ideally.
+ self.assertNotEqual(line_start, confusing_varying_prefix.rfind("+ 3"))
+ self.assertNotEqual(substr, "RUN: at line 2")
+
+unittest.main()
Index: llvm/utils/lit/tests/shtest-run-at-line.py
===================================================================
--- llvm/utils/lit/tests/shtest-run-at-line.py
+++ llvm/utils/lit/tests/shtest-run-at-line.py
@@ -1,70 +1,132 @@
-# Check that -vv makes the line number of the failing RUN command clear.
-# (-v is actually sufficient in the case of the internal shell.)
+# Check that the default terse output shows only the failing
+# line in the output, and doesn't show the script.
#
-# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
-# RUN: FileCheck --input-file %t.out %s
+# RUN: not %{lit} -j 1 %{inputs}/shtest-run-at-line > %t-terse.out
+# RUN: FileCheck --input-file %t-terse.out %s --check-prefix=TERSE
+
+# Check that --verbose outputs both Script and Command Output.
+#
+# RUN: not %{lit} -j 1 --verbose %{inputs}/shtest-run-at-line > %t-verbose.out
+# RUN: FileCheck --input-file %t-verbose.out %s --check-prefix=VERBOSE
#
# END.
+# In the case of the external shell, we check for only RUN lines in stderr in
+# case some shell implementations format "set -x" output differently.
-# CHECK: Testing: 4 tests
+################################################################################
+# Checking lines for terse output
+# TERSE-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt
-# In the case of the external shell, we check for only RUN lines in stderr in
-# case some shell implementations format "set -x" output differently.
+# TERSE-NOT: Script:
-# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt
-
-# CHECK: Script:
-# CHECK: RUN: at line 4{{.*}} true
-# CHECK-NEXT: RUN: at line 5{{.*}} false
-# CHECK-NEXT: RUN: at line 6{{.*}} true
-
-# CHECK: RUN: at line 4
-# CHECK: RUN: at line 5
-# CHECK-NOT: RUN
-
-# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt
-
-# CHECK: Script:
-# CHECK: RUN: at line 4{{.*}} echo 'foo bar' | FileCheck
-# CHECK-NEXT: RUN: at line 6{{.*}} echo 'foo baz' | FileCheck
-# CHECK-NEXT: RUN: at line 9{{.*}} echo 'foo bar' | FileCheck
-
-# CHECK: RUN: at line 4
-# CHECK: RUN: at line 6
-# CHECK-NOT: RUN
-
-
-# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt
-
-# CHECK: Script:
-# CHECK: : 'RUN: at line 1'; true
-# CHECK-NEXT: : 'RUN: at line 2'; false
-# CHECK-NEXT: : 'RUN: at line 3'; true
-
-# CHECK: Command Output (stdout)
-# CHECK: $ ":" "RUN: at line 1"
-# CHECK-NEXT: $ "true"
-# CHECK-NEXT: $ ":" "RUN: at line 2"
-# CHECK-NEXT: $ "false"
-# CHECK-NOT: RUN
-
-# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt
-
-# CHECK: Script:
-# CHECK: : 'RUN: at line 1'; : first line continued to second line
-# CHECK-NEXT: : 'RUN: at line 3'; echo 'foo bar' | FileCheck
-# CHECK-NEXT: : 'RUN: at line 5'; echo 'foo baz' | FileCheck
-# CHECK-NEXT: : 'RUN: at line 8'; echo 'foo bar' | FileCheck
-
-# CHECK: Command Output (stdout)
-# CHECK: $ ":" "RUN: at line 1"
-# CHECK-NEXT: $ ":" "first" "line" "continued" "to" "second" "line"
-# CHECK-NEXT: $ ":" "RUN: at line 3"
-# CHECK-NEXT: $ "echo" "foo bar"
-# CHECK-NEXT: $ "FileCheck" "{{.*}}"
-# CHECK-NEXT: $ ":" "RUN: at line 5"
-# CHECK-NEXT: $ "echo" "foo baz"
-# CHECK-NEXT: $ "FileCheck" "{{.*}}"
-# CHECK-NOT: RUN
+# TERSE: Command Output (stderr, truncated)
+# TERSE: --
+# TERSE-NOT: RUN: at line 4
+# TERSE-NOT: true
+# TERSE: RUN: at line 5
+# TERSE: false
+# TERSE-NOT: RUN
+# TERSE: NOTE: The failure may depend on previous RUN lines.
+
+# TERSE-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt
+
+# TERSE-NOT: Script:
+
+# TERSE: Command Output (stderr, truncated)
+# TERSE-NOT: RUN: at line 4
+# TERSE: RUN: at line 6
+# TERSE: echo 'foo baz'
+# TERSE: FileCheck
+# TERSE-NOT: RUN
+# TERSE: NOTE: The failure may depend on previous RUN lines.
+
+# TERSE-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt
+
+# TERSE-NOT: Script:
+
+# TERSE: Command Output (stdout, truncated)
+# TERSE-NOT: $ ":" "RUN: at line 1"
+# TERSE: $ ":" "RUN: at line 2"
+# TERSE-NEXT: $ "false"
+# TERSE-NOT: RUN
+# TERSE: NOTE: The failure may depend on previous RUN lines.
+
+# TERSE-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt
+
+# TERSE-NOT: Script:
+
+# TERSE: Command Output (stdout, truncated)
+# TERSE-NOT: $ ":" "RUN: at line 1"
+# TERSE-NOT: $ ":" "RUN: at line 3"
+# TERSE: $ ":" "RUN: at line 5"
+# TERSE-NEXT: $ "echo" "foo baz"
+# TERSE-NEXT: $ "FileCheck" "{{.*}}"
+# TERSE-NOT: RUN
+# TERSE: (NOTE: The failure may depend on previous RUN lines.
+# TERSE-NEXT: Use --verbose to see previous RUN lines and outputs.)
+
+################################################################################
+# Checking lines for verbose output
+
+# VERBOSE: Testing: 4 tests
+
+# VERBOSE-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt
+
+# VERBOSE: Script:
+# VERBOSE: RUN: at line 4{{.*}} true
+# VERBOSE-NEXT: RUN: at line 5{{.*}} false
+# VERBOSE-NEXT: RUN: at line 6{{.*}} true
+
+# VERBOSE: RUN: at line 4
+# VERBOSE: RUN: at line 5
+# VERBOSE-NOT: RUN: at line 6
+# VERBOSE-NOT: (NOTE: The failure may depend on previous RUN lines.
+
+# VERBOSE-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt
+
+# VERBOSE: Script:
+# VERBOSE: RUN: at line 4{{.*}} echo 'foo bar' | FileCheck
+# VERBOSE-NEXT: RUN: at line 6{{.*}} echo 'foo baz' | FileCheck
+# VERBOSE-NEXT: RUN: at line 9{{.*}} echo 'foo bar' | FileCheck
+
+# VERBOSE: RUN: at line 4
+# VERBOSE: RUN: at line 6
+# VERBOSE-NOT: RUN
+# VERBOSE-NOT: (NOTE: The failure may depend on previous RUN lines.
+
+# VERBOSE-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt
+
+# VERBOSE: Script:
+# VERBOSE: : 'RUN: at line 1'; true
+# VERBOSE-NEXT: : 'RUN: at line 2'; false
+# VERBOSE-NEXT: : 'RUN: at line 3'; true
+
+# VERBOSE: Command Output (stdout)
+# VERBOSE: $ ":" "RUN: at line 1"
+# VERBOSE-NEXT: $ "true"
+# VERBOSE-NEXT: $ ":" "RUN: at line 2"
+# VERBOSE-NEXT: $ "false"
+# VERBOSE-NOT: RUN
+# VERBOSE-NOT: (NOTE: The failure may depend on previous RUN lines.
+
+# VERBOSE-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt
+
+# VERBOSE: Script:
+# VERBOSE: : 'RUN: at line 1'; : first line continued to second line
+# VERBOSE-NEXT: : 'RUN: at line 3'; echo 'foo bar' | FileCheck
+# VERBOSE-NEXT: : 'RUN: at line 5'; echo 'foo baz' | FileCheck
+# VERBOSE-NEXT: : 'RUN: at line 8'; echo 'foo bar' | FileCheck
+
+# VERBOSE: Command Output (stdout)
+# VERBOSE: $ ":" "RUN: at line 1"
+# VERBOSE-NEXT: $ ":" "first" "line" "continued" "to" "second" "line"
+# VERBOSE-NEXT: $ ":" "RUN: at line 3"
+# VERBOSE-NEXT: $ "echo" "foo bar"
+# VERBOSE-NEXT: $ "FileCheck" "{{.*}}"
+# VERBOSE-NEXT: $ ":" "RUN: at line 5"
+# VERBOSE-NEXT: $ "echo" "foo baz"
+# VERBOSE-NEXT: $ "FileCheck" "{{.*}}"
+# VERBOSE-NOT: RUN
+# VERBOSE-NOT: (NOTE: The failure may depend on previous RUN lines.
Index: llvm/utils/lit/tests/shtest-format.py
===================================================================
--- llvm/utils/lit/tests/shtest-format.py
+++ llvm/utils/lit/tests/shtest-format.py
@@ -17,7 +17,12 @@
# CHECK-NEXT: line 2: failed test output on stdout
# CHECK: Command Output (stderr):
# CHECK-NEXT: --
-# CHECK-NEXT: cat{{(\.exe)?}}: {{cannot open does-not-exist|does-not-exist: No such file or directory}}
+# CHECK-NEXT: 'RUN: at line 3'
+#
+# Skipping some lines...
+#
+# CHECK: 'RUN: at line 5'
+# CHECK: cat{{(\.exe)?}}: {{cannot open does-not-exist|does-not-exist: No such file or directory}}
# CHECK: --
# CHECK: FAIL: shtest-format :: external_shell/fail_with_bad_encoding.txt
Index: llvm/utils/lit/lit/main.py
===================================================================
--- llvm/utils/lit/lit/main.py
+++ llvm/utils/lit/lit/main.py
@@ -37,7 +37,9 @@
isWindows=is_windows,
params=params,
config_prefix=opts.configPrefix,
- echo_all_commands=opts.echoAllCommands)
+ echo_all_commands=opts.echo_all_commands,
+ script_output_style=opts.script_output_style,
+ command_output_style=opts.command_output_style)
discovered_tests = lit.discovery.find_tests_for_inputs(lit_config, opts.test_paths)
if not discovered_tests:
Index: llvm/utils/lit/lit/display.py
===================================================================
--- llvm/utils/lit/lit/display.py
+++ llvm/utils/lit/lit/display.py
@@ -69,7 +69,7 @@
self.completed, self.tests))
# Show the test failure output, if requested.
- if (test.isFailure() and self.opts.showOutput) or \
+ if (test.isFailure() and self.opts.show_output_on_failure) or \
self.opts.showAllOutput:
if test.isFailure():
print("%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(),
Index: llvm/utils/lit/lit/cl_arguments.py
===================================================================
--- llvm/utils/lit/lit/cl_arguments.py
+++ llvm/utils/lit/lit/cl_arguments.py
@@ -3,6 +3,7 @@
import shlex
import sys
+import lit.OutputSettings as OutputSettings
import lit.reports
import lit.util
@@ -42,17 +43,21 @@
help="Suppress no error output",
action="store_true")
format_group.add_argument("-s", "--succinct",
- help="Reduce amount of output",
+ help="Reduce amount of output."
+ " Additionally, show a progress bar,"
+ " unless --no-progress-bar is specified.",
action="store_true")
format_group.add_argument("-v", "--verbose",
- dest="showOutput",
- help="Show test output for failures",
+ help="Show test output for failures."
+ " The last command under 'Output' is the failing one."
+ " All substituted commands are shown under 'Script',"
+ " including any commands after the failing command.",
action="store_true")
+ # TODO(python3): Use aliases kwarg for add_argument above.
format_group.add_argument("-vv", "--echo-all-commands",
- dest="echoAllCommands",
- action="store_true",
- help="Echo all commands as they are executed to stdout. In case of "
- "failure, last command shown will be the failing one.")
+ help="Alias for --verbose.",
+ dest="verbose",
+ action="store_true")
format_group.add_argument("-a", "--show-all",
dest="showAllOutput",
help="Display all commandlines and output",
@@ -166,9 +171,28 @@
args = sys.argv[1:] + env_args
opts = parser.parse_args(args)
- # Validate command line options
- if opts.echoAllCommands:
- opts.showOutput = True
+ # default output settings
+ opts.show_output_on_failure = True
+ opts.script_output_style = OutputSettings.NO_SCRIPT
+ opts.command_output_style = OutputSettings.ONLY_FAILING_COMMAND
+ opts.echo_all_commands = True
+
+ if opts.quiet:
+ opts.show_output_on_failure = False
+ opts.command_output_style = OutputSettings.NO_COMMAND
+ elif opts.verbose or opts.showAllOutput:
+ opts.script_output_style = OutputSettings.FULL_SCRIPT
+ opts.command_output_style = (
+ OutputSettings.UP_TO_AND_INCLUDING_FAILING_COMMAND
+ )
+
+ # If we call out to an external shell, we ask for all output,
+ # regardless of whether we're going to display all commands that were run,
+ # or only the last command which failed.
+ cmd_output_style = opts.command_output_style
+ opts.echo_all_commands = (
+ cmd_output_style == OutputSettings.UP_TO_AND_INCLUDING_FAILING_COMMAND
+ or cmd_output_style == OutputSettings.ONLY_FAILING_COMMAND)
# TODO(python3): Could be enum
if opts.shuffle:
Index: llvm/utils/lit/lit/TestRunner.py
===================================================================
--- llvm/utils/lit/lit/TestRunner.py
+++ llvm/utils/lit/lit/TestRunner.py
@@ -17,6 +17,7 @@
except ImportError:
from io import StringIO
+import lit.OutputSettings as OutputSettings
from lit.ShCommands import GlobItem, Command
import lit.ShUtil as ShUtil
import lit.Test as Test
@@ -1495,6 +1496,132 @@
return script
+def locate_last_run_line(output_str):
+ """
+ Try to locate the last run line in ``output_str`` using heuristics.
+
+ Returns a pair of:
+ - The index in ``output_str`` pointing to immediately after the preceding
+ newline, i.e. the start of the RUN line, before any shell-specific
+ prefix.
+ - The matched substring itself, including the number at the end,
+ starting with 'RUN', skipping the shell-specific prefix.
+
+ For example, if ``output_str`` is ``"+ HI\n+ RUN: at line 10"``,
+ the index will be 5 and the substring will be ``"RUN: at line 10"``.
+
+ Returns (-1, None) on failure.
+ """
+ # We try to be clever; rather than directly finding the last
+ # 'RUN: at line', we try to search from the beginning, create a regex,
+ # which hopefully takes into account any shell-specific prefix,
+ # and then uses that regex to search from the end.
+ # Searching backwards directly can lead to erroneous reporting
+ # when the command output itself has a 'RUN: at line' due to something
+ # else (possibly because we are testing ``lit`` itself).
+ #
+ # However, this is not perfect either :(. It will fail if a shell emits
+ # varying prefixes for commands on 'set +x', such as line numbers or
+ # timestamps. So we try to fallback to the simpler technique if applicable.
+
+ error = (-1, None)
+ run_str = "RUN: at line"
+ first_run_str_start = output_str.find(run_str)
+ if first_run_str_start == -1:
+ return error
+ # Use the first RUN line to create a regex for searching.
+ newline_index = output_str[:first_run_str_start].rfind('\n')
+ first_run_line_start = 0 if newline_index == -1 else newline_index + 1
+ first_run_str_end = first_run_str_start + len(run_str)
+ run_line_regex = (
+ '('
+ + re.escape(output_str[first_run_line_start:first_run_str_end])
+ + ' [0-9]+)'
+ )
+
+ run_lines = list(re.finditer(run_line_regex, output_str))
+ if len(run_lines) == 0:
+ # impossible; we should've at least gotten the first run line...
+ return error
+
+ if len(run_lines) == 1:
+ # We must've gotten our original string. That's not helpful. :-/
+ # Try using the simple strategy to make things work with shells that
+ # use a varying prefix for 'set +x'.
+ # (N.B. I don't know of any shells that actually do this, but it's
+ # not inconceivable that a CI machine would, say, print out timestamps
+ # with 'set +x')
+ simple_run_line_regex = '(RUN: at line [0-9]+)'
+ simple_run_lines = list(re.finditer(simple_run_line_regex, output_str))
+ if len(simple_run_lines) == 0:
+ # impossible; we should've at least gotten the first run line...
+ return error
+ elif len(simple_run_lines) == 1:
+ # we must've hit the line we found earlier, don't try anything
+ pass
+ else:
+ # Assume that we have a shell with a varying prefix and use
+ # the results of the simple matching.
+ # However, this creates a problem when:
+ # 1. The very first RUN line failed and it has 'RUN: at line'.
+ # 2. A shell has varying prefixes and the output of the last command
+ # has 'RUN: at line'.
+ # Can't really help these situations. :(
+ last_run_line = simple_run_lines[-1]
+ newline_index = output_str[:last_run_line.start()].rfind('\n')
+ start = 0 if newline_index == -1 else (newline_index + 1)
+ return (start, last_run_line.group(0))
+
+ last_run_line = run_lines[-1]
+ # trim the shell-specific prefix
+ delta = first_run_str_start - first_run_line_start
+ last_run_line_str = last_run_line.group(0)[delta:]
+ return (last_run_line.start(), last_run_line_str)
+
+def make_script_output(lit_config, script_lines, exit_code):
+ def make_output(script_display_lines):
+ return ("""Script:\n--\n%s\n--\nExit Code: %d\n"""
+ % ('\n'.join(script_display_lines), exit_code))
+
+ def default_output():
+ return make_output(script_lines)
+
+ if lit_config.script_output_style == OutputSettings.NO_SCRIPT:
+ return ""
+
+ assert(lit_config.script_output_style == OutputSettings.FULL_SCRIPT)
+ return default_output()
+
+def make_command_output(lit_config, cmd_output, stream_name, test_status):
+ def make_output(output_str, is_truncated=False):
+ format_str = """Command Output (%s, truncated):\n--\n%s\n--\n"""
+ format_str += (
+ "(NOTE: The failure may depend on previous RUN lines.\n"
+ " Use --verbose to see previous RUN lines and outputs.)\n"
+ )
+ if not is_truncated:
+ format_str = """Command Output (%s):\n--\n%s\n--\n"""
+ return format_str % (stream_name, output_str)
+
+ def default_output():
+ return make_output(cmd_output)
+
+ if lit_config.command_output_style == OutputSettings.NO_COMMAND:
+ return ""
+ elif not test_status.isFailure:
+ return default_output()
+
+ line_start, run_line_str = locate_last_run_line(cmd_output)
+ # maybe there was an error, or maybe we are not truncating anything
+ if run_line_str is None or line_start == 0:
+ return default_output()
+
+ if lit_config.command_output_style == OutputSettings.ONLY_FAILING_COMMAND:
+ return make_output(cmd_output[line_start:], is_truncated=True)
+
+ assert(lit_config.command_output_style
+ == OutputSettings.UP_TO_AND_INCLUDING_FAILING_COMMAND)
+ return make_output(cmd_output, is_truncated=False)
def _runShTest(test, litConfig, useExternalSh, script, tmpBase):
def runOnce(execdir):
@@ -1536,8 +1663,7 @@
status = Test.FLAKYPASS
# Form the output log.
- output = """Script:\n--\n%s\n--\nExit Code: %d\n""" % (
- '\n'.join(script), exitCode)
+ output = make_script_output(litConfig, script, exitCode)
if timeoutInfo is not None:
output += """Timeout: %s\n""" % (timeoutInfo,)
@@ -1545,9 +1671,9 @@
# Append the outputs, if present.
if out:
- output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
+ output += make_command_output(litConfig, out, "stdout", status)
if err:
- output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
+ output += make_command_output(litConfig, err, "stderr", status)
return lit.Test.Result(status, output)
Index: llvm/utils/lit/lit/OutputSettings.py
===================================================================
--- /dev/null
+++ llvm/utils/lit/lit/OutputSettings.py
@@ -0,0 +1,66 @@
+# TODO(python3): Could be an enum
+class ScriptOutputStyle:
+ def __init__(self, name):
+ self.name = name
+
+ # For print-debugging.
+ def __str__(self):
+ return '%s(%r)' % (self.__class__.__name__, self.name)
+
+NO_SCRIPT = ScriptOutputStyle("None")
+FULL_SCRIPT = ScriptOutputStyle("Full")
+
+# Consider a script
+#
+# ```
+# RUN: true
+# RUN: false
+# RUN: true
+# ```
+#
+# FULL_SCRIPT will show the whole script.
+
+# TODO(python3): Could be an enum
+class CommandOutputStyle:
+ def __init__(self, name):
+ self.name = name
+
+ # For print-debugging.
+ def __str__(self):
+ return '%s(%r)' % (self.__class__.__name__, self.name)
+
+NO_COMMAND = CommandOutputStyle("None")
+ONLY_FAILING_COMMAND = CommandOutputStyle("OnlyFailing")
+UP_TO_AND_INCLUDING_FAILING_COMMAND = CommandOutputStyle("UpToAndIncluding")
+
+# Consider a script
+#
+# ```
+# RUN: true
+# RUN: false
+# RUN: true
+# ```
+#
+# ONLY_FAILING_COMMAND will show something like
+#
+# ```
+# Command output (stderr, truncated)
+# --
+# + : 'RUN: at line 2'
+# + false
+# --
+# NOTE: The failure might depend on preceding RUN lines.
+# Use --verbose to see preceding RUN lines and outputs.
+# ```
+#
+# UP_TO_AND_INCLUDING_FAILING_COMMAND will show something like
+#
+# ```
+# Command Output (stderr):
+# --
+# + : 'RUN: at line 1'
+# + true
+# + : 'RUN: at line 2'
+# + false
+# --
+# ```
Index: llvm/utils/lit/lit/LitConfig.py
===================================================================
--- llvm/utils/lit/lit/LitConfig.py
+++ llvm/utils/lit/lit/LitConfig.py
@@ -6,6 +6,7 @@
import lit.Test
import lit.formats
+import lit.OutputSettings as OutputSettings
import lit.TestingConfig
import lit.util
@@ -26,7 +27,9 @@
params, config_prefix = None,
maxIndividualTestTime = 0,
parallelism_groups = {},
- echo_all_commands = False):
+ echo_all_commands=False,
+ script_output_style=OutputSettings.NO_SCRIPT,
+ command_output_style=OutputSettings.ONLY_FAILING_COMMAND):
# The name of the test runner.
self.progname = progname
# The items to add to the PATH environment variable.
@@ -66,6 +69,8 @@
self.maxIndividualTestTime = maxIndividualTestTime
self.parallelism_groups = parallelism_groups
self.echo_all_commands = echo_all_commands
+ self.script_output_style = script_output_style
+ self.command_output_style = command_output_style
@property
def maxIndividualTestTime(self):
Index: llvm/docs/CommandGuide/lit.rst
===================================================================
--- llvm/docs/CommandGuide/lit.rst
+++ llvm/docs/CommandGuide/lit.rst
@@ -27,8 +27,9 @@
fail.
By default :program:`lit` will use a succinct progress display and will only
-print summary information for test failures. See :ref:`output-options` for
-options controlling the :program:`lit` progress display and output.
+print summary information for test failures, such as which exact line failed.
+See :ref:`output-options` for options controlling the :program:`lit`
+progress display and output.
:program:`lit` also includes a number of options for controlling how tests are
executed (specific features may depend on the particular test format). See
@@ -81,14 +82,13 @@
.. option:: -s, --succinct
Show less output, for example don't show information on tests that pass.
+ Also shows a progress bar, unless ``--no-progress-bar`` is specificed.
.. option:: -v, --verbose
Show more information on test failures, for example the entire test output
instead of just the test result.
-.. option:: -vv, --echo-all-commands
-
Echo all commands to stdout, as they are being executed.
This can be valuable for debugging test failures, as the last echoed command
will be the one which has failed.
@@ -96,8 +96,10 @@
with argument ``'RUN: at line N'`` before each command pipeline, and this
option also causes those no-op commands to be echoed to stdout to help you
locate the source line of the failed command.
- This option implies ``--verbose``.
+.. option:: -vv, --echo-all-commands
+
+ Alias for ``-v``/``--verbose`` (for backwards compatibility).
.. option:: -a, --show-all
Show more information about all tests, for example the entire test
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
