[lit] Report line number for failed RUN command
(Relands r333584, reverted in 333592.)
When debugging test failures with -vv (or -v in the case of the
internal shell), this makes it easier to locate the RUN line that
failed. For example, clang's test/Driver/linux-ld.c has 892 total RUN
lines, and clang's test/Driver/arm-cortex-cpus.c has 424 RUN lines
after concatenation for line continuations.
When reading the generated shell script, this also makes it easier to
locate the RUN line that produced each command.
To support reporting RUN line numbers in the case of the internal
shell, this patch extends the internal shell to support the null
command, ":", except pipelines are not supported.
To support reporting RUN line numbers in the case of windows cmd.exe
as the external shell, this patch extends -vv to set "echo on" instead
of "echo off" in bat files. (Support for windows cmd.exe as a lit
external shell will likely be dropped later, but I found out too
late.)
Reviewed By: delcypher, asmith, stella.stamenova, jmorse, lebedev.ri, rnk
Differential Revision: https://reviews.llvm.org/D44598
llvm-svn: 333614
diff --git a/llvm/utils/lit/lit/TestRunner.py b/llvm/utils/lit/lit/TestRunner.py
index a6f9276..35609d98 100644
--- a/llvm/utils/lit/lit/TestRunner.py
+++ b/llvm/utils/lit/lit/TestRunner.py
@@ -40,6 +40,17 @@
kAvoidDevNull = kIsWindows
kDevNull = "/dev/null"
+# A regex that matches %dbg(ARG), which lit inserts at the beginning of each
+# run command pipeline such that ARG specifies the pipeline's source line
+# number. lit later expands each %dbg(ARG) to a command that behaves as a null
+# command in the target shell so that the line number is seen in lit's verbose
+# mode.
+#
+# This regex captures ARG. ARG must not contain a right parenthesis, which
+# terminates %dbg. ARG must not contain quotes, in which ARG might be enclosed
+# during expansion.
+kPdbgRegex = '%dbg\(([^)\'"]*)\)'
+
class ShellEnvironment(object):
"""Mutable shell environment containing things like CWD and env vars.
@@ -789,6 +800,13 @@
results.append(cmdResult)
return cmdResult.exitCode
+ if cmd.commands[0].args[0] == ':':
+ if len(cmd.commands) != 1:
+ raise InternalShellError(cmd.commands[0], "Unsupported: ':' "
+ "cannot be part of a pipeline")
+ results.append(ShellCommandResult(cmd.commands[0], '', '', 0, False))
+ return 0;
+
procs = []
default_stdin = subprocess.PIPE
stderrTempFiles = []
@@ -982,7 +1000,8 @@
def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
cmds = []
- for ln in commands:
+ for i, ln in enumerate(commands):
+ ln = commands[i] = re.sub(kPdbgRegex, ": '\\1'", ln)
try:
cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
test.config.pipefail).parse())
@@ -1066,9 +1085,16 @@
mode += 'b' # Avoid CRLFs when writing bash scripts.
f = open(script, mode)
if isWin32CMDEXE:
- f.write('@echo off\n')
- f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
+ for i, ln in enumerate(commands):
+ commands[i] = re.sub(kPdbgRegex, "echo '\\1' > nul", ln)
+ if litConfig.echo_all_commands:
+ f.write('@echo on\n')
+ else:
+ f.write('@echo off\n')
+ f.write('\n@if %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
else:
+ for i, ln in enumerate(commands):
+ commands[i] = re.sub(kPdbgRegex, ": '\\1'", ln)
if test.config.pipefail:
f.write('set -o pipefail;')
if litConfig.echo_all_commands:
@@ -1301,7 +1327,9 @@
self.parser = parser
if kind == ParserKind.COMMAND:
- self.parser = self._handleCommand
+ self.parser = lambda line_number, line, output: \
+ self._handleCommand(line_number, line, output,
+ self.keyword)
elif kind == ParserKind.LIST:
self.parser = self._handleList
elif kind == ParserKind.BOOLEAN_EXPR:
@@ -1332,7 +1360,7 @@
return (not line.strip() or output)
@staticmethod
- def _handleCommand(line_number, line, output):
+ def _handleCommand(line_number, line, output, keyword):
"""A helper for parsing COMMAND type keywords"""
# Trim trailing whitespace.
line = line.rstrip()
@@ -1351,6 +1379,14 @@
else:
if output is None:
output = []
+ pdbg = "%dbg({keyword} at line {line_number})".format(
+ keyword=keyword,
+ line_number=line_number)
+ assert re.match(kPdbgRegex + "$", pdbg), \
+ "kPdbgRegex expected to match actual %dbg usage"
+ line = "{pdbg} && {real_command}".format(
+ pdbg=pdbg,
+ real_command=line)
output.append(line)
return output
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt
new file mode 100644
index 0000000..a359c99
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/basic.txt
@@ -0,0 +1,6 @@
+# These commands must run under both bash and windows cmd.exe (with GnuWin32
+# tools).
+
+# RUN: true
+# RUN: false
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt
new file mode 100644
index 0000000..b11f0d7
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/line-continuation.txt
@@ -0,0 +1,12 @@
+# These commands must run under both bash and windows cmd.exe (with GnuWin32
+# tools).
+
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+# RUN: echo \
+# RUN: 'foo baz' \
+# RUN: | FileCheck %s
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+
+# CHECK: foo bar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg
new file mode 100644
index 0000000..5e87c72
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/external-shell/lit.local.cfg
@@ -0,0 +1,2 @@
+import lit.formats
+config.test_format = lit.formats.ShTest(execute_external=True)
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt
new file mode 100644
index 0000000..ba26954
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/basic.txt
@@ -0,0 +1,3 @@
+# RUN: true
+# RUN: false
+# RUN: true
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt
new file mode 100644
index 0000000..1e00bcb
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/line-continuation.txt
@@ -0,0 +1,11 @@
+# RUN: : first line continued \
+# RUN: to second line
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+# RUN: echo \
+# RUN: 'foo baz' \
+# RUN: | FileCheck %s
+# RUN: echo 'foo bar' \
+# RUN: | FileCheck %s
+
+# CHECK: foo bar
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg
new file mode 100644
index 0000000..b76b7a2
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/internal-shell/lit.local.cfg
@@ -0,0 +1,2 @@
+import lit.formats
+config.test_format = lit.formats.ShTest(execute_external=False)
diff --git a/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg
new file mode 100644
index 0000000..f4c7921
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-run-at-line/lit.cfg
@@ -0,0 +1,2 @@
+config.name = 'shtest-run-at-line'
+config.suffixes = ['.txt']
diff --git a/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt b/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt
new file mode 100644
index 0000000..8b84c08
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/shtest-shell/colon-error.txt
@@ -0,0 +1,3 @@
+# Check error on an unsupported ":". (cannot be part of a pipeline)
+#
+# RUN: : | echo "hello"
diff --git a/llvm/utils/lit/tests/lit.cfg b/llvm/utils/lit/tests/lit.cfg
index 75d1b5e..a5ccc2c 100644
--- a/llvm/utils/lit/tests/lit.cfg
+++ b/llvm/utils/lit/tests/lit.cfg
@@ -71,3 +71,14 @@
if directory:
path = os.path.pathsep.join((directory, path))
config.environment['PATH'] = path
+
+# These substitutions are needed only in tests where the external shell is used
+# and could be either bash or windows cmd.exe. Substitutions are expected to
+# be expanded in double quotes.
+isWin32CMDEXE = lit_config.isWindows and not lit_config.getBashPath()
+if isWin32CMDEXE:
+ config.substitutions.append(('%{pdbg0}', "echo '"))
+ config.substitutions.append(('%{pdbg1}', "' > nul"))
+else:
+ config.substitutions.append(('%{pdbg0}', ": '"))
+ config.substitutions.append(('%{pdbg1}', "'"))
diff --git a/llvm/utils/lit/tests/max-failures.py b/llvm/utils/lit/tests/max-failures.py
index 1613eee..ca107bc 100644
--- a/llvm/utils/lit/tests/max-failures.py
+++ b/llvm/utils/lit/tests/max-failures.py
@@ -8,7 +8,7 @@
#
# END.
-# CHECK: Failing Tests (26)
+# CHECK: Failing Tests (27)
# CHECK: Failing Tests (1)
# CHECK: Failing Tests (2)
# CHECK: error: Setting --max-failures to 0 does not have any effect.
diff --git a/llvm/utils/lit/tests/shtest-format.py b/llvm/utils/lit/tests/shtest-format.py
index 94d74e3..33ed2fe 100644
--- a/llvm/utils/lit/tests/shtest-format.py
+++ b/llvm/utils/lit/tests/shtest-format.py
@@ -39,6 +39,7 @@
#
# CHECK: Command Output (stdout):
# CHECK-NEXT: --
+# CHECK-NEXT: $ ":" "RUN: at line 1"
# CHECK-NEXT: $ "printf"
# CHECK-NEXT: # command output:
# CHECK-NEXT: line 1: failed test output on stdout
diff --git a/llvm/utils/lit/tests/shtest-output-printing.py b/llvm/utils/lit/tests/shtest-output-printing.py
index 2a85cf9..2344ef2 100644
--- a/llvm/utils/lit/tests/shtest-output-printing.py
+++ b/llvm/utils/lit/tests/shtest-output-printing.py
@@ -16,12 +16,15 @@
#
# CHECK: Command Output
# CHECK-NEXT: --
+# CHECK-NEXT: $ ":" "RUN: at line 1"
# CHECK-NEXT: $ "true"
+# CHECK-NEXT: $ ":" "RUN: at line 2"
# CHECK-NEXT: $ "echo" "hi"
# CHECK-NEXT: # command output:
# CHECK-NEXT: hi
#
-# CHECK: $ "wc" "missing-file"
+# CHECK: $ ":" "RUN: at line 3"
+# CHECK-NEXT: $ "wc" "missing-file"
# CHECK-NEXT: # redirected output from '{{.*(/|\\\\)}}basic.txt.tmp.out':
# CHECK-NEXT: missing-file{{.*}} No such file or directory
# CHECK: note: command had no output on stdout or stderr
diff --git a/llvm/utils/lit/tests/shtest-run-at-line.py b/llvm/utils/lit/tests/shtest-run-at-line.py
new file mode 100644
index 0000000..adb7af8
--- /dev/null
+++ b/llvm/utils/lit/tests/shtest-run-at-line.py
@@ -0,0 +1,70 @@
+# Check that -vv makes the line number of the failing RUN command clear.
+# (-v is actually sufficient in the case of the internal shell.)
+#
+# RUN: not %{lit} -j 1 -vv %{inputs}/shtest-run-at-line > %t.out
+# RUN: FileCheck --input-file %t.out -Dpdbg0="%{pdbg0}" -Dpdbg1="%{pdbg1}" %s
+#
+# END.
+
+
+# CHECK: Testing: 4 tests
+
+
+# In the case of the external shell, we check for only RUN lines in stderr in
+# case some shell implementations format "set -x" output differently.
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/basic.txt
+
+# CHECK: Script:
+# CHECK: [[pdbg0]]RUN: at line 4[[pdbg1]] && true
+# CHECK-NEXT: [[pdbg0]]RUN: at line 5[[pdbg1]] && false
+# CHECK-NEXT: [[pdbg0]]RUN: at line 6[[pdbg1]] && true
+
+# CHECK: RUN: at line 4
+# CHECK: RUN: at line 5
+# CHECK-NOT: RUN
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: external-shell/line-continuation.txt
+
+# CHECK: Script:
+# CHECK: [[pdbg0]]RUN: at line 4[[pdbg1]] && echo 'foo bar' | FileCheck
+# CHECK-NEXT: [[pdbg0]]RUN: at line 6[[pdbg1]] && echo 'foo baz' | FileCheck
+# CHECK-NEXT: [[pdbg0]]RUN: at line 9[[pdbg1]] && echo 'foo bar' | FileCheck
+
+# CHECK: RUN: at line 4
+# CHECK: RUN: at line 6
+# CHECK-NOT: RUN
+
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/basic.txt
+
+# CHECK: Script:
+# CHECK: : 'RUN: at line 1' && true
+# CHECK-NEXT: : 'RUN: at line 2' && false
+# CHECK-NEXT: : 'RUN: at line 3' && true
+
+# CHECK: Command Output (stdout)
+# CHECK: $ ":" "RUN: at line 1"
+# CHECK-NEXT: $ "true"
+# CHECK-NEXT: $ ":" "RUN: at line 2"
+# CHECK-NEXT: $ "false"
+# CHECK-NOT: RUN
+
+# CHECK-LABEL: FAIL: shtest-run-at-line :: internal-shell/line-continuation.txt
+
+# CHECK: Script:
+# CHECK: : 'RUN: at line 1' && : first line continued to second line
+# CHECK-NEXT: : 'RUN: at line 3' && echo 'foo bar' | FileCheck
+# CHECK-NEXT: : 'RUN: at line 5' && echo 'foo baz' | FileCheck
+# CHECK-NEXT: : 'RUN: at line 8' && echo 'foo bar' | FileCheck
+
+# CHECK: Command Output (stdout)
+# CHECK: $ ":" "RUN: at line 1"
+# CHECK-NEXT: $ ":" "first" "line" "continued" "to" "second" "line"
+# CHECK-NEXT: $ ":" "RUN: at line 3"
+# CHECK-NEXT: $ "echo" "foo bar"
+# CHECK-NEXT: $ "FileCheck" "{{.*}}"
+# CHECK-NEXT: $ ":" "RUN: at line 5"
+# CHECK-NEXT: $ "echo" "foo baz"
+# CHECK-NEXT: $ "FileCheck" "{{.*}}"
+# CHECK-NOT: RUN
diff --git a/llvm/utils/lit/tests/shtest-shell.py b/llvm/utils/lit/tests/shtest-shell.py
index b3d55a0..016d1f6 100644
--- a/llvm/utils/lit/tests/shtest-shell.py
+++ b/llvm/utils/lit/tests/shtest-shell.py
@@ -26,6 +26,14 @@
# CHECK: error: command failed with exit status: 1
# CHECK: ***
+# CHECK: FAIL: shtest-shell :: colon-error.txt
+# CHECK: *** TEST 'shtest-shell :: colon-error.txt' FAILED ***
+# CHECK: $ ":"
+# CHECK: # command stderr:
+# CHECK: Unsupported: ':' cannot be part of a pipeline
+# CHECK: error: command failed with exit status: 127
+# CHECK: ***
+
# CHECK: FAIL: shtest-shell :: diff-error-0.txt
# CHECK: *** TEST 'shtest-shell :: diff-error-0.txt' FAILED ***
# CHECK: $ "diff" "diff-error-0.txt" "diff-error-0.txt"
@@ -153,7 +161,7 @@
#
# CHECK: FAIL: shtest-shell :: error-1.txt
# CHECK: *** TEST 'shtest-shell :: error-1.txt' FAILED ***
-# CHECK: shell parser error on: 'echo "missing quote'
+# CHECK: shell parser error on: ': \'RUN: at line 3\' && echo "missing quote'
# CHECK: ***
# CHECK: FAIL: shtest-shell :: error-2.txt
@@ -219,4 +227,4 @@
# CHECK: PASS: shtest-shell :: sequencing-0.txt
# CHECK: XFAIL: shtest-shell :: sequencing-1.txt
# CHECK: PASS: shtest-shell :: valid-shell.txt
-# CHECK: Failing Tests (26)
+# CHECK: Failing Tests (27)
diff --git a/llvm/utils/lit/tests/unit/TestRunner.py b/llvm/utils/lit/tests/unit/TestRunner.py
index 874bf27..4ade359 100644
--- a/llvm/utils/lit/tests/unit/TestRunner.py
+++ b/llvm/utils/lit/tests/unit/TestRunner.py
@@ -99,8 +99,8 @@
cmd_parser = self.get_parser(parsers, 'MY_RUN:')
value = cmd_parser.getValue()
self.assertEqual(len(value), 2) # there are only two run lines
- self.assertEqual(value[0].strip(), 'baz')
- self.assertEqual(value[1].strip(), 'foo bar')
+ self.assertEqual(value[0].strip(), "%dbg(MY_RUN: at line 4) && baz")
+ self.assertEqual(value[1].strip(), "%dbg(MY_RUN: at line 7) && foo bar")
def test_custom(self):
parsers = self.make_parsers()