pw_build: Update path and target resolution
- Don't implicitly convert paths and labels in python_runner.py.
- To enable finding target outputs, support explicit resolution with
the <TARGET_FILE($target)> expression.
- Use the standard GN rebase_path approach to resolve GN paths to
filesystem paths for scripts. Remove workarounds that are no longer
necessary.
- Update build files to use rebase_path and <TARGET_FILE(target)>.
- Add tests for python_runner.py.
Bug: 110
Change-Id: Iae262820bb265c648c270c2b78d058f20e1d3d1f
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/14801
Reviewed-by: Alexei Frolov <frolv@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Commit-Queue: Wyatt Hepler <hepler@google.com>
diff --git a/pw_bloat/bloat.gni b/pw_bloat/bloat.gni
index bb6a653..14e64a4 100644
--- a/pw_bloat/bloat.gni
+++ b/pw_bloat/bloat.gni
@@ -99,9 +99,7 @@
"Size report binaries must define 'label' and 'target' variables")
_all_target_dependencies += [ binary.target ]
- _target_dir = get_label_info(binary.target, "target_out_dir")
- _target_name = get_label_info(binary.target, "name")
- _binary_path = get_path_info(_target_dir, "abspath") + ":$_target_name"
+ _binary_path = "<TARGET_FILE(${binary.target})>"
# If the binary defines its own base, use that instead of the global base.
if (defined(binary.base)) {
@@ -120,9 +118,7 @@
_bloaty_configs += [ get_path_info(pw_bloat_BLOATY_CONFIG, "abspath") ]
}
- _base_dir = get_label_info(_binary_base, "target_out_dir")
- _base_name = get_label_info(_binary_base, "name")
- _binary_path += ";" + get_path_info(_base_dir, "abspath") + ":$_base_name"
+ _binary_path += ";" + "<TARGET_FILE($_binary_base)>"
_binary_paths += [ _binary_path ]
_binary_labels += [ binary.label ]
@@ -167,7 +163,7 @@
pw_doc_sources = rebase_path([ _doc_rst_output ], root_build_dir)
}
script = "$dir_pw_bloat/py/no_bloaty.py"
- args = [ _doc_rst_output ]
+ args = [ rebase_path(_doc_rst_output) ]
outputs = [ _doc_rst_output ]
}
@@ -330,7 +326,7 @@
pw_doc_sources = rebase_path([ _doc_rst_output ], root_build_dir)
}
script = "$dir_pw_bloat/py/no_toolchains.py"
- args = [ _doc_rst_output ]
+ args = [ rebase_path(_doc_rst_output) ]
outputs = [ _doc_rst_output ]
}
}
diff --git a/pw_build/docs.rst b/pw_build/docs.rst
index 50a80bc..05b31dc 100644
--- a/pw_build/docs.rst
+++ b/pw_build/docs.rst
@@ -82,10 +82,9 @@
pw_python_script
^^^^^^^^^^^^^^^^
The ``pw_python_script`` template is a convenience wrapper around ``action`` for
-running Python scripts. The main benefit it provides is automatic resolution of
-GN paths to filesystem paths and GN target names to compiled binary files. This
-allows Python scripts to be written independent of GN, taking only filesystem as
-arguments.
+running Python scripts. The main benefit it provides is resolution of GN target
+labels to compiled binary files. This allows Python scripts to be written
+independently of GN, taking only filesystem paths as arguments.
Another convenience provided by the template is to allow running scripts without
any outputs. Sometimes scripts run in a build do not directly produce output
@@ -102,15 +101,59 @@
output file for the script. This allows running scripts without specifying any
``outputs``.
+**Expressions**
+
+``pw_python_script`` evaluates expressions in ``args``, the arguments passed to
+the script. These expressions function similarly to generator expressions in
+CMake. Expressions may be passed as a standalone argument or as part of another
+argument. A single argument may contain multiple expressions.
+
+Generally, these expressions are used within templates rather than directly in
+BUILD.gn files. This allows build code to use GN labels without having to worry
+about converting them to files.
+
+Currently, only one expression is supported.
+
+.. describe:: <TARGET_FILE(gn_target)>
+
+ Evaluates to the output file of the provided GN target. For example, the
+ expression
+
+ .. code::
+
+ "<TARGET_FILE(//foo/bar:static_lib)>"
+
+ might expand to
+
+ .. code::
+
+ "/home/User/project_root/out/obj/foo/bar/static_lib.a"
+
+ ``TARGET_FILE`` parses the ``.ninja`` file for the GN target, so it should
+ always find the correct output file, regardless of the toolchain's or target's
+ configuration. Some targets, such as ``source_set`` and ``group`` targets, do
+ not have an output file, and attempting to use ``TARGET_FILE`` with them
+ results in an error.
+
+ ``TARGET_FILE`` only resolves GN target labels to their outputs. To resolve
+ paths generally, use the standard GN approach of applying the
+ ``rebase_path(path)`` function. With default arguments, ``rebase_path``
+ converts the provided GN path or list of paths to be relative to the build
+ directory, from which all build commands and scripts are executed.
+
**Example**
.. code::
import("$dir_pw_build/python_script.gni")
- python_script("hello_world") {
- script = "py/say_hello.py"
- args = [ "world" ]
+ pw_python_script("postprocess_main_image") {
+ script = "py/postprocess_binary.py"
+ args = [
+ "--database",
+ rebase_path("my/database.csv"),
+ "--binary=<TARGET_FILE(//firmware/images:main)>",
+ ]
stamp = true
}
diff --git a/pw_build/exec.gni b/pw_build/exec.gni
index 9641f26..787d69c 100644
--- a/pw_build/exec.gni
+++ b/pw_build/exec.gni
@@ -72,14 +72,14 @@
if (defined(invoker.env_file)) {
_script_args += [
"--env-file",
- get_path_info(invoker.env_file, "abspath"),
+ rebase_path(invoker.env_file),
]
}
if (defined(invoker.args_file)) {
_script_args += [
"--args-file",
- get_path_info(invoker.args_file, "abspath"),
+ rebase_path(invoker.args_file),
]
if (defined(invoker.skip_empty_args) && invoker.skip_empty_args) {
diff --git a/pw_build/go.gni b/pw_build/go.gni
index bc1efe3..ce84061 100644
--- a/pw_build/go.gni
+++ b/pw_build/go.gni
@@ -138,7 +138,7 @@
args = [
"build",
"-o",
- target_out_dir,
+ rebase_path(target_out_dir),
invoker.package,
]
deps = [
@@ -147,5 +147,15 @@
]
env = [ "GOPATH+=$_default_gopath" ]
env_file = _metadata_file
+
+ _binary_name = get_path_info(invoker.package, "name")
+
+ if (host_os == "win") {
+ _extension = ".exe"
+ } else {
+ _extension = ""
+ }
+
+ outputs = [ "$target_out_dir/$_binary_name$_extension" ]
}
}
diff --git a/pw_build/host_tool.gni b/pw_build/host_tool.gni
index 4ace101..7c092bb 100644
--- a/pw_build/host_tool.gni
+++ b/pw_build/host_tool.gni
@@ -23,33 +23,20 @@
# Defines a Pigweed host tool and installs it into the host_tools directory.
#
# Args:
- # deps: List containing exactly one target which outputs a binary tool.
+ # tool: The target that outputs the binary tool.
# name: Optional name for the installed program. Defaults to the name of
# the compiled binary.
template("pw_host_tool") {
- assert(defined(invoker.deps),
- "pw_host_tool must specify an executable as a dependency")
-
- num_deps = 0
- foreach(_dep, invoker.deps) {
- num_deps += 1
- }
- assert(num_deps == 1, "pw_host_tool must have exactly one dependency")
-
- _host_tools_dir = "$root_out_dir/host_tools"
-
- # Can't do invoker.deps[0] in GN.
- _deps = invoker.deps
- _out_target = get_label_info(_deps[0], "target_out_dir") + ":" +
- get_label_info(_deps[0], "name")
+ assert(defined(invoker.tool),
+ "pw_host_tool must specify an executable as the tool variable")
_script_args = [
"--src",
- _out_target,
+ "<TARGET_FILE(${invoker.tool})>",
"--dst",
- _host_tools_dir,
+ rebase_path("$root_out_dir/host_tools"),
"--out-root",
- root_out_dir,
+ rebase_path(root_out_dir),
]
if (defined(invoker.name) && invoker.name != "") {
@@ -62,7 +49,7 @@
pw_python_script(target_name) {
script = "$dir_pw_build/py/pw_build/host_tool.py"
args = _script_args
- deps = _deps
+ deps = [ invoker.tool ]
stamp = true
}
}
diff --git a/pw_build/linker_script.gni b/pw_build/linker_script.gni
index ba4a4d8..47dfd8f 100644
--- a/pw_build/linker_script.gni
+++ b/pw_build/linker_script.gni
@@ -74,7 +74,7 @@
# Treat the following file as a C file.
"-x",
"c",
- get_path_info(invoker.linker_script, "abspath"),
+ rebase_path(invoker.linker_script),
]
# Include any explicitly listed c flags.
@@ -90,7 +90,7 @@
# Set output file.
args += [
"-o",
- _final_linker_script,
+ rebase_path(_final_linker_script),
]
outputs = [ _final_linker_script ]
}
diff --git a/pw_build/py/pw_build/python_runner.py b/pw_build/py/pw_build/python_runner.py
index 167de87..164b996 100755
--- a/pw_build/py/pw_build/python_runner.py
+++ b/pw_build/py/pw_build/python_runner.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
@@ -11,63 +11,52 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
-"""Script that preprocesses a Python command then runs it."""
+"""Script that preprocesses a Python command then runs it.
+
+This script evaluates expressions in the Python command's arguments then invokes
+the command. Only one expression is supported currently:
+
+ <TARGET_FILE(gn_target)> -- gets the target output file (e.g. .elf, .a,, .so)
+ for a GN target; raises an error for targets with no output file, such as
+ a source_set or group
+"""
import argparse
+from dataclasses import dataclass
import logging
-import os
from pathlib import Path
import re
import shlex
import subprocess
import sys
-from typing import Collection, Iterator
+from typing import Callable, Dict, Iterator, List, NamedTuple, Optional, Tuple
_LOG = logging.getLogger(__name__)
-# Internally, all GN absolute paths start with a forward slash. This means that
-# Windows absolute paths take the form
-#
-# /C:/foo/bar
-#
-# These are not valid filesystem paths, and break if used. As this script has
-# to duplicate GN's path resolution logic to convert internal paths to real
-# filesystem paths, it has to try to detect strings of this form and correct
-# them to well-formed paths.
-#
-# TODO(pwbug/110): This is the latest hack in a series of edge case handling
-# implemented by this script, which is run on every string in sys.argv and could
-# have unintended consequences. This script shouldn't have to exist--GN should
-# standardize a way of finding a compiled binary for a build target.
-def _resembles_internal_gn_windows_path(path: str) -> bool:
- return os.name == 'nt' and bool(re.match(r'^/[a-zA-Z]:[/\\]', path))
-
-
-def _fix_windows_absolute_path(path: str) -> str:
- return path[1:] if _resembles_internal_gn_windows_path(path) else path
-
-
-def parse_args() -> argparse.Namespace:
+def _parse_args() -> argparse.Namespace:
"""Parses arguments for this script, splitting out the command to run."""
parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument(
- '--gn-root',
- type=_fix_windows_absolute_path,
- required=True,
- help='Path to the root of the GN tree',
- )
- parser.add_argument(
- '--out-dir',
- type=_fix_windows_absolute_path,
- required=True,
- help='Path to the GN build output directory',
- )
+ parser.add_argument('--gn-root',
+ type=Path,
+ required=True,
+ help=('Path to the root of the GN tree; '
+ 'value of rebase_path("//")'))
+ parser.add_argument('--current-path',
+ type=Path,
+ required=True,
+ help='Value of rebase_path(".")')
+ parser.add_argument('--default-toolchain',
+ required=True,
+ help='Value of default_toolchain')
+ parser.add_argument('--current-toolchain',
+ required=True,
+ help='Value of current_toolchain')
parser.add_argument(
'--touch',
- type=_fix_windows_absolute_path,
- help='File to touch after command is run',
+ type=Path,
+ help='File to touch after the command is run',
)
parser.add_argument(
'--capture-output',
@@ -75,147 +64,288 @@
help='Capture subcommand output; display only on error',
)
parser.add_argument(
- 'command',
+ 'original_cmd',
nargs=argparse.REMAINDER,
help='Python script with arguments to run',
)
return parser.parse_args()
-class TooManyFilesError(Exception):
- def __init__(self, target: Path, files: Collection[Path]):
- super().__init__(f'Found {len(files)} files for target {target}')
- self.target = target
- self.files = files
+class GnPaths(NamedTuple):
+ """The set of paths needed to resolve GN paths to filesystem paths."""
+ root: Path
+ build: Path
+ cwd: Path
+
+ # Toolchain label or '' if using the default toolchain
+ toolchain: str
+
+ def resolve(self, gn_path: str) -> Path:
+ """Resolves a GN path to a filesystem path."""
+ if gn_path.startswith('//'):
+ return self.root.joinpath(gn_path[2:]).resolve()
+
+ return self.cwd.joinpath(gn_path).resolve()
+
+ def resolve_paths(self, gn_paths: str, sep: str = ';') -> str:
+ """Resolves GN paths to filesystem paths in a delimited string."""
+ return sep.join(
+ str(self.resolve(path)) for path in gn_paths.split(sep))
-# Look for files with these extensions.
-_EXTENSIONS = '', '.elf', '.exe', '.a', '.so'
+@dataclass(frozen=True)
+class Label:
+ """Represents a GN label."""
+ name: str
+ dir: Path
+ relative_dir: Path
+ toolchain: Optional['Label']
+ out_dir: Path
+ gen_dir: Path
+
+ def __init__(self, paths: GnPaths, label: str):
+ # Use this lambda to set attributes on this frozen dataclass.
+ set_attr = lambda attr, val: object.__setattr__(self, attr, val)
+
+ # Handle explicitly-specified toolchains
+ if label.endswith(')'):
+ label, toolchain = label[:-1].rsplit('(', 1)
+ else:
+ # Prevent infinite recursion for toolchains
+ toolchain = paths.toolchain if paths.toolchain != label else ''
+
+ set_attr('toolchain', Label(paths, toolchain) if toolchain else None)
+
+ # Split off the :target, if provided, or use the last part of the path.
+ try:
+ directory, name = label.rsplit(':', 1)
+ except ValueError:
+ directory, name = label, label.rsplit('/', 1)[-1]
+
+ set_attr('name', name)
+
+ # Resolve the directory to an absolute path
+ set_attr('dir', paths.resolve(directory))
+ set_attr('relative_dir', self.dir.relative_to(paths.root))
+
+ set_attr(
+ 'out_dir',
+ paths.build / self.toolchain_name() / 'obj' / self.relative_dir)
+ set_attr(
+ 'gen_dir',
+ paths.build / self.toolchain_name() / 'gen' / self.relative_dir)
+
+ def gn_label(self) -> str:
+ label = f'//{self.relative_dir.as_posix()}:{self.name}'
+ return f'{label}({self.toolchain!r})' if self.toolchain else label
+
+ def toolchain_name(self) -> str:
+ return self.toolchain.name if self.toolchain else ''
+
+ def __repr__(self) -> str:
+ return self.gn_label()
-def _find_potential_files(path: Path, target_name: str) -> Iterator[Path]:
- for extension in _EXTENSIONS:
- potential_file = path / f'{target_name}{extension}'
- if potential_file.is_file():
- yield potential_file
+class _Artifact(NamedTuple):
+ path: Path
+ variables: Dict[str, str]
-def find_binary(target: Path) -> str:
- """Tries to find a binary for a gn build target.
+_GN_NINJA_BUILD_STATEMENT = re.compile(r'^build (.+):[ \n]')
- Args:
- target: Relative filesystem path to the target's output directory and
- target name, separated by a colon.
- Returns:
- Full path to the target's binary.
+def _parse_build_artifacts(build_dir: Path, fd) -> Iterator[_Artifact]:
+ """Partially parses the build statements in a Ninja file."""
+ lines = iter(fd)
- Raises:
- FileNotFoundError: No binary found for the target.
- TooManyFilesError: Found multiple binaries for the target.
+ def next_line():
+ try:
+ return next(lines)
+ except StopIteration:
+ return None
+
+ # Serves as the parse state (only two states)
+ artifact: Optional[_Artifact] = None
+
+ line = next_line()
+
+ while line is not None:
+ if artifact:
+ if line.startswith(' '): # build variable statements are indented
+ key, value = (a.strip() for a in line.split('=', 1))
+ artifact.variables[key] = value
+ line = next_line()
+ else:
+ yield artifact
+ artifact = None
+ else:
+ match = _GN_NINJA_BUILD_STATEMENT.match(line)
+ if match:
+ artifact = _Artifact(build_dir / match.group(1), {})
+
+ line = next_line()
+
+ if artifact:
+ yield artifact
+
+
+def _search_target_ninja(ninja_file: Path, paths: GnPaths,
+ target: Label) -> Tuple[Optional[Path], List[Path]]:
+ """Parses the main output file and object files from <target>.ninja."""
+
+ artifact: Optional[Path] = None
+ objects: List[Path] = []
+
+ _LOG.debug('Parsing target Ninja file %s for %s', ninja_file, target)
+
+ with ninja_file.open() as fd:
+ for path, variables in _parse_build_artifacts(paths.build, fd):
+ # GN uses .stamp files when there is no build artifact.
+ if path.suffix == '.stamp':
+ continue
+
+ if variables:
+ assert not artifact, f'Multiple artifacts for {target}!'
+ artifact = path
+ else:
+ objects.append(path)
+
+ return artifact, objects
+
+
+def _search_toolchain_ninja(paths: GnPaths, target: Label) -> Optional[Path]:
+ """Searches the toolchain.ninja file for <target>.stamp.
+
+ Files created by an action appear in toolchain.ninja instead of in their own
+ <target>.ninja. If the specified target has a single output file in
+ toolchain.ninja, this function returns its path.
"""
+ ninja_file = paths.build / target.toolchain_name() / 'toolchain.ninja'
- target_dirname, target_name = target.name.rsplit(':', 1)
+ _LOG.debug('Searching toolchain Ninja file %s for %s', ninja_file, target)
- potential_files = set(
- _find_potential_files(target.parent / target_dirname, target_name))
+ stamp_dir = target.out_dir.relative_to(paths.build).as_posix()
+ stamp_tool = f'{target.toolchain_name()}_stamp'
+ stamp_statement = f'build {stamp_dir}/{target.name}.stamp: {stamp_tool} '
- if not potential_files:
- raise FileNotFoundError(
- f'Could not find output binary for build target {target}')
+ with ninja_file.open() as fd:
+ for line in fd:
+ if line.startswith(stamp_statement):
+ output_files = line[len(stamp_statement):].strip().split()
+ if len(output_files) == 1:
+ return paths.build / output_files[0]
- if len(potential_files) > 1:
- raise TooManyFilesError(target, potential_files)
+ break
- return str(next(iter(potential_files)))
+ return None
-def _resolve_path(gn_root: str, out_dir: str, string: str) -> str:
- """Resolves a string to a filesystem path if it is a GN path.
+@dataclass(frozen=True)
+class TargetInfo:
+ """Provides information about a target parsed from a .ninja file."""
- If the path specifies a GN target, attempts to find an compiled output
- binary for the target name.
- """
+ label: Label
+ artifact: Optional[Path]
+ object_files: Tuple[Path]
- string = _fix_windows_absolute_path(string)
+ def __init__(self, paths: GnPaths, target: str):
+ object.__setattr__(self, 'label', Label(paths, target))
- is_gn_path = string.startswith('//')
- is_out_path = string.startswith(out_dir)
- if not (is_gn_path or is_out_path):
- # If the string is not a path, do nothing.
- return string
+ ninja = self.label.out_dir / f'{self.label.name}.ninja'
+ if ninja.exists():
+ artifact, objects = _search_target_ninja(ninja, paths, self.label)
+ else:
+ artifact = _search_toolchain_ninja(paths, self.label)
+ objects = []
- full_path = gn_root + string[2:] if is_gn_path else string
- resolved_path = Path(full_path).resolve()
+ object.__setattr__(self, 'artifact', artifact)
+ object.__setattr__(self, 'object_files', tuple(objects))
- # GN targets exist in the out directory and have the format
- # '/path/to/directory:target_name'.
- #
- # Pathlib interprets 'directory:target_name' as the filename, so check if it
- # contains a colon.
- if is_out_path and ':' in resolved_path.name:
- return find_binary(resolved_path)
-
- return str(resolved_path)
+ def __repr__(self) -> str:
+ return repr(self.label)
-def resolve_path(gn_root: str, out_dir: str, string: str) -> str:
- """Resolves GN paths to filesystem paths in a semicolon-separated string.
-
- GN paths are assumed to be absolute, starting with "//". This is replaced
- with the relative filesystem path of the GN root directory.
-
- If the string is not a GN path, it is returned unmodified.
-
- If a path refers to the GN output directory and a target name is defined,
- attempts to locate a binary file for the target within the out directory.
- """
- return ';'.join(
- _resolve_path(gn_root, out_dir, path) for path in string.split(';'))
+class ExpressionError(Exception):
+ """An error occurred while parsing an expression."""
-def main() -> int:
+def _target_output_file(paths: GnPaths, target_name: str) -> str:
+ target = TargetInfo(paths, target_name)
+
+ if not target.artifact:
+ raise ExpressionError(f'Target {target} has no output file!')
+
+ return str(target.artifact)
+
+
+_FUNCTIONS: Dict['str', Callable[[GnPaths, str], str]] = {
+ 'TARGET_FILE': _target_output_file,
+}
+
+_START_EXPRESSION = re.compile(fr'<({"|".join(_FUNCTIONS)})\(')
+
+
+def _expand_expressions(paths: GnPaths, string: str) -> Iterator[str]:
+ pos = None
+
+ for match in _START_EXPRESSION.finditer(string):
+ yield string[pos:match.start()]
+
+ pos = string.find(')>', match.end())
+ if pos == -1:
+ raise ExpressionError('Parse error: no terminating ")>" '
+ f'was found for "{string[match.start():]}"')
+
+ yield _FUNCTIONS[match.group(1)](paths, string[match.end():pos])
+
+ pos += 2 # skip the terminating ')>'
+
+ yield string[pos:]
+
+
+def expand_expressions(paths: GnPaths, arg: str) -> str:
+ """Expands <FUNCTION(...)> expressions."""
+ return ''.join(_expand_expressions(paths, arg))
+
+
+def main(
+ gn_root: Path,
+ current_path: Path,
+ original_cmd: List[str],
+ default_toolchain: str,
+ current_toolchain: str,
+ capture_output: bool,
+ touch: Optional[Path],
+) -> int:
"""Script entry point."""
- args = parse_args()
- if not args.command or args.command[0] != '--':
+ if not original_cmd or original_cmd[0] != '--':
_LOG.error('%s requires a command to run', sys.argv[0])
return 1
+ # GN build scripts are executed from the root build directory.
+ root_build_dir = Path.cwd().resolve()
+
+ tool = current_toolchain if current_toolchain != default_toolchain else ''
+ paths = GnPaths(root=gn_root.resolve(),
+ build=root_build_dir,
+ cwd=current_path.resolve(),
+ toolchain=tool)
+
+ command = [sys.executable]
try:
- resolved_command = [
- resolve_path(args.gn_root, args.out_dir, arg)
- for arg in args.command[1:]
- ]
- except FileNotFoundError as err:
+ command += (expand_expressions(paths, arg) for arg in original_cmd[1:])
+ except ExpressionError as err:
_LOG.error('%s: %s', sys.argv[0], err)
return 1
- except TooManyFilesError as err:
- _LOG.error('%s: %s', sys.argv[0], err)
- _LOG.error('Files found for %s target:\n%s', err.target.name,
- '\n'.join(str(f) for f in err.files))
- _LOG.error('Exactly one file must be found for each target.')
- out_dir_name = args.out_dir.strip('/').split('/', 1)[0]
- _LOG.error(
- 'To fix this, delete and recreate the output directory. '
- 'For example:\n\n'
- ' rm -rf %s\n'
- ' gen gen %s\n', out_dir_name, out_dir_name)
- _LOG.error(
- 'If clearing the output directory does not work, the file '
- 'resolution logic in this script (%s) may need updating.',
- __file__)
- return 1
-
- command = [sys.executable] + resolved_command
_LOG.debug('RUN %s', ' '.join(shlex.quote(arg) for arg in command))
- if args.capture_output:
+ if capture_output:
completed_process = subprocess.run(
command,
- # Combine stdout and stderr so that error messages are
- # correctly interleaved with the rest of the output.
+ # Combine stdout and stderr so that error messages are correctly
+ # interleaved with the rest of the output.
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
@@ -227,17 +357,16 @@
completed_process.returncode)
# TODO(pwbug/34): Print a cross-platform pastable-in-shell command, to
# help users track down what is happening when a command is broken.
- if args.capture_output:
+ if capture_output:
sys.stdout.buffer.write(completed_process.stdout)
- elif args.touch:
+ elif touch:
# If a stamp file is provided and the command executed successfully,
# touch the stamp file to indicate a successful run of the command.
- touch_file = resolve_path(args.gn_root, args.out_dir, args.touch)
- _LOG.debug('TOUCH %s', touch_file)
- Path(touch_file).touch()
+ _LOG.debug('TOUCH %s', touch)
+ touch.touch()
return completed_process.returncode
if __name__ == '__main__':
- sys.exit(main())
+ sys.exit(main(**vars(_parse_args())))
diff --git a/pw_build/py/python_runner_test.py b/pw_build/py/python_runner_test.py
new file mode 100755
index 0000000..bf5f297
--- /dev/null
+++ b/pw_build/py/python_runner_test.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Tests for the Python runner."""
+
+import os
+from pathlib import Path
+import tempfile
+import unittest
+
+from pw_build.python_runner import GnPaths, Label, TargetInfo
+
+TEST_PATHS = GnPaths(Path('/gn_root'), Path('/gn_root/out'),
+ Path('/gn_root/some/cwd'), '//toolchains/cool:ToolChain')
+
+
+class LabelTest(unittest.TestCase):
+ """Tests GN label parsing."""
+ def setUp(self):
+ self._paths_and_toolchain_name = [
+ (TEST_PATHS, 'ToolChain'),
+ (GnPaths(*TEST_PATHS[:3], ''), ''),
+ ]
+
+ def test_root(self):
+ for paths, toolchain in self._paths_and_toolchain_name:
+ label = Label(paths, '//')
+ self.assertEqual(label.name, '')
+ self.assertEqual(label.dir, Path('/gn_root'))
+ self.assertEqual(label.out_dir,
+ Path('/gn_root/out', toolchain, 'obj'))
+ self.assertEqual(label.gen_dir,
+ Path('/gn_root/out', toolchain, 'gen'))
+
+ def test_absolute(self):
+ for paths, toolchain in self._paths_and_toolchain_name:
+ label = Label(paths, '//foo/bar:baz')
+ self.assertEqual(label.name, 'baz')
+ self.assertEqual(label.dir, Path('/gn_root/foo/bar'))
+ self.assertEqual(label.out_dir,
+ Path('/gn_root/out', toolchain, 'obj/foo/bar'))
+ self.assertEqual(label.gen_dir,
+ Path('/gn_root/out', toolchain, 'gen/foo/bar'))
+
+ def test_absolute_implicit_target(self):
+ for paths, toolchain in self._paths_and_toolchain_name:
+ label = Label(paths, '//foo/bar')
+ self.assertEqual(label.name, 'bar')
+ self.assertEqual(label.dir, Path('/gn_root/foo/bar'))
+ self.assertEqual(label.out_dir,
+ Path('/gn_root/out', toolchain, 'obj/foo/bar'))
+ self.assertEqual(label.gen_dir,
+ Path('/gn_root/out', toolchain, 'gen/foo/bar'))
+
+ def test_relative(self):
+ for paths, toolchain in self._paths_and_toolchain_name:
+ label = Label(paths, ':tgt')
+ self.assertEqual(label.name, 'tgt')
+ self.assertEqual(label.dir, Path('/gn_root/some/cwd'))
+ self.assertEqual(label.out_dir,
+ Path('/gn_root/out', toolchain, 'obj/some/cwd'))
+ self.assertEqual(label.gen_dir,
+ Path('/gn_root/out', toolchain, 'gen/some/cwd'))
+
+ def test_relative_subdir(self):
+ for paths, toolchain in self._paths_and_toolchain_name:
+ label = Label(paths, 'tgt')
+ self.assertEqual(label.name, 'tgt')
+ self.assertEqual(label.dir, Path('/gn_root/some/cwd/tgt'))
+ self.assertEqual(
+ label.out_dir,
+ Path('/gn_root/out', toolchain, 'obj/some/cwd/tgt'))
+ self.assertEqual(
+ label.gen_dir,
+ Path('/gn_root/out', toolchain, 'gen/some/cwd/tgt'))
+
+ def test_relative_parent_dir(self):
+ for paths, toolchain in self._paths_and_toolchain_name:
+ label = Label(paths, '..:tgt')
+ self.assertEqual(label.name, 'tgt')
+ self.assertEqual(label.dir, Path('/gn_root/some'))
+ self.assertEqual(label.out_dir,
+ Path('/gn_root/out', toolchain, 'obj/some'))
+ self.assertEqual(label.gen_dir,
+ Path('/gn_root/out', toolchain, 'gen/some'))
+
+
+class ResolvePathTest(unittest.TestCase):
+ """Tests GN path resolution."""
+ def test_resolve_absolute(self):
+ self.assertEqual(TEST_PATHS.resolve('//'), TEST_PATHS.root)
+ self.assertEqual(TEST_PATHS.resolve('//foo/bar'),
+ TEST_PATHS.root / 'foo' / 'bar')
+ self.assertEqual(TEST_PATHS.resolve('//foo/../baz'),
+ TEST_PATHS.root / 'baz')
+
+ def test_resolve_relative(self):
+ self.assertEqual(TEST_PATHS.resolve(''), TEST_PATHS.cwd)
+ self.assertEqual(TEST_PATHS.resolve('foo'), TEST_PATHS.cwd / 'foo')
+ self.assertEqual(TEST_PATHS.resolve('..'), TEST_PATHS.root / 'some')
+
+
+NINJA_EXECUTABLE = '''\
+defines =
+framework_dirs =
+include_dirs = -I../fake_module/public
+cflags = -g3 -Og -fdiagnostics-color -g -fno-common -Wall -Wextra -Werror
+cflags_c =
+cflags_cc = -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register
+target_output_name = this_is_a_test
+
+build fake_toolchain/obj/fake_module/fake_test.fake_test.cc.o: fake_toolchain_cxx ../fake_module/fake_test.cc
+build fake_toolchain/obj/fake_module/fake_test.fake_test_c.c.o: fake_toolchain_cc ../fake_module/fake_test_c.c
+
+build fake_toolchain/obj/fake_module/test/fake_test.elf: fake_tolchain_link fake_tolchain/obj/fake_module/fake_test.fake_test.cc.o fake_tolchain/obj/fake_module/fake_test.fake_test_c.c.o
+ ldflags = -Og -fdiagnostics-color
+ libs =
+ frameworks =
+ output_extension =
+ output_dir = host_clang_debug/obj/fake_module/test
+'''
+
+NINJA_SOURCE_SET = '''\
+defines =
+framework_dirs =
+include_dirs = -I../fake_module/public
+cflags = -g3 -Og -fdiagnostics-color -g -fno-common -Wall -Wextra -Werror
+cflags_c =
+cflags_cc = -fno-rtti -Wnon-virtual-dtor -std=c++17 -Wno-register
+target_output_name = this_is_a_test
+
+build fake_toolchain/obj/fake_module/fake_source_set.file_a.cc.o: fake_toolchain_cxx ../fake_module/file_a.cc
+build fake_toolchain/obj/fake_module/fake_source_set.file_b.c.o: fake_toolchain_cc ../fake_module/file_b.c
+
+build fake_toolchain/obj/fake_module/fake_source_set.stamp: fake_tolchain_link fake_tolchain/obj/fake_module/fake_source_set.file_a.cc.o fake_tolchain/obj/fake_module/fake_source_set.file_b.c.o
+ ldflags = -Og -fdiagnostics-color -Wno-error=deprecated
+ libs =
+ frameworks =
+ output_extension =
+ output_dir = host_clang_debug/obj/fake_module
+'''
+
+
+class TargetTest(unittest.TestCase):
+ """Tests querying GN target information."""
+ def setUp(self):
+ self._tempdir = tempfile.TemporaryDirectory(prefix='pw_build_test')
+ self._paths = GnPaths(root=Path(self._tempdir.name),
+ build=Path(self._tempdir.name, 'out'),
+ cwd=Path(self._tempdir.name, 'some', 'module'),
+ toolchain='//tools:fake_toolchain')
+
+ module = Path(self._tempdir.name, 'out', 'fake_toolchain', 'obj',
+ 'fake_module')
+ os.makedirs(module)
+ module.joinpath('fake_test.ninja').write_text(NINJA_EXECUTABLE)
+ module.joinpath('fake_source_set.ninja').write_text(NINJA_SOURCE_SET)
+
+ self._outdir = Path(self._tempdir.name, 'out', 'fake_toolchain', 'obj',
+ 'fake_module')
+
+ def tearDown(self):
+ self._tempdir.cleanup()
+
+ def test_source_set_artifact(self):
+ target = TargetInfo(self._paths, '//fake_module:fake_source_set')
+ self.assertIsNone(target.artifact)
+
+ def test_source_set_object_files(self):
+ target = TargetInfo(self._paths, '//fake_module:fake_source_set')
+ self.assertEqual(
+ set(target.object_files), {
+ self._outdir / 'fake_source_set.file_a.cc.o',
+ self._outdir / 'fake_source_set.file_b.c.o',
+ })
+
+ def test_executable_object_files(self):
+ target = TargetInfo(self._paths, '//fake_module:fake_test')
+ self.assertEqual(
+ set(target.object_files), {
+ self._outdir / 'fake_test.fake_test.cc.o',
+ self._outdir / 'fake_test.fake_test_c.c.o',
+ })
+
+ def test_executable_artifact(self):
+ target = TargetInfo(self._paths, '//fake_module:fake_test')
+ self.assertEqual(target.artifact,
+ self._outdir / 'test' / 'fake_test.elf')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/pw_build/python_script.gni b/pw_build/python_script.gni
index b251a41..ef53d19 100644
--- a/pw_build/python_script.gni
+++ b/pw_build/python_script.gni
@@ -75,11 +75,14 @@
# GN root directory relative to the build directory (in which the runner
# script is invoked).
"--gn-root",
- rebase_path("//", root_build_dir),
+ rebase_path("//"),
- # Output directory root, used to determine whether to search for a binary.
- "--out-dir",
- root_out_dir,
+ # Current directory, used to resolve relative paths.
+ "--current-path",
+ rebase_path("."),
+
+ "--default-toolchain=$default_toolchain",
+ "--current-toolchain=$current_toolchain",
]
if (defined(invoker.inputs)) {
@@ -104,16 +107,16 @@
_outputs += [ _stamp_file ]
_script_args += [
"--touch",
- _stamp_file,
+ rebase_path(_stamp_file),
]
}
# Capture output or not.
# Note: capture defaults to true.
- if (!defined(invoker.capture_output)) {
- capture_output = true
- } else {
+ if (defined(invoker.capture_output)) {
forward_variables_from(invoker, [ "capture_output" ])
+ } else {
+ capture_output = true
}
if (capture_output) {
_script_args += [ "--capture-output" ]
@@ -124,7 +127,8 @@
# of the Python script to run.
_script_args += [ "--" ]
- _script_args += [ get_path_info(invoker.script, "abspath") ]
+ _script_args += [ rebase_path(invoker.script) ]
+
if (defined(invoker.args)) {
_script_args += invoker.args
}
diff --git a/pw_docgen/docs.gni b/pw_docgen/docs.gni
index 9437878..194c4a9 100644
--- a/pw_docgen/docs.gni
+++ b/pw_docgen/docs.gni
@@ -93,21 +93,18 @@
"--gn-gen-root",
rebase_path(root_gen_dir, root_build_dir) + "/",
"--sphinx-build-dir",
- get_path_info("$target_gen_dir/pw_docgen_tree", "abspath"),
+ rebase_path("$target_gen_dir/pw_docgen_tree"),
"--conf",
- get_path_info(invoker.conf, "abspath"),
+ rebase_path(invoker.conf),
"--out-dir",
- get_path_info(invoker.output_directory, "abspath"),
+ rebase_path(invoker.output_directory),
"--metadata",
]
# Metadata JSON file path.
- _script_args +=
- get_path_info(get_target_outputs(":$_metadata_file_target"), "abspath")
+ _script_args += rebase_path(get_target_outputs(":$_metadata_file_target"))
- foreach(path, invoker.sources) {
- _script_args += [ get_path_info(path, "abspath") ]
- }
+ _script_args += rebase_path(invoker.sources)
if (pw_docgen_BUILD_DOCS) {
pw_python_script(target_name) {
diff --git a/pw_protobuf_compiler/proto.gni b/pw_protobuf_compiler/proto.gni
index 373f5ef..d2652b5 100644
--- a/pw_protobuf_compiler/proto.gni
+++ b/pw_protobuf_compiler/proto.gni
@@ -65,12 +65,12 @@
"--language",
"cc",
"--module-path",
- _module_path,
+ rebase_path(_module_path),
"--include-file",
- invoker.include_file,
+ rebase_path(invoker.include_file),
"--out-dir",
- _proto_gen_dir,
- ] + get_path_info(invoker.protos, "abspath")
+ rebase_path(_proto_gen_dir),
+ ] + rebase_path(invoker.protos)
inputs = invoker.protos
outputs = _outputs
deps = invoker.deps
@@ -128,14 +128,14 @@
"--language",
"nanopb_rpc",
"--module-path",
- _module_path,
+ rebase_path(_module_path),
"--include-paths",
- "$dir_pw_third_party_nanopb/generator/proto",
+ rebase_path("$dir_pw_third_party_nanopb/generator/proto"),
"--include-file",
- invoker.include_file,
+ rebase_path(invoker.include_file),
"--out-dir",
- _proto_gen_dir,
- ] + get_path_info(invoker.protos, "abspath")
+ rebase_path(_proto_gen_dir),
+ ] + rebase_path(invoker.protos)
inputs = invoker.protos
outputs = _outputs
@@ -208,16 +208,17 @@
"--language",
"nanopb",
"--module-path",
- _module_path,
+ rebase_path(_module_path),
"--include-paths",
- "$dir_pw_third_party_nanopb/generator/proto",
+ rebase_path("$dir_pw_third_party_nanopb/generator/proto"),
"--include-file",
- invoker.include_file,
+ rebase_path(invoker.include_file),
"--out-dir",
- _proto_gen_dir,
+ rebase_path(_proto_gen_dir),
"--custom-plugin",
- get_path_info(_nanopb_plugin, "abspath"),
- ] + get_path_info(invoker.protos, "abspath")
+ rebase_path(_nanopb_plugin),
+ ] + rebase_path(invoker.protos)
+
inputs = invoker.protos
outputs = _outputs
@@ -273,12 +274,12 @@
"--language",
"go",
"--module-path",
- "//",
+ rebase_path("//"),
"--include-file",
- invoker.include_file,
+ rebase_path(invoker.include_file),
"--out-dir",
- _proto_gen_dir,
- ] + get_path_info(invoker.protos, "abspath")
+ rebase_path(_proto_gen_dir),
+ ] + rebase_path(invoker.protos)
inputs = invoker.protos
deps = invoker.deps + invoker.gen_deps
stamp = true
diff --git a/pw_target_runner/go/BUILD.gn b/pw_target_runner/go/BUILD.gn
index fd1a8b9..4b071f1 100644
--- a/pw_target_runner/go/BUILD.gn
+++ b/pw_target_runner/go/BUILD.gn
@@ -33,9 +33,9 @@
}
pw_host_tool("simple_client") {
- deps = [ ":pw_target_runner_client" ]
+ tool = ":pw_target_runner_client"
}
pw_host_tool("simple_server") {
- deps = [ ":pw_target_runner_server" ]
+ tool = ":pw_target_runner_server"
}
diff --git a/pw_unit_test/test.gni b/pw_unit_test/test.gni
index f288341..d7d2417 100644
--- a/pw_unit_test/test.gni
+++ b/pw_unit_test/test.gni
@@ -167,9 +167,9 @@
script = "$dir_pw_unit_test/py/pw_unit_test/test_runner.py"
args = [
"--runner",
- pw_unit_test_AUTOMATIC_RUNNER,
+ rebase_path(pw_unit_test_AUTOMATIC_RUNNER),
"--test",
- get_path_info("$_test_output_dir:$_test_to_run", "abspath"),
+ "<TARGET_FILE(:$_test_to_run)>",
]
stamp = true
}