pw_build: Mirror all Python sources under one dir
- Create package_metadata.json files for each python Package in the GN
build graph.
- New GN template to copy Python package sources under one directory.
- New GN target to copy all Pigweed's Python packages. This will
operate on both in-tree and generated Python packages.
pw_create_python_source_tree("build_pigweed_python_source_tree")
Change-Id: If2ae60860fba856804f7c0b2788f35c0ffc59a44
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/56841
Commit-Queue: Anthony DiGirolamo <tonymd@google.com>
Reviewed-by: Wyatt Hepler <hepler@google.com>
diff --git a/BUILD.gn b/BUILD.gn
index a020856..cbaea9d 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -46,6 +46,7 @@
":host",
":integration_tests",
":stm32f429i",
+ "$dir_pw_env_setup:build_pigweed_python_source_tree",
"$dir_pw_env_setup:python.install",
"$dir_pw_env_setup:python.lint",
"$dir_pw_env_setup:python.tests",
diff --git a/pw_build/py/BUILD.gn b/pw_build/py/BUILD.gn
index 50de9a6..1f4a973 100644
--- a/pw_build/py/BUILD.gn
+++ b/pw_build/py/BUILD.gn
@@ -26,6 +26,7 @@
"pw_build/__init__.py",
"pw_build/collect_wheels.py",
"pw_build/copy_from_cipd.py",
+ "pw_build/create_python_tree.py",
"pw_build/error.py",
"pw_build/exec.py",
"pw_build/generate_python_package.py",
@@ -34,11 +35,13 @@
"pw_build/host_tool.py",
"pw_build/mirror_tree.py",
"pw_build/nop.py",
+ "pw_build/python_package.py",
"pw_build/python_runner.py",
"pw_build/python_wheels.py",
"pw_build/zip.py",
]
tests = [
+ "create_python_tree_test.py",
"python_runner_test.py",
"zip_test.py",
]
diff --git a/pw_build/py/create_python_tree_test.py b/pw_build/py/create_python_tree_test.py
new file mode 100644
index 0000000..2314cc7
--- /dev/null
+++ b/pw_build/py/create_python_tree_test.py
@@ -0,0 +1,313 @@
+# Copyright 2021 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Tests for pw_build.create_python_tree"""
+
+import os
+from pathlib import Path
+import tempfile
+from typing import List
+import unittest
+
+from parameterized import parameterized # type: ignore
+
+from pw_build.python_package import PythonPackage
+from pw_build.create_python_tree import build_python_tree, copy_extra_files
+
+
+def _setup_cfg(package_name: str) -> str:
+ return f'''
+[metadata]
+name = {package_name}
+version = 0.0.1
+author = Pigweed Authors
+author_email = pigweed-developers@googlegroups.com
+description = Pigweed swiss-army knife
+
+[options]
+packages = find:
+zip_safe = False
+
+[options.package_data]
+{package_name} =
+ py.typed
+ '''
+
+
+def _create_fake_python_package(location: Path, files: List[str],
+ package_name: str) -> None:
+ for file in files:
+ destination = location / file
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ text = f'"""{package_name}"""'
+ if str(destination).endswith('setup.cfg'):
+ text = _setup_cfg(package_name)
+ destination.write_text(text)
+
+
+class TestCreatePythonTree(unittest.TestCase):
+ """Integration tests for create_python_tree."""
+ def setUp(self):
+ self.maxDiff = None # pylint: disable=invalid-name
+ # Save the starting working directory for returning to later.
+ self.start_dir = Path.cwd()
+ # Create a temp out directory
+ self.temp_dir = tempfile.TemporaryDirectory()
+
+ def tearDown(self):
+ # cd to the starting dir before cleaning up the temp out directory
+ os.chdir(self.start_dir)
+ # Delete the TemporaryDirectory
+ self.temp_dir.cleanup()
+
+ def _check_result_paths_equal(self, install_dir, expected_results) -> None:
+ # Normalize path strings to posix before comparing.
+ expected_paths = set(Path(p).as_posix() for p in expected_results)
+ actual_paths = set(
+ p.relative_to(install_dir).as_posix()
+ for p in install_dir.glob('**/*') if p.is_file())
+ self.assertEqual(expected_paths, actual_paths)
+
+ @parameterized.expand([
+ (
+ # Test name
+ 'working case',
+ # Package name
+ 'mars',
+ # File list
+ [
+ 'planets/BUILD.mars_rocket',
+ 'planets/mars/__init__.py',
+ 'planets/mars/__main__.py',
+ 'planets/mars/moons/__init__.py',
+ 'planets/mars/moons/deimos.py',
+ 'planets/mars/moons/phobos.py',
+ 'planets/hohmann_transfer_test.py',
+ 'planets/pyproject.toml',
+ 'planets/setup.cfg',
+ ],
+ # Extra_files
+ [],
+ # Package definition
+ {
+ 'generate_setup': {
+ 'metadata': {
+ 'name': 'mars',
+ 'version': '0.0.1'
+ },
+ },
+ 'inputs': [
+ ],
+ 'setup_sources': [
+ 'planets/pyproject.toml',
+ 'planets/setup.cfg',
+ ],
+ 'sources': [
+ 'planets/mars/__init__.py',
+ 'planets/mars/__main__.py',
+ 'planets/mars/moons/__init__.py',
+ 'planets/mars/moons/deimos.py',
+ 'planets/mars/moons/phobos.py',
+ ],
+ 'tests': [
+ 'planets/hohmann_transfer_test.py',
+ ],
+ },
+ # Output file list
+ [
+ 'mars/__init__.py',
+ 'mars/__main__.py',
+ 'mars/moons/__init__.py',
+ 'mars/moons/deimos.py',
+ 'mars/moons/phobos.py',
+ 'mars/tests/hohmann_transfer_test.py',
+ ],
+ ),
+
+ (
+ # Test name
+ 'with extra files',
+ # Package name
+ 'saturn',
+ # File list
+ [
+ 'planets/BUILD.saturn_rocket',
+ 'planets/hohmann_transfer_test.py',
+ 'planets/pyproject.toml',
+ 'planets/saturn/__init__.py',
+ 'planets/saturn/__main__.py',
+ 'planets/saturn/misson.py',
+ 'planets/saturn/moons/__init__.py',
+ 'planets/saturn/moons/enceladus.py',
+ 'planets/saturn/moons/iapetus.py',
+ 'planets/saturn/moons/rhea.py',
+ 'planets/saturn/moons/titan.py',
+ 'planets/setup.cfg',
+ 'planets/setup.py',
+ ],
+ # Extra files
+ [
+ 'planets/BUILD.saturn_rocket > out/saturn/BUILD.rocket',
+ ],
+ # Package definition
+ {
+ 'inputs': [
+ ],
+ 'setup_sources': [
+ 'planets/pyproject.toml',
+ 'planets/setup.cfg',
+ 'planets/setup.py',
+ ],
+ 'sources': [
+ 'planets/saturn/__init__.py',
+ 'planets/saturn/__main__.py',
+ 'planets/saturn/misson.py',
+ 'planets/saturn/moons/__init__.py',
+ 'planets/saturn/moons/enceladus.py',
+ 'planets/saturn/moons/iapetus.py',
+ 'planets/saturn/moons/rhea.py',
+ 'planets/saturn/moons/titan.py',
+ ],
+ 'tests': [
+ 'planets/hohmann_transfer_test.py',
+ ]
+ },
+ # Output file list
+ [
+ 'saturn/BUILD.rocket',
+ 'saturn/__init__.py',
+ 'saturn/__main__.py',
+ 'saturn/misson.py',
+ 'saturn/moons/__init__.py',
+ 'saturn/moons/enceladus.py',
+ 'saturn/moons/iapetus.py',
+ 'saturn/moons/rhea.py',
+ 'saturn/moons/titan.py',
+ 'saturn/tests/hohmann_transfer_test.py',
+ ],
+ ),
+ ]) # yapf: disable
+ def test_build_python_tree(
+ self,
+ _test_name,
+ package_name,
+ file_list,
+ extra_files,
+ package_definition,
+ expected_file_list,
+ ) -> None:
+ """Check results of build_python_tree and copy_extra_files."""
+ temp_root = Path(self.temp_dir.name)
+ _create_fake_python_package(temp_root, file_list, package_name)
+
+ os.chdir(temp_root)
+ install_dir = temp_root / 'out'
+
+ package = PythonPackage.from_dict(**package_definition)
+ build_python_tree(python_packages=[package],
+ tree_destination_dir=install_dir,
+ include_tests=True)
+ copy_extra_files(extra_files)
+
+ # Check expected files are in place.
+ self._check_result_paths_equal(install_dir, expected_file_list)
+
+ @parameterized.expand([
+ (
+ # Test name
+ 'everything in correct locations',
+ # Package name
+ 'planets',
+ # File list
+ [
+ 'BUILD.mars_rocket',
+ ],
+ # Extra_files
+ [
+ 'BUILD.mars_rocket > out/mars/BUILD.rocket',
+ ],
+ # Output file list
+ [
+ 'mars/BUILD.rocket',
+ ],
+ # Should raise exception
+ None,
+ ),
+ (
+ # Test name
+ 'missing source files',
+ # Package name
+ 'planets',
+ # File list
+ [
+ 'BUILD.mars_rocket',
+ ],
+ # Extra_files
+ [
+ 'BUILD.venus_rocket > out/venus/BUILD.rocket',
+ ],
+ # Output file list
+ [],
+ # Should raise exception
+ FileNotFoundError,
+ ),
+ (
+ # Test name
+ 'existing destination files',
+ # Package name
+ 'planets',
+ # File list
+ [
+ 'BUILD.jupiter_rocket',
+ 'out/jupiter/BUILD.rocket',
+ ],
+ # Extra_files
+ [
+ 'BUILD.jupiter_rocket > out/jupiter/BUILD.rocket',
+ ],
+ # Output file list
+ [],
+ # Should raise exception
+ FileExistsError,
+ ),
+ ]) # yapf: disable
+ def test_copy_extra_files(
+ self,
+ _test_name,
+ package_name,
+ file_list,
+ extra_files,
+ expected_file_list,
+ should_raise_exception,
+ ) -> None:
+ """Check results of build_python_tree and copy_extra_files."""
+ temp_root = Path(self.temp_dir.name)
+ _create_fake_python_package(temp_root, file_list, package_name)
+
+ os.chdir(temp_root)
+ install_dir = temp_root / 'out'
+
+ # If exceptions should be raised
+ if should_raise_exception:
+ with self.assertRaises(should_raise_exception):
+ copy_extra_files(extra_files)
+ return
+
+ # Do the copy
+ copy_extra_files(extra_files)
+ # Check expected files are in place.
+ self._check_result_paths_equal(install_dir, expected_file_list)
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/pw_build/py/pw_build/create_python_tree.py b/pw_build/py/pw_build/create_python_tree.py
new file mode 100644
index 0000000..64b4e9c
--- /dev/null
+++ b/pw_build/py/pw_build/create_python_tree.py
@@ -0,0 +1,172 @@
+# Copyright 2021 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Build a Python Source tree."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import shutil
+import tempfile
+from typing import Iterable, List
+
+import setuptools # type: ignore
+
+from pw_build.python_package import PythonPackage
+
+
+def _parse_args():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--tree-destination-dir',
+ type=Path,
+ help='Path to output directory.')
+ parser.add_argument('--include-tests',
+ action='store_true',
+ help='Include tests in the tests dir.')
+ parser.add_argument(
+ '--extra-files',
+ nargs='+',
+ help='Paths to extra files that should be included in the output dir.')
+ parser.add_argument(
+ '--input-list-files',
+ nargs='+',
+ type=Path,
+ help='Paths to text files containing lists of Python package metadata '
+ 'json files.')
+
+ return parser.parse_args()
+
+
+# TODO(tonymd): Implement a way to merge all configs into one.
+def merge_configs():
+ pass
+
+
+def load_packages(input_list_files: Iterable[Path]) -> List[PythonPackage]:
+ """Load Python package metadata and configs."""
+
+ packages = []
+ for input_path in input_list_files:
+
+ with input_path.open() as input_file:
+ # Each line contains the path to a json file.
+ for json_file in input_file.readlines():
+ # Load the json as a dict.
+ json_file_path = Path(json_file.strip()).resolve()
+ with json_file_path.open() as json_fp:
+ json_dict = json.load(json_fp)
+
+ packages.append(PythonPackage.from_dict(**json_dict))
+ return packages
+
+
+def build_python_tree(python_packages: Iterable[PythonPackage],
+ tree_destination_dir: Path,
+ include_tests: bool = False) -> None:
+ """Install PythonPackages to a destination directory."""
+
+ # Save the current out directory
+ out_dir = Path.cwd()
+
+ # Create the root destination directory.
+ destination_path = tree_destination_dir.resolve()
+ # Delete any existing files
+ shutil.rmtree(destination_path, ignore_errors=True)
+ destination_path.mkdir(exist_ok=True)
+
+ # Define a temporary location to run setup.py build in.
+ with tempfile.TemporaryDirectory() as build_base_name:
+ build_base = Path(build_base_name)
+ lib_dir_path = build_base / 'lib'
+
+ for pkg in python_packages:
+ # Create the temp install dir
+ lib_dir_path.mkdir(parents=True, exist_ok=True)
+
+ # cd to the location of setup.py
+ setup_dir_path = out_dir / pkg.setup_dir
+ os.chdir(setup_dir_path)
+ # Run build with temp build-base location
+ # Note: New files will be placed inside lib_dir_path
+ setuptools.setup(script_args=[
+ 'build',
+ '--force',
+ '--build-base',
+ str(build_base),
+ ])
+
+ new_pkg_dir = lib_dir_path / pkg.package_name
+
+ # If tests should be included, copy them to the tests dir
+ if include_tests and pkg.tests:
+ test_dir_path = new_pkg_dir / 'tests'
+ test_dir_path.mkdir(parents=True, exist_ok=True)
+
+ for test_source_path in pkg.tests:
+ shutil.copy(out_dir / test_source_path, test_dir_path)
+
+ # Move installed files from the temp build-base into
+ # destination_path.
+ for new_file in lib_dir_path.glob('*'):
+ # Use str(Path) since shutil.move only accepts path-like objects
+ # in Python 3.9 and up:
+ # https://docs.python.org/3/library/shutil.html#shutil.move
+ shutil.move(str(new_file), str(destination_path))
+
+ # Clean build base lib folder for next install
+ shutil.rmtree(lib_dir_path, ignore_errors=True)
+
+ # cd back to out directory
+ os.chdir(out_dir)
+
+
+def copy_extra_files(extra_file_strings: Iterable[str]) -> None:
+ """Copy extra files to their destinations."""
+ if not extra_file_strings:
+ return
+
+ for extra_file_string in extra_file_strings:
+ # Convert 'source > destination' strings to Paths.
+ input_output = re.split(r' *> *', extra_file_string)
+ source_file = Path(input_output[0])
+ dest_file = Path(input_output[1])
+
+ if not source_file.exists():
+ raise FileNotFoundError(f'extra_file "{source_file}" not found.\n'
+ f' Defined by: "{extra_file_string}"')
+
+ # Copy files and make parent directories.
+ dest_file.parent.mkdir(parents=True, exist_ok=True)
+ # Raise an error if the destination file already exists.
+ if dest_file.exists():
+ raise FileExistsError(
+ f'Copying "{source_file}" would overwrite "{dest_file}"')
+
+ shutil.copy(source_file, dest_file)
+
+
+def main():
+ args = _parse_args()
+
+ py_packages = load_packages(args.input_list_files)
+
+ build_python_tree(python_packages=py_packages,
+ tree_destination_dir=args.tree_destination_dir,
+ include_tests=args.include_tests)
+ copy_extra_files(args.extra_files)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/pw_build/py/pw_build/python_package.py b/pw_build/py/pw_build/python_package.py
new file mode 100644
index 0000000..26c6a65
--- /dev/null
+++ b/pw_build/py/pw_build/python_package.py
@@ -0,0 +1,94 @@
+# Copyright 2021 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Dataclass for a Python package."""
+
+import configparser
+import copy
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Optional
+
+
+@dataclass
+class PythonPackage:
+ """Class to hold a single Python package's metadata."""
+
+ sources: List[Path]
+ setup_sources: List[Path]
+ tests: List[Path]
+ inputs: List[Path]
+ gn_target_name: Optional[str] = None
+ generate_setup: Optional[Dict] = None
+ config: Optional[configparser.ConfigParser] = None
+
+ @staticmethod
+ def from_dict(**kwargs) -> 'PythonPackage':
+ """Build a PythonPackage instance from a dictionary."""
+ transformed_kwargs = copy.copy(kwargs)
+
+ # Transform string filenames to Paths
+ for attribute in ['sources', 'tests', 'inputs', 'setup_sources']:
+ transformed_kwargs[attribute] = [
+ Path(s) for s in kwargs[attribute]
+ ]
+
+ return PythonPackage(**transformed_kwargs)
+
+ def __post_init__(self):
+ # Read the setup.cfg file if possible
+ if not self.config:
+ self.config = self._load_config()
+
+ @property
+ def setup_dir(self) -> Path:
+ assert len(self.setup_sources) > 0
+ # Assuming all setup_source files live in the same parent directory.
+ return self.setup_sources[0].parent
+
+ @property
+ def setup_py(self) -> Path:
+ setup_py = [
+ setup_file for setup_file in self.setup_sources
+ if str(setup_file).endswith('setup.py')
+ ]
+ # setup.py will not exist for GN generated Python packages
+ assert len(setup_py) == 1
+ return setup_py[0]
+
+ @property
+ def setup_cfg(self) -> Path:
+ setup_cfg = [
+ setup_file for setup_file in self.setup_sources
+ if str(setup_file).endswith('setup.cfg')
+ ]
+ assert len(setup_cfg) == 1
+ return setup_cfg[0]
+
+ @property
+ def package_name(self) -> str:
+ assert self.config
+ return self.config['metadata']['name']
+
+ @property
+ def package_dir(self) -> Path:
+ return self.setup_cfg.parent / self.package_name
+
+ def _load_config(self) -> Optional[configparser.ConfigParser]:
+ config = configparser.ConfigParser()
+ # Check for a setup.cfg and load that config.
+ if self.setup_cfg:
+ with self.setup_cfg.open() as config_file:
+ config.read_file(config_file)
+ return config
+ return None
diff --git a/pw_build/python.gni b/pw_build/python.gni
index a75ef10..4ab540c 100644
--- a/pw_build/python.gni
+++ b/pw_build/python.gni
@@ -315,6 +315,39 @@
# pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the
# targets in this toolchain.
if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
+ # Create the package_metadata.json file. This is used by the
+ # pw_create_python_source_tree template.
+ _package_metadata_json_file =
+ "$target_gen_dir/$target_name/package_metadata.json"
+
+ # Get Python package metadata and write to disk as JSON.
+ _package_metadata = {
+ gn_target_name = get_label_info(invoker.target_name, "label_no_toolchain")
+
+ # Get package source files
+ sources = rebase_path(_sources, root_build_dir)
+
+ # Get setup.cfg, pyproject.toml, or setup.py file
+ setup_sources = rebase_path(_setup_sources, root_build_dir)
+
+ # Get test source files
+ tests = rebase_path(_test_sources, root_build_dir)
+
+ # Get package input files (package data)
+ inputs = []
+ if (defined(invoker.inputs)) {
+ inputs = rebase_path(invoker.inputs, root_build_dir)
+ }
+
+ # Get generate_setup
+ if (defined(invoker.generate_setup)) {
+ generate_setup = invoker.generate_setup
+ }
+ }
+
+ # Finally, write out the json
+ write_file(_package_metadata_json_file, _package_metadata, "json")
+
# Declare the main Python package group. This represents the Python files,
# but does not take any actions. GN targets can depend on the package name
# to run when any files in the package change.
@@ -369,6 +402,10 @@
# Generate the setup.py, py.typed, and __init__.py files as needed.
action(target_name) {
+ metadata = {
+ pw_python_package_metadata_json = [ _package_metadata_json_file ]
+ }
+
script = "$dir_pw_build/py/pw_build/generate_python_package.py"
args = [
"--label",
@@ -405,6 +442,9 @@
} else {
# If the package is not generated, use an input group for the sources.
pw_input_group(target_name) {
+ metadata = {
+ pw_python_package_metadata_json = [ _package_metadata_json_file ]
+ }
inputs = _all_py_files
if (defined(invoker.inputs)) {
inputs += invoker.inputs
diff --git a/pw_build/python.rst b/pw_build/python.rst
index 5904885..9b99e2b 100644
--- a/pw_build/python.rst
+++ b/pw_build/python.rst
@@ -8,6 +8,8 @@
.. seealso:: :ref:`docs-python-build`
+.. _module-pw_build-pw_python_package:
+
pw_python_package
=================
The main Python template is ``pw_python_package``. Each ``pw_python_package``
@@ -164,7 +166,7 @@
.. _module-pw_build-python-dist:
---------------------
-Python distributables
+Python Distributables
---------------------
Pigweed also provides some templates to make it easier to bundle Python packages
for deployment. These templates are found in ``pw_build/python_dist.gni``. See
@@ -244,3 +246,81 @@
packages = [ ":some_python_package" ]
inputs = [ "$dir_pw_build/python_dist/README.md > /${target_name}/" ]
}
+
+pw_create_python_source_tree
+============================
+
+Generates a directory of Python packages from source files suitable for
+deployment outside of the project developer environment. The resulting directory
+contains only files mentioned in each package's ``setup.cfg`` file. This is
+useful for bundling multiple Python packages up into a single package for
+distribution to other locations like `<http://pypi.org>`_.
+
+Arguments
+---------
+
+- ``packages`` - A list of :ref:`module-pw_build-pw_python_package` targets to be installed into
+ the build directory. Their dependencies will be pulled in as wheels also.
+
+- ``include_tests`` - If true, copy Python package tests to a ``tests`` subdir.
+
+- ``extra_files`` - A list of extra files that should be included in the output.
+ The format of each item in this list follows this convention:
+
+ .. code-block:: text
+
+ //some/nested/source_file > nested/destination_file
+
+ - Source and destination file should be separated by ``>``.
+
+ - The source file should be a GN target label (starting with ``//``).
+
+ - The destination file will be relative to the generated output
+ directory. Parent directories are automatically created for each file. If a
+ file would be overwritten an error is raised.
+
+
+Example
+-------
+
+:octicon:`file;1em` ./pw_env_setup/BUILD.gn
+
+.. code-block::
+
+ import("//build_overrides/pigweed.gni")
+
+ import("$dir_pw_build/python_dist.gni")
+
+ pw_create_python_source_tree("build_python_source_tree") {
+ packages = [
+ ":some_python_package",
+ ":another_python_package",
+ ]
+ include_tests = true
+ extra_files = [
+ "//README.md > ./README.md",
+ "//some_python_package/py/BUILD.bazel > some_python_package/BUILD.bazel",
+ "//another_python_package/py/BUILD.bazel > another_python_package/BUILD.bazel",
+ ]
+ }
+
+:octicon:`file-directory;1em` ./out/obj/pw_env_setup/build_python_source_tree/
+
+.. code-block:: text
+
+ $ tree ./out/obj/pw_env_setup/build_python_source_tree/
+ ├── README.md
+ ├── some_python_package
+ │ ├── BUILD.bazel
+ │ ├── __init__.py
+ │ ├── py.typed
+ │ ├── some_source_file.py
+ │ └── tests
+ │ └── some_source_test.py
+ └── another_python_package
+ ├── BUILD.bazel
+ ├── __init__.py
+ ├── another_source_file.py
+ ├── py.typed
+ └── tests
+ └── another_source_test.py
diff --git a/pw_build/python_dist.gni b/pw_build/python_dist.gni
index c14697a..bd3d3a0 100644
--- a/pw_build/python_dist.gni
+++ b/pw_build/python_dist.gni
@@ -127,3 +127,99 @@
deps = [ ":${_outer_name}.wheels" ]
}
}
+
+# Generates a directory of Python packages from source files suitable for
+# deployment outside of the project developer environment.
+#
+# The resulting directory contains only files mentioned in each package's
+# setup.cfg file. This is useful for bundling multiple Python packages up
+# into a single package for distribution to other locations like
+# http://pypi.org.
+#
+# Args:
+# packages: A list of pw_python_package targets to be installed into the build
+# directory. Their dependencies will be pulled in as wheels also.
+#
+# include_tests: If true, copy Python package tests to a `tests` subdir.
+#
+# extra_files: A list of extra files that should be included in the output. The
+# format of each item in this list follows this convention:
+# //some/nested/source_file > nested/destination_file
+template("pw_create_python_source_tree") {
+ _output_dir = "${target_out_dir}/${target_name}/"
+ _metadata_json_file_list =
+ "${target_gen_dir}/${target_name}_metadata_path_list.txt"
+
+ # Convert extra_file strings to input, outputs and create_python_tree.py args.
+ _delimiter = ">"
+ _extra_file_inputs = []
+ _extra_file_outputs = []
+ _extra_file_args = []
+ if (defined(invoker.extra_files)) {
+ foreach(input, invoker.extra_files) {
+ # Remove spaces before and after the delimiter
+ input = string_replace(input, " $_delimiter", _delimiter)
+ input = string_replace(input, "$_delimiter ", _delimiter)
+
+ input_list = []
+ input_list = string_split(input, _delimiter)
+
+ # Save the input file
+ _extra_file_inputs += [ input_list[0] ]
+
+ # Save the output file
+ _this_output = _output_dir + "/" + input_list[1]
+ _extra_file_outputs += [ _this_output ]
+
+ # Compose an arg for passing to create_python_tree.py with properly
+ # rebased paths.
+ _extra_file_args +=
+ [ string_join(" $_delimiter ",
+ [
+ rebase_path(input_list[0], root_build_dir),
+ rebase_path(_this_output, root_build_dir),
+ ]) ]
+ }
+ }
+
+ _include_tests = defined(invoker.include_tests) && invoker.include_tests
+
+ # Build a list of relative paths containing all the python
+ # package_metadata.json files we depend on.
+ generated_file("${target_name}._metadata_path_list.txt") {
+ data_keys = [ "pw_python_package_metadata_json" ]
+ rebase = root_build_dir
+ deps = invoker.packages
+ outputs = [ _metadata_json_file_list ]
+ }
+
+ # Run the python action on the metadata_path_list.txt file
+ pw_python_action(target_name) {
+ deps =
+ invoker.packages + [ ":${invoker.target_name}._metadata_path_list.txt" ]
+ script = "$dir_pw_build/py/pw_build/create_python_tree.py"
+ inputs = _extra_file_inputs
+
+ args = [
+ "--tree-destination-dir",
+ rebase_path(_output_dir, root_build_dir),
+ "--input-list-files",
+ rebase_path(_metadata_json_file_list, root_build_dir),
+ ]
+
+ if (_extra_file_args == []) {
+ # No known output files - stamp instead.
+ stamp = true
+ } else {
+ args += [ "--extra-files" ]
+ args += _extra_file_args
+
+ # Include extra_files as outputs
+ outputs = _extra_file_outputs
+ }
+
+ if (_include_tests) {
+ args += [ "--include-tests" ]
+ }
+ }
+}
diff --git a/pw_env_setup/BUILD.gn b/pw_env_setup/BUILD.gn
index 0fd80f5..a92b7a4 100644
--- a/pw_env_setup/BUILD.gn
+++ b/pw_env_setup/BUILD.gn
@@ -15,6 +15,7 @@
import("//build_overrides/pigweed.gni")
import("$dir_pw_build/python.gni")
+import("$dir_pw_build/python_dist.gni")
import("$dir_pw_docgen/docs.gni")
pw_doc_group("docs") {
@@ -22,43 +23,47 @@
sources = [ "docs.rst" ]
}
-pw_python_group("python") {
- python_deps = [
- # Python packages
- "$dir_pw_allocator/py",
- "$dir_pw_arduino_build/py",
- "$dir_pw_bloat/py",
- "$dir_pw_build/py",
- "$dir_pw_build_info/py",
- "$dir_pw_cli/py",
- "$dir_pw_console/py",
- "$dir_pw_cpu_exception_cortex_m/py",
- "$dir_pw_docgen/py",
- "$dir_pw_doctor/py",
- "$dir_pw_env_setup/py",
- "$dir_pw_hdlc/py",
- "$dir_pw_log:protos.python",
- "$dir_pw_log_tokenized/py",
- "$dir_pw_module/py",
- "$dir_pw_package/py",
- "$dir_pw_presubmit/py",
- "$dir_pw_protobuf/py",
- "$dir_pw_protobuf_compiler/py",
- "$dir_pw_rpc/py",
- "$dir_pw_snapshot/py",
- "$dir_pw_status/py",
- "$dir_pw_stm32cube_build/py",
- "$dir_pw_symbolizer/py",
- "$dir_pw_thread/py",
- "$dir_pw_tls_client/py",
- "$dir_pw_tokenizer/py",
- "$dir_pw_toolchain/py",
- "$dir_pw_trace/py",
- "$dir_pw_trace_tokenized/py",
- "$dir_pw_transfer/py",
- "$dir_pw_unit_test/py",
- "$dir_pw_watch/py",
+_pigweed_python_deps = [
+ # Python packages
+ "$dir_pw_allocator/py",
+ "$dir_pw_arduino_build/py",
+ "$dir_pw_bloat/py",
+ "$dir_pw_build/py",
+ "$dir_pw_build_info/py",
+ "$dir_pw_cli/py",
+ "$dir_pw_console/py",
+ "$dir_pw_cpu_exception_cortex_m/py",
+ "$dir_pw_docgen/py",
+ "$dir_pw_doctor/py",
+ "$dir_pw_env_setup/py",
+ "$dir_pw_hdlc/py",
+ "$dir_pw_log:protos.python",
+ "$dir_pw_log_tokenized/py",
+ "$dir_pw_module/py",
+ "$dir_pw_package/py",
+ "$dir_pw_presubmit/py",
+ "$dir_pw_protobuf/py",
+ "$dir_pw_protobuf_compiler/py",
+ "$dir_pw_rpc/py",
+ "$dir_pw_snapshot/py:pw_snapshot",
+ "$dir_pw_snapshot/py:pw_snapshot_metadata",
+ "$dir_pw_status/py",
+ "$dir_pw_stm32cube_build/py",
+ "$dir_pw_symbolizer/py",
+ "$dir_pw_thread/py",
+ "$dir_pw_tls_client/py",
+ "$dir_pw_tokenizer/py",
+ "$dir_pw_toolchain/py",
+ "$dir_pw_trace/py",
+ "$dir_pw_trace_tokenized/py",
+ "$dir_pw_transfer/py",
+ "$dir_pw_unit_test/py",
+ "$dir_pw_watch/py",
+]
+pw_python_group("python") {
+ python_deps = _pigweed_python_deps
+ python_deps += [
# Standalone scripts
"$dir_pw_hdlc/rpc_example:example_script",
"$dir_pw_rpc/py:python_client_cpp_server_test",
@@ -85,3 +90,15 @@
"robotframework==3.1",
]
}
+
+pw_create_python_source_tree("build_pigweed_python_source_tree") {
+ packages = _pigweed_python_deps
+ include_tests = true
+ extra_files = [
+ "$dir_pigweed/pw_cli/py/BUILD.bazel > pw_cli/BUILD.bazel",
+ "$dir_pigweed/pw_protobuf/py/BUILD.bazel > pw_protobuf/BUILD.bazel",
+ "$dir_pigweed/pw_protobuf_compiler/py/BUILD.bazel > pw_protobuf_compiler/BUILD.bazel",
+ "$dir_pigweed/pw_rpc/py/BUILD.bazel > pw_rpc/BUILD.bazel",
+ "$dir_pigweed/pw_status/py/BUILD.bazel > pw_status/BUILD.bazel",
+ ]
+}