pw_build: Optionally generate setup.py & nest protos
- Support the generated_setup argument for pw_python_package. This
mirrors the package to the out directory and generates a setup.py
for it there.
- Allow pw_proto_library targets to add their protos to an existing
Python package rather than generating a protos-only package.
- Only reinstall --editable packages when setup.py changes rather than
when any file changes.
Requires: pigweed-internal:10840
Change-Id: I35ed555c2667e60d844468eb614ce0a6f76c3d32
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/36504
Commit-Queue: Wyatt Hepler <hepler@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
Reviewed-by: Joe Ethier <jethier@google.com>
Reviewed-by: Alexei Frolov <frolv@google.com>
diff --git a/pw_protobuf_compiler/docs.rst b/pw_protobuf_compiler/docs.rst
index ac659fb..8651c13 100644
--- a/pw_protobuf_compiler/docs.rst
+++ b/pw_protobuf_compiler/docs.rst
@@ -66,6 +66,8 @@
compiled with protoc as ``"nested/foo.proto"``.
* ``strip_prefix``: Remove this prefix from the source protos. All source and
input files must be nested under this path.
+* ``python_package``: Label of Python package in which to nest the proto
+ modules.
**Example**
@@ -149,6 +151,44 @@
└── internal
└── gamma.proto
+.. _module-pw_protobuf_compiler-add-to-python-package:
+
+Adding Python proto modules to an existing package
+--------------------------------------------------
+By default, generated Python proto modules are organized into their own Python
+package. These proto modules can instead be added to an existing Python package
+declared with ``pw_python_library``. This is done by setting the
+``python_package`` argument on the ``pw_proto_library`` and the
+``proto_library`` argument on the ``pw_python_package``.
+
+For example, the protos declared in ``my_protos`` will be nested in the Python
+package declared by ``my_package``.
+
+.. code-block::
+
+ pw_proto_library("my_protos") {
+ sources = [ "hello.proto ]
+ prefix = "foo"
+ python_package = ":my_package"
+ }
+
+ pw_python_pacakge("my_package") {
+ generate_setup = {
+ name = "foo"
+ version = "1.0"
+ }
+ sources = [ "foo/cool_module.py" ]
+ proto_library = ":my_protos"
+ }
+
+The ``hello_pb2.py`` proto module can be used alongside other files in the
+``foo`` package.
+
+.. code-block:: python
+
+ from foo import cool_module, hello_pb2
+
+
Working with externally defined protos
--------------------------------------
``pw_proto_library`` targets may be used to build ``.proto`` sources from
diff --git a/pw_protobuf_compiler/proto.gni b/pw_protobuf_compiler/proto.gni
index 27c608f..c877b28 100644
--- a/pw_protobuf_compiler/proto.gni
+++ b/pw_protobuf_compiler/proto.gni
@@ -49,7 +49,6 @@
rebase_path(get_target_outputs(":${invoker.base_target}._includes"))
pw_python_action("$target_name._gen") {
- forward_variables_from(invoker, [ "metadata" ])
script =
"$dir_pw_protobuf_compiler/py/pw_protobuf_compiler/generate_protos.py"
@@ -89,6 +88,15 @@
} else {
stamp = true
}
+
+ if (defined(invoker.metadata)) {
+ metadata = invoker.metadata
+ } else {
+ metadata = {
+ protoc_outputs = rebase_path(outputs)
+ root = [ rebase_path(_out_dir) ]
+ }
+ }
}
}
@@ -252,42 +260,49 @@
# the generated files. This is internal and should not be used outside of this
# file. Use pw_proto_library instead.
template("_pw_python_proto_library") {
- _target = target_name
-
_pw_invoke_protoc(target_name) {
- forward_variables_from(invoker, "*", _forwarded_vars)
+ forward_variables_from(invoker, "*", _forwarded_vars + [ "python_package" ])
language = "python"
python_deps = [ "$dir_pw_protobuf_compiler:protobuf_requirements" ]
}
- _setup_py = "${invoker.base_out_dir}/python/setup.py"
+ if (defined(invoker.python_package) && invoker.python_package != "") {
+ # If nested in a Python package, write the package's name to a file so
+ # pw_python_package can check that the dependencies are correct.
+ write_file("${invoker.base_out_dir}/python_package.txt",
+ get_label_info(invoker.python_package, "label_no_toolchain"))
- # Create the setup and init files for the Python package.
- action(target_name + "._package_gen") {
- script = "$dir_pw_protobuf_compiler/py/pw_protobuf_compiler/generate_python_package.py"
- args = [
- "--setup",
- rebase_path(_setup_py),
- "--package",
- invoker._package_dir,
- ] + rebase_path(invoker.outputs, "${invoker.base_out_dir}/python")
-
- if (invoker.module_as_package != "") {
- args += [ "--module-as-package" ]
+ # If anyone attempts to depend on this Python package, print an error.
+ pw_error(target_name) {
+ _pkg = get_label_info(invoker.python_package, "label_no_toolchain")
+ message_lines = [
+ "This proto Python package is embedded in the $_pkg Python package.",
+ "It cannot be used directly; instead, depend on $_pkg.",
+ ]
}
+ foreach(subtarget, pw_python_package_subtargets) {
+ group("$target_name.$subtarget") {
+ deps = [ ":${invoker.target_name}" ]
+ }
+ }
+ } else {
+ write_file("${invoker.base_out_dir}/python_package.txt", "")
- public_deps = [ ":$_target._gen($default_toolchain)" ]
- outputs = [ _setup_py ]
- }
+ # Create a Python package with the generated source files.
+ pw_python_package(target_name) {
+ forward_variables_from(invoker, _forwarded_vars)
+ generate_setup = {
+ name = invoker._package_dir
+ version = "0.0.1" # TODO(hepler): Need to be able to set this verison.
+ }
+ sources = invoker.outputs
+ strip_prefix = "${invoker.base_out_dir}/python"
+ python_deps = invoker.deps
+ other_deps = [ ":$target_name._gen($default_toolchain)" ]
+ lint = false
- # Create a Python package with the generated source files.
- pw_python_package(target_name) {
- forward_variables_from(invoker, _forwarded_vars)
- setup = [ _setup_py ]
- sources = invoker.outputs
- python_deps = invoker.deps
- other_deps = [ ":$_target._package_gen($default_toolchain)" ]
- _pw_generated = true
+ _pw_module_as_package = invoker.module_as_package != ""
+ }
}
}
@@ -309,6 +324,7 @@
# compiled with protoc as "nested/foo.proto".
# strip_prefix: Remove this prefix from the source protos. All source and
# input files must be nested under this path.
+# python_package: Label of Python package in which to nest the proto modules.
#
template("pw_proto_library") {
assert(defined(invoker.sources) && invoker.sources != [],
@@ -348,8 +364,9 @@
# This is the output directory for all files related to this proto library.
# Sources are mirrored to "$base_out_dir/sources" and protoc puts outputs in
# "$base_out_dir/$language" by default.
- base_out_dir = get_label_info(":$target_name($default_toolchain)",
- "target_gen_dir") + "/$target_name"
+ base_out_dir =
+ get_label_info(":$target_name($default_toolchain)", "target_gen_dir") +
+ "/$target_name.proto_library"
compile_dir = "$base_out_dir/sources"
@@ -422,7 +439,7 @@
deps = process_file_template(_deps, "{{source}}._includes")
data_keys = [ "protoc_includes" ]
- outputs = [ "$target_gen_dir/${_common.base_target}/includes.txt" ]
+ outputs = [ "${_common.base_out_dir}/includes.txt" ]
# Indicate this library's base directory for its dependents.
metadata = {
@@ -537,6 +554,7 @@
_pw_python_proto_library("$target_name.python") {
forward_variables_from(_common, "*")
+ forward_variables_from(invoker, [ "python_package" ])
module_as_package = _module_as_package
deps = []
diff --git a/pw_protobuf_compiler/py/BUILD.gn b/pw_protobuf_compiler/py/BUILD.gn
index 1d5a10b..687cb1a 100644
--- a/pw_protobuf_compiler/py/BUILD.gn
+++ b/pw_protobuf_compiler/py/BUILD.gn
@@ -22,7 +22,6 @@
sources = [
"pw_protobuf_compiler/__init__.py",
"pw_protobuf_compiler/generate_protos.py",
- "pw_protobuf_compiler/generate_python_package.py",
"pw_protobuf_compiler/proto_target_invalid.py",
"pw_protobuf_compiler/python_protos.py",
]
diff --git a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_python_package.py b/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_python_package.py
deleted file mode 100644
index 9147f08..0000000
--- a/pw_protobuf_compiler/py/pw_protobuf_compiler/generate_python_package.py
+++ /dev/null
@@ -1,112 +0,0 @@
-# Copyright 2020 The Pigweed Authors
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may not
-# use this file except in compliance with the License. You may obtain a copy of
-# the License at
-#
-# https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations under
-# the License.
-"""Generates a setup.py and __init__.py for a Python package."""
-
-import argparse
-from collections import defaultdict
-from pathlib import Path
-import sys
-from typing import Dict, List, Set
-
-# Make sure dependencies are optional, since this script may be run when
-# installing Python package dependencies through GN.
-try:
- from pw_cli.log import install as setup_logging
-except ImportError:
- from logging import basicConfig as setup_logging # type: ignore
-
-_SETUP_TEMPLATE = """# Generated file. Do not modify.
-import setuptools
-
-setuptools.setup(
- name={name!r},
- version='0.0.1',
- author='Pigweed Authors',
- author_email='pigweed-developers@googlegroups.com',
- description='Generated protobuf files',
- packages={packages!r},
- package_data={package_data!r},
- include_package_data=True,
- zip_safe=False,
- install_requires=['protobuf'],
-)
-"""
-
-
-def _parse_args():
- """Parses and returns the command line arguments."""
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument('--package',
- required=True,
- help='Name of the generated Python package')
- parser.add_argument('--setup',
- required=True,
- type=Path,
- help='Path to setup.py file')
- parser.add_argument('--module-as-package',
- action='store_true',
- help='The package is a standalone external proto')
- parser.add_argument('sources',
- type=Path,
- nargs='+',
- help='Relative paths to the .py and .pyi files')
- return parser.parse_args()
-
-
-def main(package: str, setup: Path, module_as_package: bool,
- sources: List[Path]) -> int:
- """Generates __init__.py and py.typed files and a setup.py."""
- assert not module_as_package or len(sources) == 2
-
- base = setup.parent.resolve()
- base.mkdir(exist_ok=True)
-
- # Find all directories in the package, including empty ones.
- subpackages: Set[Path] = set()
- for source in sources:
- subpackages.update(base / path for path in source.parents)
- subpackages.remove(base)
-
- pkg_data: Dict[str, List[str]] = defaultdict(list)
-
- # Create __init__.py and py.typed files for each subdirectory.
- for pkg in subpackages:
- pkg.mkdir(exist_ok=True, parents=True)
- pkg.joinpath('__init__.py').write_text('')
-
- package_name = pkg.relative_to(base).as_posix().replace('/', '.')
- pkg.joinpath('py.typed').touch()
- pkg_data[package_name].append('py.typed')
-
- # Add the Mypy stub (.pyi) for each source file.
- for mypy_stub in (s for s in sources if s.suffix == '.pyi'):
- pkg = base / mypy_stub.parent
- package_name = pkg.relative_to(base).as_posix().replace('/', '.')
- pkg_data[package_name].append(mypy_stub.name)
-
- if module_as_package:
- pkg.joinpath('__init__.py').write_text(
- f'from {mypy_stub.stem}.{mypy_stub.stem} import *\n')
-
- setup.write_text(
- _SETUP_TEMPLATE.format(name=package,
- packages=list(pkg_data),
- package_data=dict(pkg_data)))
-
- return 0
-
-
-if __name__ == '__main__':
- setup_logging()
- sys.exit(main(**vars(_parse_args())))