pw_build: pw_python_package template
- The pw_python_package template brings Python code into the GN build
and pw watch. Supports mypy, pylint, and test running. Package
installation and wheel building will be supported in the future.
- Declare the pw_tokenizer package with pw_python_package as an example.
- Provide a top-level "python" GN target and add Python linting and test
running to the default group.
Bug: 239
Change-Id: Iebc47dc4e23d911600ca0911d831910f69e66aca
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/20641
Commit-Queue: Wyatt Hepler <hepler@google.com>
Reviewed-by: Keir Mierle <keir@google.com>
diff --git a/.pylintrc b/.pylintrc
index 53635ca..3b3216d 100644
--- a/.pylintrc
+++ b/.pylintrc
@@ -61,7 +61,8 @@
# --enable=similarities". If you want to run only the classes checker, but have
# no Warning level messages displayed, use "--disable=all --enable=classes
# --disable=W".
-disable=fixme,
+disable=bad-continuation, # Rely on yapf for formatting
+ fixme,
subprocess-run-check
# Enable the message, report, category or checker with the given id(s). You can
diff --git a/BUILD.gn b/BUILD.gn
index d0641df..0c0fba8 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -16,6 +16,7 @@
import("$dir_pw_arduino_build/arduino.gni")
import("$dir_pw_build/host_tool.gni")
+import("$dir_pw_build/python.gni")
import("$dir_pw_docgen/docs.gni")
import("$dir_pw_toolchain/generate_toolchain.gni")
import("$dir_pw_unit_test/test.gni")
@@ -69,19 +70,21 @@
}
}
+# Select a default toolchain based on host OS.
+if (host_os == "linux") {
+ _default_toolchain_prefix = "$dir_pigweed/targets/host:host_clang_"
+} else if (host_os == "mac") {
+ _default_toolchain_prefix = "$dir_pigweed/targets/host:host_clang_"
+} else if (host_os == "win") {
+ _default_toolchain_prefix = "$dir_pigweed/targets/host:host_gcc_"
+} else {
+ assert(false, "Please define a host config for your system: $host_os")
+}
+
# Below are a list of GN targets you can build to force Pigweed to build for a
# specific Pigweed target.
_build_pigweed_default_at_all_optimization_levels("host") {
- # Select a toolchain based on host OS.
- if (host_os == "linux") {
- toolchain_prefix = "$dir_pigweed/targets/host:host_clang_"
- } else if (host_os == "mac") {
- toolchain_prefix = "$dir_pigweed/targets/host:host_clang_"
- } else if (host_os == "win") {
- toolchain_prefix = "$dir_pigweed/targets/host:host_gcc_"
- } else {
- assert(false, "Please define a host config for your system: $host_os")
- }
+ toolchain_prefix = _default_toolchain_prefix
}
_build_pigweed_default_at_all_optimization_levels("host_clang") {
@@ -110,6 +113,10 @@
deps = [ ":pigweed_default($dir_pigweed/targets/docs)" ]
}
+pw_python_group("python") {
+ python_deps = [ ":python_packages($_default_toolchain_prefix$pw_default_optimization_level)" ]
+}
+
# By default, Pigweed will build this target when invoking ninja.
group("pigweed_default") {
deps = []
@@ -119,7 +126,11 @@
if (pw_docgen_BUILD_DOCS) {
deps += [ "$dir_pigweed/docs" ]
} else {
- deps += [ ":apps" ]
+ deps += [
+ ":apps",
+ ":python.lint",
+ ":python.tests",
+ ]
if (pw_unit_test_AUTOMATIC_RUNNER == "") {
# Without a test runner defined, build the tests but don't run them.
deps += [ ":pw_module_tests" ]
@@ -149,6 +160,13 @@
# Prevent the default toolchain from parsing any other BUILD.gn files.
if (current_toolchain != default_toolchain) {
+ pw_python_group("python_packages") {
+ python_deps = [
+ "$dir_pw_tokenizer/py",
+ # TODO(pwbug/239): Add all Pigweed Python packages
+ ]
+ }
+
group("apps") {
# Application images built for all targets.
deps = [ "$dir_pw_hdlc_lite/rpc_example" ]
diff --git a/pw_build/python.gni b/pw_build/python.gni
new file mode 100644
index 0000000..f2ca4b0
--- /dev/null
+++ b/pw_build/python.gni
@@ -0,0 +1,239 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("//build_overrides/pigweed.gni")
+
+import("python_script.gni")
+import("target_types.gni")
+
+# Defines a Python package. GN Python packages contain several GN targets:
+#
+# - $name - Provides the Python files in the build, but does not take any
+# actions. All subtargets depend on this target.
+# - $name.lint - Runs static analyis tools on the Python code. This is a group
+# of two subtargets:
+# - $name.lint.mypy - Runs mypy.
+# - $name.lint.pylint - Runs pylint.
+# - $name.tests - Runs all tests for this package.
+# - $name.install - Installs the package in a venv. (Not implemented.)
+# - $name.wheel - Builds a Python wheel for the package. (Not implemented.)
+#
+# TODO(pwbug/239): Implement installation and wheel building.
+#
+# Args:
+# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
+# which must all be in the same directory.
+# sources: Python sources files in the package.
+# tests: Test files for this Python package.
+# python_deps: Dependencies on other pw_python_packages in the GN build.
+# other_deps: Dependencies on GN targets that are not pw_python_packages.
+# inputs: Other files to track, such as package_data.
+#
+template("pw_python_package") {
+ assert(defined(invoker.setup) && invoker.setup != [],
+ "pw_python_package requires 'setup' to point to a setup.py or " +
+ "pyproject.toml and setup.cfg file")
+
+ if (defined(invoker.python_deps)) {
+ _python_deps = invoker.python_deps
+ } else {
+ _python_deps = []
+ }
+
+ if (defined(invoker.sources)) {
+ _all_sources = invoker.sources
+ } else {
+ _all_sources = []
+ }
+
+ if (defined(invoker.tests)) {
+ _test_sources = invoker.tests
+ } else {
+ _test_sources = []
+ }
+
+ _all_sources += _test_sources
+
+ # Get the directory of the setup files. All files must be in the same dir.
+ _setup_dirs = get_path_info(invoker.setup, "dir")
+ _setup_dir = _setup_dirs[0]
+
+ foreach(dir, _setup_dirs) {
+ assert(dir == _setup_dir,
+ "All files in 'setup' must be in the same directory")
+ }
+
+ # Declare the main Python package group. This represents the Python files, but
+ # does not take any actions. GN targets can depend on the package name to run
+ # when any files in the package change.
+ pw_source_set(target_name) {
+ inputs = _all_sources + invoker.setup
+ if (defined(invoker.inputs)) {
+ inputs += invoker.inputs
+ }
+
+ deps = _python_deps
+
+ if (defined(invoker.other_deps)) {
+ deps += invoker.other_deps
+ }
+ }
+
+ _package_target = ":$target_name"
+
+ # TODO(pwbug/239): Add support for installing this package and dependencies
+ # with correct dependency ordering in a virtual environment. The code
+ # below is incomplete and untested.
+ pw_python_script("$target_name.install") {
+ module = "pip"
+ args = [
+ "install",
+ "--editable",
+ rebase_path(_setup_dir),
+ ]
+
+ stamp = true
+
+ deps = [ _package_target ]
+ foreach(dep, _python_deps) {
+ deps += [ "$dep.install" ]
+ }
+ }
+
+ # TODO(pwbug/239): Add support for building groups of wheels. The code below
+ # is incomplete and untested.
+ pw_python_script("$target_name.wheel") {
+ script = "$dir_pw_build/py/pw_build/python_wheels.py"
+
+ args = [
+ "--out_dir",
+ rebase_path(target_out_dir),
+ ]
+ args += rebase_path(invoker.sources)
+
+ deps = [ _package_target ]
+ stamp = true
+ }
+
+ # Define the static analysis targets for this package.
+ group("$target_name.lint") {
+ deps = [
+ "$_package_target.lint.mypy",
+ "$_package_target.lint.pylint",
+ ]
+ }
+
+ pw_python_script_foreach("$target_name.lint.mypy") {
+ module = "mypy"
+ args = [
+ "{{source}}",
+ "--pretty",
+ "--show-error-codes",
+ "--color-output",
+ ]
+ sources = _all_sources
+
+ # Use this environment variable to force mypy to colorize output.
+ # See https://github.com/python/mypy/issues/7771
+ environment = [ "MYPY_FORCE_COLOR=1" ]
+
+ stamp = "$target_gen_dir/{{source_file_part}}.mypy.pw_pystamp"
+
+ deps = [ _package_target ]
+ foreach(dep, _python_deps) {
+ deps += [ "$dep.lint.mypy" ]
+ }
+ }
+
+ pw_python_script_foreach("$target_name.lint.pylint") {
+ module = "pylint"
+ args = [
+ "{{source_root_relative_dir}}/{{source_file_part}}",
+ "--jobs=1",
+ ]
+ sources = _all_sources
+
+ stamp = "$target_gen_dir/{{source_file_part}}.pylint.pw_pystamp"
+
+ # Run pylint from the source root so that pylint detects rcfiles (.pylintrc)
+ # in the source tree.
+ directory = rebase_path("//")
+
+ deps = [ _package_target ]
+ foreach(dep, _python_deps) {
+ deps += [ "$dep.lint.pylint" ]
+ }
+ }
+
+ # Create a target for each test file.
+ _test_targets = []
+
+ foreach(test, _test_sources) {
+ _test_name = string_replace(test, "/", "_")
+ _test_target = "$target_name.tests.$_test_name"
+
+ pw_python_script(_test_target) {
+ script = test
+ stamp = true
+
+ deps = [ _package_target ]
+ foreach(dep, _python_deps) {
+ deps += [ "$dep.tests" ]
+ }
+ }
+
+ _test_targets += [ ":$_test_target" ]
+ }
+
+ group("$target_name.tests") {
+ deps = _test_targets
+ }
+}
+
+# Declares a group of Python packages or other Python groups. pw_python_groups
+# expose the same set of subtargets as pw_python_package (e.g.
+# "$group_name.lint" and "$group_name.tests"), but these apply to all packages
+# in deps and their dependencies.
+template("pw_python_group") {
+ if (defined(invoker.python_deps)) {
+ _python_deps = invoker.python_deps
+ } else {
+ _python_deps = []
+ }
+
+ group(target_name) {
+ deps = _python_deps
+ }
+
+ _subtargets = [
+ "tests",
+ "lint",
+ "lint.mypy",
+ "lint.pylint",
+ "install",
+ "wheel",
+ ]
+
+ foreach(subtarget, _subtargets) {
+ group("$target_name.$subtarget") {
+ deps = []
+ foreach(dep, _python_deps) {
+ # Split out the toolchain to support deps with a toolchain specified.
+ _target = get_label_info(dep, "label_no_toolchain")
+ _toolchain = get_label_info(dep, "toolchain")
+ deps += [ "$_target.$subtarget($_toolchain)" ]
+ }
+ }
+ }
+}
diff --git a/pw_tokenizer/py/BUILD.gn b/pw_tokenizer/py/BUILD.gn
new file mode 100644
index 0000000..7335afe
--- /dev/null
+++ b/pw_tokenizer/py/BUILD.gn
@@ -0,0 +1,40 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("//build_overrides/pigweed.gni")
+
+import("$dir_pw_build/python.gni")
+
+pw_python_package("py") {
+ setup = [ "setup.py" ]
+ tests = [
+ "database_test.py",
+ "detokenize_test.py",
+ "encode_test.py",
+ "tokens_test.py",
+ "decode_test.py",
+ "elf_reader_test.py",
+ ]
+ sources = [
+ "pw_tokenizer/__init__.py",
+ "pw_tokenizer/__main__.py",
+ "pw_tokenizer/database.py",
+ "pw_tokenizer/decode.py",
+ "pw_tokenizer/detokenize.py",
+ "pw_tokenizer/elf_reader.py",
+ "pw_tokenizer/encode.py",
+ "pw_tokenizer/serial_detokenizer.py",
+ "pw_tokenizer/tokens.py",
+ ]
+}
diff --git a/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py b/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py
index d017a8c..e010225 100644
--- a/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py
+++ b/pw_tokenizer/py/pw_tokenizer/serial_detokenizer.py
@@ -21,7 +21,7 @@
import sys
from typing import BinaryIO, Iterable
-import serial
+import serial # type: ignore
from pw_tokenizer import database, detokenize, tokens