pw_build: Support specifying mypy/pylint config
- Add pylintrc and mypy_ini options to the pw_python_package template.
These can be used to specify configuration files to use for Pylint and
Mypy.
- Run pylint and mypy from the setup directory. This allows tools to
find per-package configuration files (if they aren't specified by the
pylintrc or mypy_ini arguments).
- Fix some incorrect import ordering that Pylint detects now that it
runs from package directories. PEP8 states imports should be grouped
by standard library, third party, and local package imports.
Change-Id: I8017341178ac5920d623ebbed4535432d69527c3
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/26700
Reviewed-by: Keir Mierle <keir@google.com>
Reviewed-by: Alexei Frolov <frolv@google.com>
diff --git a/pw_allocator/py/BUILD.gn b/pw_allocator/py/BUILD.gn
index 2baa9eb..0322cb3 100644
--- a/pw_allocator/py/BUILD.gn
+++ b/pw_allocator/py/BUILD.gn
@@ -23,4 +23,5 @@
"pw_allocator/heap_viewer.py",
]
python_deps = [ "$dir_pw_cli/py" ]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_arduino_build/py/BUILD.gn b/pw_arduino_build/py/BUILD.gn
index c0f2e00..8921293 100644
--- a/pw_arduino_build/py/BUILD.gn
+++ b/pw_arduino_build/py/BUILD.gn
@@ -34,5 +34,5 @@
"builder_test.py",
"file_operations_test.py",
]
- python_deps = [ "$dir_pw_cli/py" ]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_arduino_build/py/pw_arduino_build/unit_test_server.py b/pw_arduino_build/py/pw_arduino_build/unit_test_server.py
index 0fafd71..edbae1d 100644
--- a/pw_arduino_build/py/pw_arduino_build/unit_test_server.py
+++ b/pw_arduino_build/py/pw_arduino_build/unit_test_server.py
@@ -21,6 +21,7 @@
from typing import IO, List, Optional
import pw_cli.process
+
import pw_arduino_build.log
from pw_arduino_build import teensy_detector
from pw_arduino_build.file_operations import decode_file_json
diff --git a/pw_bloat/py/BUILD.gn b/pw_bloat/py/BUILD.gn
index 74f574f..53198ae 100644
--- a/pw_bloat/py/BUILD.gn
+++ b/pw_bloat/py/BUILD.gn
@@ -26,4 +26,5 @@
"pw_bloat/no_bloaty.py",
"pw_bloat/no_toolchains.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_bloat/py/pw_bloat/bloat.py b/pw_bloat/py/pw_bloat/bloat.py
index c3e145b..ee79d79 100755
--- a/pw_bloat/py/pw_bloat/bloat.py
+++ b/pw_bloat/py/pw_bloat/bloat.py
@@ -20,14 +20,13 @@
import os
import subprocess
import sys
-
from typing import List, Iterable, Optional
+import pw_cli.log
+
from pw_bloat.binary_diff import BinaryDiff
from pw_bloat import bloat_output
-import pw_cli.log
-
_LOG = logging.getLogger(__name__)
diff --git a/pw_build/input_group.gni b/pw_build/input_group.gni
index a7b93b4..3bf8b36 100644
--- a/pw_build/input_group.gni
+++ b/pw_build/input_group.gni
@@ -21,8 +21,6 @@
# This is typically used for targets that don't output any artifacts (e.g.
# metadata-only targets) which list input files relevant to the build.
template("pw_input_group") {
- assert(defined(invoker.inputs), "pw_input_group requires some inputs")
-
pw_python_action(target_name) {
ignore_vars = [
"args",
diff --git a/pw_build/py/BUILD.gn b/pw_build/py/BUILD.gn
index 04a30e2..fde1846 100644
--- a/pw_build/py/BUILD.gn
+++ b/pw_build/py/BUILD.gn
@@ -33,4 +33,5 @@
"pw_build/zip_test.py",
]
tests = [ "python_runner_test.py" ]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_build/python.gni b/pw_build/python.gni
index 929a2cd..5321ece 100644
--- a/pw_build/python.gni
+++ b/pw_build/python.gni
@@ -23,13 +23,13 @@
# actions. All subtargets depend on this target.
# - $name.lint - Runs static analyis tools on the Python code. This is a group
# of two subtargets:
-# - $name.lint.mypy - Runs mypy.
-# - $name.lint.pylint - Runs pylint.
+# - $name.lint.mypy - Runs mypy (if enabled).
+# - $name.lint.pylint - Runs pylint (if enabled).
# - $name.tests - Runs all tests for this package.
# - $name.install - Installs the package in a venv.
# - $name.wheel - Builds a Python wheel for the package. (Not implemented.)
#
-# TODO(pwbug/239): Implement installation and wheel building.
+# TODO(pwbug/239): Implement wheel building.
#
# Args:
# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
@@ -39,9 +39,16 @@
# python_deps: Dependencies on other pw_python_packages in the GN build.
# other_deps: Dependencies on GN targets that are not pw_python_packages.
# inputs: Other files to track, such as package_data.
-# lint: If true, applies mypy and pylint to the package. If false or
-# indefined, does not.
-#
+# lint: If true (default), applies mypy and pylint to the package. If false,
+# does not.
+# pylintrc: Optional path to a pylintrc configuration file to use. If not
+# provided, Pylint's default rcfile search is used. Pylint is executed
+# from the package's setup directory, so pylintrc files in that directory
+# will take precedence over others.
+# mypy_ini: Optional path to a mypy configuration file to use. If not
+# provided, mypy's default configuration file search is used. mypy is
+# executed from the package's setup directory, so mypy.ini files in that
+# directory will take precedence over others.
template("pw_python_package") {
if (defined(invoker.sources)) {
_all_py_files = invoker.sources
@@ -189,12 +196,27 @@
# For packages that are not generated, create targets to run mypy and pylint.
# Linting is not performed on generated packages.
if (_should_lint) {
+ # Run lint tools from the setup or target directory so that the tools detect
+ # config files (e.g. pylintrc or mypy.ini) in that directory. Config files
+ # may be explicitly specified with the pylintrc or mypy_ini arguments.
+ if (defined(_setup_dir)) {
+ _lint_directory = rebase_path(_setup_dir)
+ } else {
+ _lint_directory = rebase_path(".")
+ }
+
pw_python_action("$target_name.lint.mypy") {
module = "mypy"
args = [
"--pretty",
"--show-error-codes",
]
+
+ if (defined(invoker.mypy_ini)) {
+ args += [ "--config-file=" + rebase_path(invoker.mypy_ini) ]
+ inputs = [ invoker.mypy_ini ]
+ }
+
if (_is_package) {
args += [ rebase_path(_setup_dir) ]
} else {
@@ -205,6 +227,7 @@
# See https://github.com/python/mypy/issues/7771
environment = [ "MYPY_FORCE_COLOR=1" ]
+ directory = _lint_directory
stamp = true
deps = [ _package_target ]
@@ -218,11 +241,16 @@
pw_python_action_foreach("$target_name.lint.pylint") {
module = "pylint"
args = [
- "{{source_root_relative_dir}}/{{source_file_part}}",
+ rebase_path(".") + "/{{source_target_relative}}",
"--jobs=1",
"--output-format=colorized",
]
+ if (defined(invoker.pylintrc)) {
+ args += [ "--rcfile=" + rebase_path(invoker.pylintrc) ]
+ inputs = [ invoker.pylintrc ]
+ }
+
if (host_os == "win") {
# Allow CRLF on Windows, in case Git is set to switch line endings.
args += [ "--disable=unexpected-line-ending-format" ]
@@ -230,11 +258,8 @@
sources = _all_py_files
- stamp = "$target_gen_dir/{{source_target_relative}}.pylint.pw_pystamp"
-
- # Run pylint from the source root so that pylint detects rcfiles
- # (.pylintrc) in the source tree.
- directory = rebase_path("//")
+ directory = _lint_directory
+ stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"
deps = [ _package_target ]
foreach(dep, _python_deps) {
@@ -242,9 +267,15 @@
}
}
} else {
- group("$target_name.lint.mypy") {
+ pw_input_group("$target_name.lint.mypy") {
+ if (defined(invoker.pylintrc)) {
+ inputs = [ invoker.pylintrc ]
+ }
}
- group("$target_name.lint.pylint") {
+ pw_input_group("$target_name.lint.pylint") {
+ if (defined(invoker.mypy_ini)) {
+ inputs = [ invoker.mypy_ini ]
+ }
}
}
@@ -326,6 +357,7 @@
"python_deps",
"other_deps",
"inputs",
+ "pylintrc",
]
pw_python_package(target_name) {
diff --git a/pw_cli/py/BUILD.gn b/pw_cli/py/BUILD.gn
index 46e803e..9bf6ec6 100644
--- a/pw_cli/py/BUILD.gn
+++ b/pw_cli/py/BUILD.gn
@@ -31,4 +31,5 @@
"pw_cli/plugins.py",
"pw_cli/process.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_docgen/py/BUILD.gn b/pw_docgen/py/BUILD.gn
index 28d4734..1397437 100644
--- a/pw_docgen/py/BUILD.gn
+++ b/pw_docgen/py/BUILD.gn
@@ -22,4 +22,5 @@
"pw_docgen/__init__.py",
"pw_docgen/docgen.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_doctor/py/BUILD.gn b/pw_doctor/py/BUILD.gn
index 0408128..82c2d86 100644
--- a/pw_doctor/py/BUILD.gn
+++ b/pw_doctor/py/BUILD.gn
@@ -22,4 +22,5 @@
"pw_doctor/__init__.py",
"pw_doctor/doctor.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_env_setup/py/BUILD.gn b/pw_env_setup/py/BUILD.gn
index 2ad8b67..81657ad 100644
--- a/pw_env_setup/py/BUILD.gn
+++ b/pw_env_setup/py/BUILD.gn
@@ -34,4 +34,5 @@
"pw_env_setup/virtualenv_setup/install.py",
"pw_env_setup/windows_env_start.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_hdlc_lite/py/BUILD.gn b/pw_hdlc_lite/py/BUILD.gn
index 1b18011..9ef1a81 100644
--- a/pw_hdlc_lite/py/BUILD.gn
+++ b/pw_hdlc_lite/py/BUILD.gn
@@ -34,4 +34,5 @@
"$dir_pw_protobuf_compiler/py",
"$dir_pw_rpc/py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_hdlc_lite/py/pw_hdlc_lite/rpc.py b/pw_hdlc_lite/py/pw_hdlc_lite/rpc.py
index 2e79b5a..642f2f3 100644
--- a/pw_hdlc_lite/py/pw_hdlc_lite/rpc.py
+++ b/pw_hdlc_lite/py/pw_hdlc_lite/rpc.py
@@ -19,11 +19,12 @@
import time
from typing import Any, BinaryIO, Callable, Dict, Iterable, NoReturn
-from pw_hdlc_lite.decode import Frame, FrameDecoder
-from pw_hdlc_lite import encode
+from pw_protobuf_compiler import python_protos
import pw_rpc
from pw_rpc import callback_client
-from pw_protobuf_compiler import python_protos
+
+from pw_hdlc_lite.decode import Frame, FrameDecoder
+from pw_hdlc_lite import encode
_LOG = logging.getLogger(__name__)
diff --git a/pw_hdlc_lite/rpc_example/BUILD.gn b/pw_hdlc_lite/rpc_example/BUILD.gn
index 1ac444c..e0e31fa 100644
--- a/pw_hdlc_lite/rpc_example/BUILD.gn
+++ b/pw_hdlc_lite/rpc_example/BUILD.gn
@@ -40,4 +40,5 @@
pw_python_script("example_script") {
sources = [ "example_script.py" ]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_module/py/BUILD.gn b/pw_module/py/BUILD.gn
index 09f7896..da1b132 100644
--- a/pw_module/py/BUILD.gn
+++ b/pw_module/py/BUILD.gn
@@ -23,4 +23,5 @@
"pw_module/check.py",
]
tests = [ "check_test.py" ]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_package/py/BUILD.gn b/pw_package/py/BUILD.gn
index 6388bc0..0e4296e 100644
--- a/pw_package/py/BUILD.gn
+++ b/pw_package/py/BUILD.gn
@@ -26,4 +26,5 @@
"pw_package/packages/nanopb.py",
"pw_package/pigweed_packages.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_presubmit/py/BUILD.gn b/pw_presubmit/py/BUILD.gn
index a9b96e7..6efb5eb 100644
--- a/pw_presubmit/py/BUILD.gn
+++ b/pw_presubmit/py/BUILD.gn
@@ -39,4 +39,5 @@
"$dir_pw_cli/py",
"$dir_pw_package/py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_protobuf/py/BUILD.gn b/pw_protobuf/py/BUILD.gn
index edee3aa..e977f5d 100644
--- a/pw_protobuf/py/BUILD.gn
+++ b/pw_protobuf/py/BUILD.gn
@@ -25,4 +25,5 @@
"pw_protobuf/plugin.py",
"pw_protobuf/proto_tree.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_protobuf_compiler/py/BUILD.gn b/pw_protobuf_compiler/py/BUILD.gn
index 5924241..9c4187d 100644
--- a/pw_protobuf_compiler/py/BUILD.gn
+++ b/pw_protobuf_compiler/py/BUILD.gn
@@ -27,4 +27,5 @@
]
tests = [ "python_protos_test.py" ]
python_deps = [ "$dir_pw_cli/py" ]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_rpc/py/BUILD.gn b/pw_rpc/py/BUILD.gn
index 6590476..adab838 100644
--- a/pw_rpc/py/BUILD.gn
+++ b/pw_rpc/py/BUILD.gn
@@ -43,4 +43,5 @@
"$dir_pw_protobuf_compiler/py",
"$dir_pw_status/py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_rpc/py/callback_client_test.py b/pw_rpc/py/callback_client_test.py
index 0ae02c4..2b45279 100755
--- a/pw_rpc/py/callback_client_test.py
+++ b/pw_rpc/py/callback_client_test.py
@@ -19,9 +19,10 @@
from typing import List, Tuple
from pw_protobuf_compiler import python_protos
-from pw_rpc import callback_client, client, packet_pb2, packets
from pw_status import Status
+from pw_rpc import callback_client, client, packet_pb2, packets
+
TEST_PROTO_1 = """\
syntax = "proto3";
diff --git a/pw_rpc/py/client_test.py b/pw_rpc/py/client_test.py
index cb78992..edf5db8 100755
--- a/pw_rpc/py/client_test.py
+++ b/pw_rpc/py/client_test.py
@@ -17,10 +17,11 @@
import unittest
from pw_protobuf_compiler import python_protos
+from pw_status import Status
+
from pw_rpc import callback_client, client, packets
from pw_rpc.packet_pb2 import RpcPacket
import pw_rpc.ids
-from pw_status import Status
TEST_PROTO_1 = """\
syntax = "proto3";
diff --git a/pw_rpc/py/packets_test.py b/pw_rpc/py/packets_test.py
index c8ed99e..a58c194 100755
--- a/pw_rpc/py/packets_test.py
+++ b/pw_rpc/py/packets_test.py
@@ -16,9 +16,10 @@
import unittest
+from pw_status import Status
+
from pw_rpc import packets
from pw_rpc.packet_pb2 import RpcPacket
-from pw_status import Status
_TEST_REQUEST = RpcPacket(type=packets.PacketType.REQUEST,
channel_id=1,
diff --git a/pw_rpc/py/pw_rpc/callback_client.py b/pw_rpc/py/pw_rpc/callback_client.py
index f357154..6995ef7 100644
--- a/pw_rpc/py/pw_rpc/callback_client.py
+++ b/pw_rpc/py/pw_rpc/callback_client.py
@@ -45,9 +45,10 @@
import queue
from typing import Any, Callable, Optional, Tuple
+from pw_status import Status
+
from pw_rpc import client
from pw_rpc.descriptors import Channel, Method, Service
-from pw_status import Status
_LOG = logging.getLogger(__name__)
diff --git a/pw_rpc/py/pw_rpc/client.py b/pw_rpc/py/pw_rpc/client.py
index 4d3ec55..3624989 100644
--- a/pw_rpc/py/pw_rpc/client.py
+++ b/pw_rpc/py/pw_rpc/client.py
@@ -19,11 +19,12 @@
from typing import Collection, Dict, Iterable, Iterator, List, NamedTuple
from typing import Optional
+from pw_status import Status
+
from pw_rpc import descriptors, packets
from pw_rpc.packet_pb2 import RpcPacket
from pw_rpc.packets import PacketType
from pw_rpc.descriptors import Channel, Service, Method
-from pw_status import Status
_LOG = logging.getLogger(__name__)
diff --git a/pw_rpc/py/pw_rpc/descriptors.py b/pw_rpc/py/pw_rpc/descriptors.py
index c51aa1a..03ae62b 100644
--- a/pw_rpc/py/pw_rpc/descriptors.py
+++ b/pw_rpc/py/pw_rpc/descriptors.py
@@ -21,9 +21,10 @@
from google.protobuf import descriptor_pb2
from google.protobuf.descriptor import FieldDescriptor
-from pw_rpc import ids
from pw_protobuf_compiler import python_protos
+from pw_rpc import ids
+
@dataclass(frozen=True)
class Channel:
diff --git a/pw_status/py/BUILD.gn b/pw_status/py/BUILD.gn
index d6d12c9..7f4e01c 100644
--- a/pw_status/py/BUILD.gn
+++ b/pw_status/py/BUILD.gn
@@ -22,4 +22,5 @@
"pw_status/__init__.py",
"pw_status/update_style.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_tokenizer/py/BUILD.gn b/pw_tokenizer/py/BUILD.gn
index a8a1c1b..e9e4468 100644
--- a/pw_tokenizer/py/BUILD.gn
+++ b/pw_tokenizer/py/BUILD.gn
@@ -46,4 +46,5 @@
"example_binary_with_tokenized_strings.elf",
"example_legacy_binary_with_tokenized_strings.elf",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_trace/py/BUILD.gn b/pw_trace/py/BUILD.gn
index caf8bc5..736258c 100644
--- a/pw_trace/py/BUILD.gn
+++ b/pw_trace/py/BUILD.gn
@@ -23,4 +23,5 @@
"pw_trace/trace.py",
]
tests = [ "trace_test.py" ]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_trace_tokenized/py/BUILD.gn b/pw_trace_tokenized/py/BUILD.gn
index bae2cde..2116cc5 100644
--- a/pw_trace_tokenized/py/BUILD.gn
+++ b/pw_trace_tokenized/py/BUILD.gn
@@ -22,4 +22,5 @@
"pw_trace_tokenized/__init__.py",
"pw_trace_tokenized/trace_tokenized.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_unit_test/py/BUILD.gn b/pw_unit_test/py/BUILD.gn
index 8f37417..808a07a 100644
--- a/pw_unit_test/py/BUILD.gn
+++ b/pw_unit_test/py/BUILD.gn
@@ -23,4 +23,5 @@
"pw_unit_test/test_runner.py",
]
python_deps = [ "$dir_pw_cli/py" ]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/pw_watch/py/BUILD.gn b/pw_watch/py/BUILD.gn
index a8fe59d..b6c76f8 100644
--- a/pw_watch/py/BUILD.gn
+++ b/pw_watch/py/BUILD.gn
@@ -24,4 +24,5 @@
"pw_watch/watch.py",
"pw_watch/watch_test.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/targets/lm3s6965evb-qemu/py/BUILD.gn b/targets/lm3s6965evb-qemu/py/BUILD.gn
index 8962902..36373ff 100644
--- a/targets/lm3s6965evb-qemu/py/BUILD.gn
+++ b/targets/lm3s6965evb-qemu/py/BUILD.gn
@@ -22,4 +22,5 @@
"lm3s6965evb_qemu_utils/__init__.py",
"lm3s6965evb_qemu_utils/unit_test_runner.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/targets/stm32f429i-disc1/py/BUILD.gn b/targets/stm32f429i-disc1/py/BUILD.gn
index b93104b..d4b229e 100644
--- a/targets/stm32f429i-disc1/py/BUILD.gn
+++ b/targets/stm32f429i-disc1/py/BUILD.gn
@@ -25,4 +25,5 @@
"stm32f429i_disc1_utils/unit_test_runner.py",
"stm32f429i_disc1_utils/unit_test_server.py",
]
+ pylintrc = "$dir_pigweed/.pylintrc"
}
diff --git a/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_server.py b/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_server.py
index c3bfd3d..fec6ee2 100644
--- a/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_server.py
+++ b/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_server.py
@@ -20,11 +20,11 @@
import tempfile
from typing import IO, List, Optional
-from stm32f429i_disc1_utils import stm32f429i_detector
-
import pw_cli.process
import pw_cli.log
+from stm32f429i_disc1_utils import stm32f429i_detector
+
_LOG = logging.getLogger('unit_test_server')
_TEST_RUNNER_COMMAND = 'stm32f429i_disc1_unit_test_runner'