pw_build: pw_python_package and mypy fixes
- Use the target-relative path for pylint's stamp so it is unique when
there are multiple files with the same basename (e.g. __init__.py).
- Serialize pip install commands since in-parallel --editable installs
do not work correctly.
- Run mypy over the entire package directory rather than individual
files.
- Fix various mypy issues so that mypy passes without
--ignore-missing-imports.
Change-Id: I8129144d7c963616e5b836dd2f082c41f1dc1416
Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/22201
Reviewed-by: Alexei Frolov <frolv@google.com>
diff --git a/pw_allocator/py/pw_allocator/heap_viewer.py b/pw_allocator/py/pw_allocator/heap_viewer.py
index f866764..72da6ff 100644
--- a/pw_allocator/py/pw_allocator/heap_viewer.py
+++ b/pw_allocator/py/pw_allocator/heap_viewer.py
@@ -19,7 +19,7 @@
import logging
from typing import Optional
from dataclasses import dataclass
-import coloredlogs
+import coloredlogs # type: ignore
@dataclass
diff --git a/pw_allocator/py/setup.py b/pw_allocator/py/setup.py
index 2679154..1d94f71 100644
--- a/pw_allocator/py/setup.py
+++ b/pw_allocator/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_allocator"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_allocator',
diff --git a/pw_arduino_build/py/builder_test.py b/pw_arduino_build/py/builder_test.py
index 60eb144..c911e89 100644
--- a/pw_arduino_build/py/builder_test.py
+++ b/pw_arduino_build/py/builder_test.py
@@ -15,7 +15,7 @@
import shlex
import unittest
-from parameterized import parameterized
+from parameterized import parameterized # type: ignore
class TestShellArgumentSplitting(unittest.TestCase):
diff --git a/pw_arduino_build/py/file_operations_test.py b/pw_arduino_build/py/file_operations_test.py
index 7306fbb..9d1fe03 100644
--- a/pw_arduino_build/py/file_operations_test.py
+++ b/pw_arduino_build/py/file_operations_test.py
@@ -18,7 +18,7 @@
import tempfile
import unittest
from pathlib import Path
-from parameterized import parameterized
+from parameterized import parameterized # type: ignore
import pw_arduino_build.file_operations as file_operations
diff --git a/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py b/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py
index 24ae655..3457f18 100755
--- a/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py
+++ b/pw_arduino_build/py/pw_arduino_build/unit_test_runner.py
@@ -23,7 +23,7 @@
import time
from typing import List
-import serial
+import serial # type: ignore
import pw_arduino_build.log
from pw_arduino_build import teensy_detector
from pw_arduino_build.file_operations import decode_file_json
diff --git a/pw_arduino_build/py/setup.py b/pw_arduino_build/py/setup.py
index 0c7afc6..7761fd9 100644
--- a/pw_arduino_build/py/setup.py
+++ b/pw_arduino_build/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_arduino_build"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_arduino_build',
diff --git a/pw_bloat/py/setup.py b/pw_bloat/py/setup.py
index 8827b83..29545a8 100644
--- a/pw_bloat/py/setup.py
+++ b/pw_bloat/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_bloat"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_bloat',
diff --git a/pw_build/BUILD.gn b/pw_build/BUILD.gn
index 70b083a..6044286 100644
--- a/pw_build/BUILD.gn
+++ b/pw_build/BUILD.gn
@@ -117,6 +117,10 @@
group("empty") {
}
+pool("pip_pool") {
+ depth = 1
+}
+
pw_doc_group("docs") {
sources = [ "docs.rst" ]
}
diff --git a/pw_build/py/setup.py b/pw_build/py/setup.py
index f9f104c..755b4c0 100644
--- a/pw_build/py/setup.py
+++ b/pw_build/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_build"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_build',
diff --git a/pw_build/python.gni b/pw_build/python.gni
index ce0b145..080a601 100644
--- a/pw_build/python.gni
+++ b/pw_build/python.gni
@@ -45,10 +45,12 @@
"pw_python_package requires 'setup' to point to a setup.py or " +
"pyproject.toml and setup.cfg file")
+ _python_deps = []
if (defined(invoker.python_deps)) {
- _python_deps = invoker.python_deps
- } else {
- _python_deps = []
+ foreach(dep, invoker.python_deps) {
+ # Use the fully qualified name so the subtarget can be appended as needed.
+ _python_deps += [ get_label_info(dep, "label_no_toolchain") ]
+ }
}
if (defined(invoker.sources)) {
@@ -65,6 +67,8 @@
_all_sources += _test_sources
+ assert(_all_sources != [], "At least one source or test must be provided")
+
# Get the directory of the setup files. All files must be in the same dir.
_setup_dirs = get_path_info(invoker.setup, "dir")
_setup_dir = _setup_dirs[0]
@@ -92,9 +96,8 @@
_package_target = ":$target_name"
- # TODO(pwbug/239): Add support for installing this package and dependencies
- # with correct dependency ordering in a virtual environment. The code
- # below is incomplete and untested.
+ # Install this Python package and its dependencies in the current Python
+ # environment.
pw_python_action("$target_name.install") {
module = "pip"
args = [
@@ -105,6 +108,9 @@
stamp = true
+ # Parallel pip installations don't work, so serialize pip invocations.
+ pool = "$dir_pw_build:pip_pool"
+
deps = [ _package_target ]
foreach(dep, _python_deps) {
deps += [ "$dep.install" ]
@@ -120,7 +126,7 @@
"--out_dir",
rebase_path(target_out_dir),
]
- args += rebase_path(invoker.sources)
+ args += rebase_path(_all_sources)
deps = [ _package_target ]
stamp = true
@@ -140,8 +146,8 @@
"--pretty",
"--show-error-codes",
"--color-output",
+ rebase_path(_setup_dir),
]
- args += rebase_path(_all_sources)
# Use this environment variable to force mypy to colorize output.
# See https://github.com/python/mypy/issues/7771
@@ -170,7 +176,7 @@
sources = _all_sources
- stamp = "$target_gen_dir/{{source_file_part}}.pylint.pw_pystamp"
+ stamp = "$target_gen_dir/{{source_target_relative}}.pylint.pw_pystamp"
# Run pylint from the source root so that pylint detects rcfiles (.pylintrc)
# in the source tree.
diff --git a/pw_cli/py/pw_cli/process.py b/pw_cli/py/pw_cli/process.py
index 825dc08..a3e686b 100644
--- a/pw_cli/py/pw_cli/process.py
+++ b/pw_cli/py/pw_cli/process.py
@@ -35,7 +35,8 @@
"""Information about a process executed in run_async."""
def __init__(self, process: 'asyncio.subprocess.Process',
output: Union[bytes, IO[bytes]]):
- self.returncode = process.returncode
+ assert process.returncode is not None
+ self.returncode: int = process.returncode
self.pid = process.pid
self._output = output
diff --git a/pw_cli/py/setup.py b/pw_cli/py/setup.py
index 94df754..d9a163b 100644
--- a/pw_cli/py/setup.py
+++ b/pw_cli/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_cli"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_cli',
diff --git a/pw_doctor/py/setup.py b/pw_doctor/py/setup.py
index 752f5e2..2c61210 100644
--- a/pw_doctor/py/setup.py
+++ b/pw_doctor/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""The pw_doctor package."""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_doctor',
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py b/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
index 730c123..572f4ab 100755
--- a/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
@@ -31,14 +31,14 @@
import base64
try:
- import httplib
+ import httplib # type: ignore
except ImportError:
- import http.client as httplib # type: ignore
+ import http.client as httplib # type: ignore[no-redef]
try:
- import urlparse # Python 2.
+ import urlparse # type: ignore
except ImportError:
- import urllib.parse as urlparse # type: ignore
+ import urllib.parse as urlparse # type: ignore[no-redef]
try:
SCRIPT_DIR = os.path.dirname(__file__)
diff --git a/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
index 0bb2d32..06cca9e 100644
--- a/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
+++ b/pw_env_setup/py/pw_env_setup/virtualenv_setup/__main__.py
@@ -19,7 +19,9 @@
# TODO(pwbug/67) switch back to 'from pw_env_setup import virtualenv_setup'.
# from pw_env_setup import virtualenv_setup
-import install as virtualenv_setup # pylint: disable=import-error
+# pylint: disable=import-error
+import install as virtualenv_setup # type: ignore
+# pylint: enable=import-error
def _main():
diff --git a/pw_env_setup/py/pw_env_setup/windows_env_start.py b/pw_env_setup/py/pw_env_setup/windows_env_start.py
index 46ce452..dc8c8f1 100644
--- a/pw_env_setup/py/pw_env_setup/windows_env_start.py
+++ b/pw_env_setup/py/pw_env_setup/windows_env_start.py
@@ -26,7 +26,7 @@
import os
import sys
-from colors import Color, enable_colors
+from colors import Color, enable_colors # type: ignore
_PIGWEED_BANNER = u'''
▒█████▄ █▓ ▄███▒ ▒█ ▒█ ░▓████▒ ░▓████▒ ▒▓████▄
diff --git a/pw_env_setup/py/setup.py b/pw_env_setup/py/setup.py
index a0beace..f178fa0 100644
--- a/pw_env_setup/py/setup.py
+++ b/pw_env_setup/py/setup.py
@@ -11,9 +11,9 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
-"""env_setup module definition for PyOxidizer."""
+"""pw_env_setup package definition."""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_env_setup',
diff --git a/pw_hdlc_lite/py/decode_test.py b/pw_hdlc_lite/py/decode_test.py
index 55c3f35..7924632 100755
--- a/pw_hdlc_lite/py/decode_test.py
+++ b/pw_hdlc_lite/py/decode_test.py
@@ -250,7 +250,7 @@
# Decode byte-by-byte
decoder = FrameDecoder()
- decoded_frames = []
+ decoded_frames: List[Frame] = []
for i in range(len(data)):
decoded_frames += decoder.process(data[i:i + 1])
diff --git a/pw_hdlc_lite/py/pw_hdlc_lite/rpc.py b/pw_hdlc_lite/py/pw_hdlc_lite/rpc.py
index fd7c733..aa67b8d 100644
--- a/pw_hdlc_lite/py/pw_hdlc_lite/rpc.py
+++ b/pw_hdlc_lite/py/pw_hdlc_lite/rpc.py
@@ -76,7 +76,7 @@
output.flush()
else:
_LOG.error('Unhandled frame for address %d: %s', frame.address,
- frame.data.decoder(errors='replace'))
+ frame.data.decode(errors='replace'))
_PathOrModule = Union[str, Path, ModuleType]
@@ -123,7 +123,7 @@
daemon=True,
args=(self.client, device, output)).start()
- def rpcs(self, channel_id: int = None) -> pw_rpc.client.Services:
+ def rpcs(self, channel_id: int = None) -> Any:
"""Returns object for accessing services on the specified channel.
This skips some intermediate layers to make it simpler to invoke RPCs
diff --git a/pw_hdlc_lite/py/pw_hdlc_lite/rpc_console.py b/pw_hdlc_lite/py/pw_hdlc_lite/rpc_console.py
index 64ce81a..0ebeffe 100644
--- a/pw_hdlc_lite/py/pw_hdlc_lite/rpc_console.py
+++ b/pw_hdlc_lite/py/pw_hdlc_lite/rpc_console.py
@@ -37,8 +37,8 @@
import sys
from typing import Collection, Iterable, Iterator, BinaryIO
-import IPython
-import serial
+import IPython # type: ignore
+import serial # type: ignore
from pw_hdlc_lite.rpc import HdlcRpcClient
diff --git a/pw_hdlc_lite/py/setup.py b/pw_hdlc_lite/py/setup.py
index dad879d..ddb2075 100644
--- a/pw_hdlc_lite/py/setup.py
+++ b/pw_hdlc_lite/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_hdlc_lite"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_hdlc_lite',
diff --git a/pw_module/py/setup.py b/pw_module/py/setup.py
index 46e1ab1..a43fba3 100644
--- a/pw_module/py/setup.py
+++ b/pw_module/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_module"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_module',
diff --git a/pw_presubmit/py/pw_presubmit/presubmit.py b/pw_presubmit/py/pw_presubmit/presubmit.py
index f0231b2..51ef1f3 100644
--- a/pw_presubmit/py/pw_presubmit/presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/presubmit.py
@@ -49,8 +49,8 @@
import re
import subprocess
import time
-from typing import Callable, Collection, Dict, Iterable, Iterator, List
-from typing import NamedTuple, Optional, Pattern, Sequence, Set, Tuple
+from typing import (Callable, Collection, Dict, Iterable, Iterator, List,
+ NamedTuple, Optional, Pattern, Sequence, Set, Tuple, Union)
from pw_presubmit import git_repo, tools
from pw_presubmit.tools import plural
@@ -537,7 +537,7 @@
def filter_paths(endswith: Iterable[str] = (''),
- exclude: Iterable[str] = (),
+ exclude: Iterable[Union[Pattern[str], str]] = (),
always_run: bool = False) -> Callable[[Callable], _Check]:
"""Decorator for filtering the paths list for a presubmit check function.
diff --git a/pw_presubmit/py/setup.py b/pw_presubmit/py/setup.py
index 855082c..439d8eb 100644
--- a/pw_presubmit/py/setup.py
+++ b/pw_presubmit/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""The pw_presubmit package."""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_presubmit',
diff --git a/pw_protobuf/py/setup.py b/pw_protobuf/py/setup.py
index 7134f1a..d1d617c 100644
--- a/pw_protobuf/py/setup.py
+++ b/pw_protobuf/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_protobuf"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_protobuf',
diff --git a/pw_protobuf_compiler/py/setup.py b/pw_protobuf_compiler/py/setup.py
index ebee3fd..12edebb 100644
--- a/pw_protobuf_compiler/py/setup.py
+++ b/pw_protobuf_compiler/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_protobuf_compiler"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_protobuf_compiler',
diff --git a/pw_rpc/py/setup.py b/pw_rpc/py/setup.py
index b1cba53..55e4515 100644
--- a/pw_rpc/py/setup.py
+++ b/pw_rpc/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_rpc"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_rpc',
diff --git a/pw_status/py/setup.py b/pw_status/py/setup.py
index 45387d7..eba46d6 100644
--- a/pw_status/py/setup.py
+++ b/pw_status/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_status"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_status',
diff --git a/pw_tokenizer/py/setup.py b/pw_tokenizer/py/setup.py
index 9d9a96e..1520e50 100644
--- a/pw_tokenizer/py/setup.py
+++ b/pw_tokenizer/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""The pw_tokenizer package."""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_tokenizer',
diff --git a/pw_trace/py/setup.py b/pw_trace/py/setup.py
index 65e6126..a1371e5 100644
--- a/pw_trace/py/setup.py
+++ b/pw_trace/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""The pw_trace package."""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_trace',
diff --git a/pw_trace_tokenized/py/setup.py b/pw_trace_tokenized/py/setup.py
index fb7cc07..3266c84 100644
--- a/pw_trace_tokenized/py/setup.py
+++ b/pw_trace_tokenized/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""The pw_trace_tokenized package."""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_trace_tokenized',
diff --git a/pw_unit_test/py/setup.py b/pw_unit_test/py/setup.py
index bfe6b39..864b254 100644
--- a/pw_unit_test/py/setup.py
+++ b/pw_unit_test/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_unit_test"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_unit_test',
diff --git a/pw_watch/py/pw_watch/watch.py b/pw_watch/py/pw_watch/watch.py
index aae623e..2b948ed 100755
--- a/pw_watch/py/pw_watch/watch.py
+++ b/pw_watch/py/pw_watch/watch.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python
# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
@@ -25,10 +26,9 @@
import threading
from typing import List, NamedTuple, Optional, Sequence, Tuple
-from watchdog.events import FileSystemEventHandler
-from watchdog.observers import Observer
-from watchdog.utils import has_attribute
-from watchdog.utils import unicode_paths
+from watchdog.events import FileSystemEventHandler # type: ignore
+from watchdog.observers import Observer # type: ignore
+from watchdog.utils import has_attribute, unicode_paths # type: ignore
import pw_cli.branding
import pw_cli.color
diff --git a/pw_watch/py/setup.py b/pw_watch/py/setup.py
index f099427..4f5adfc 100644
--- a/pw_watch/py/setup.py
+++ b/pw_watch/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""pw_watch"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='pw_watch',
diff --git a/targets/lm3s6965evb-qemu/py/setup.py b/targets/lm3s6965evb-qemu/py/setup.py
index a60dc3f..a297f52 100644
--- a/targets/lm3s6965evb-qemu/py/setup.py
+++ b/targets/lm3s6965evb-qemu/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""lm3s6965evb_qemu_utils"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='lm3s6965evb_qemu_utils',
diff --git a/targets/stm32f429i-disc1/py/setup.py b/targets/stm32f429i-disc1/py/setup.py
index 1c47928..ffbc1d0 100644
--- a/targets/stm32f429i-disc1/py/setup.py
+++ b/targets/stm32f429i-disc1/py/setup.py
@@ -13,7 +13,7 @@
# the License.
"""stm32f429i_disc1_utils"""
-import setuptools
+import setuptools # type: ignore
setuptools.setup(
name='stm32f429i_disc1_utils',