pw_presubmit: Enable mypy and get it passing
- Fix several typing issues.
- Disable type checking in several places where mypy wasn't working
correctly.
- Enable mypy.
- Execute individual steps in the same order as they are provided with
--step.
Change-Id: I229cf8ee39a4db5067c1923b4acfc5fcd164f733
diff --git a/pw_build/py/exec.py b/pw_build/py/exec.py
index f9c5439..e30840e 100644
--- a/pw_build/py/exec.py
+++ b/pw_build/py/exec.py
@@ -132,7 +132,7 @@
else:
output_args = {}
- process = subprocess.run(command, env=env, **output_args)
+ process = subprocess.run(command, env=env, **output_args) # type: ignore
if process.returncode != 0 and args.capture_output:
_LOG.error('')
diff --git a/pw_build/py/python_runner.py b/pw_build/py/python_runner.py
index 41fab8a..62061b6 100755
--- a/pw_build/py/python_runner.py
+++ b/pw_build/py/python_runner.py
@@ -40,7 +40,7 @@
# have unintended consequences. This script shouldn't have to exist--GN should
# standardize a way of finding a compiled binary for a build target.
def _resembles_internal_gn_windows_path(path: str) -> bool:
- return os.name == 'nt' and re.match(r'^/[a-zA-Z]:[/\\]', path)
+ return os.name == 'nt' and bool(re.match(r'^/[a-zA-Z]:[/\\]', path))
def _fix_windows_absolute_path(path: str) -> str:
diff --git a/pw_cli/py/pw_cli/__main__.py b/pw_cli/py/pw_cli/__main__.py
index e95c7d8..cf43c32 100644
--- a/pw_cli/py/pw_cli/__main__.py
+++ b/pw_cli/py/pw_cli/__main__.py
@@ -25,6 +25,7 @@
import logging
import importlib
import pkgutil
+from typing import NoReturn
from pw_cli.color import colors
import pw_cli.log
@@ -41,7 +42,7 @@
class ArgumentParser(argparse.ArgumentParser):
- def error(self, message: str) -> None:
+ def error(self, message: str) -> NoReturn:
print(colors().magenta(_PIGWEED_BANNER), file=sys.stderr)
self.print_usage(sys.stderr)
self.exit(2, '%s: error: %s\n' % (self.prog, message))
diff --git a/pw_cli/py/pw_cli/color.py b/pw_cli/py/pw_cli/color.py
index b5ca84b..6559def 100644
--- a/pw_cli/py/pw_cli/color.py
+++ b/pw_cli/py/pw_cli/color.py
@@ -66,7 +66,7 @@
if enabled and os.name == 'nt':
# Enable ANSI color codes in Windows cmd.exe.
- kernel32 = ctypes.windll.kernel32
+ kernel32 = ctypes.windll.kernel32 # type: ignore
kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7)
return _Color() if enabled else _NoColor()
diff --git a/pw_cli/py/pw_cli/envparse.py b/pw_cli/py/pw_cli/envparse.py
index da38fe3..c65330b 100644
--- a/pw_cli/py/pw_cli/envparse.py
+++ b/pw_cli/py/pw_cli/envparse.py
@@ -77,7 +77,9 @@
def add_var(
self,
name: str,
- type: TypeConversion[T] = str, # pylint: disable=redefined-builtin
+ # pylint: disable=redefined-builtin
+ type: TypeConversion[T] = str, # type: ignore
+ # pylint: enable=redefined-builtin
default: Optional[T] = None,
) -> None:
"""Registers an environment variable.
@@ -95,7 +97,10 @@
raise ValueError(
f'Variable {name} does not have prefix {self._prefix}')
- self._variables[name] = VariableDescriptor(name, type, default)
+ self._variables[name] = VariableDescriptor(
+ name,
+ type, # type: ignore
+ default) # type: ignore
def parse_env(self,
env: Optional[Mapping[str, str]] = None) -> EnvNamespace:
diff --git a/pw_cli/py/pw_cli/plugins.py b/pw_cli/py/pw_cli/plugins.py
index dfde792..f2f0d3e 100644
--- a/pw_cli/py/pw_cli/plugins.py
+++ b/pw_cli/py/pw_cli/plugins.py
@@ -15,11 +15,10 @@
import argparse
import logging
-from typing import Callable
-from typing import NamedTuple
+from typing import Any, Callable, NamedTuple
_LOG = logging.getLogger(__name__)
-DefineArgsFunction = Callable[[argparse.ArgumentParser], None]
+DefineArgsFunction = Callable[[argparse.ArgumentParser], Any]
class Plugin(NamedTuple):
diff --git a/pw_cli/py/pw_cli/process.py b/pw_cli/py/pw_cli/process.py
index 7467df8..cfb2d63 100644
--- a/pw_cli/py/pw_cli/process.py
+++ b/pw_cli/py/pw_cli/process.py
@@ -29,21 +29,24 @@
PW_SUBPROCESS_ENV = 'PW_SUBPROCESS'
-async def run_async(*args: str, silent: bool = False) -> int:
+async def run_async(program: str, *args: str, silent: bool = False) -> int:
"""Runs a command, capturing and logging its output.
Returns the exit status of the command.
"""
- command = args[0]
- _LOG.debug('Running `%s`', shlex.join(command))
+ _LOG.debug('Running `%s`', shlex.join([program, *args]))
env = os.environ.copy()
env[PW_SUBPROCESS_ENV] = '1'
stdout = asyncio.subprocess.DEVNULL if silent else asyncio.subprocess.PIPE
process = await asyncio.create_subprocess_exec(
- *command, stdout=stdout, stderr=asyncio.subprocess.STDOUT, env=env)
+ program,
+ *args,
+ stdout=stdout,
+ stderr=asyncio.subprocess.STDOUT,
+ env=env)
if process.stdout is not None:
while True:
@@ -57,13 +60,13 @@
status = await process.wait()
if status == 0:
- _LOG.info('%s exited successfully', command[0])
+ _LOG.info('%s exited successfully', program)
else:
- _LOG.error('%s exited with status %d', command[0], status)
+ _LOG.error('%s exited with status %d', program, status)
return status
-def run(*args: str, silent: bool = False) -> int:
+def run(program: str, *args: str, silent: bool = False) -> int:
"""Synchronous wrapper for run_async."""
- return asyncio.run(run_async(args, silent))
+ return asyncio.run(run_async(program, *args, silent=silent))
diff --git a/pw_doctor/py/pw_doctor/doctor.py b/pw_doctor/py/pw_doctor/doctor.py
index f18c5a8..8c47739 100755
--- a/pw_doctor/py/pw_doctor/doctor.py
+++ b/pw_doctor/py/pw_doctor/doctor.py
@@ -23,6 +23,7 @@
import subprocess
import sys
import tempfile
+from typing import Callable, List
def call_stdout(*args, **kwargs):
@@ -74,7 +75,7 @@
return decorate
-CHECKS = []
+CHECKS: List[Callable] = []
@register_into(CHECKS)
diff --git a/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py b/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
index ac8c87d..f656a55 100755
--- a/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
+++ b/pw_env_setup/py/pw_env_setup/cipd_setup/wrapper.py
@@ -31,12 +31,12 @@
try:
import httplib
except ImportError:
- import http.client as httplib
+ import http.client as httplib # type: ignore
try:
import urlparse # Python 2.
except ImportError:
- import urllib.parse as urlparse
+ import urllib.parse as urlparse # type: ignore
SCRIPT_DIR = os.path.dirname(__file__)
VERSION_FILE = os.path.join(SCRIPT_DIR, '.cipd_version')
@@ -54,7 +54,7 @@
stderr=outs,
).strip().decode('utf-8')
except subprocess.CalledProcessError:
- PW_ROOT = None
+ PW_ROOT = ''
# Get default install dir from environment since args cannot always be passed
# through this script (args are passed as-is to cipd).
@@ -63,7 +63,7 @@
elif PW_ROOT:
DEFAULT_INSTALL_DIR = os.path.join(PW_ROOT, '.cipd')
else:
- DEFAULT_INSTALL_DIR = None
+ DEFAULT_INSTALL_DIR = ''
def platform_normalized():
diff --git a/pw_env_setup/py/pw_env_setup/env_setup.py b/pw_env_setup/py/pw_env_setup/env_setup.py
index f6c1caf..f99d8d3 100755
--- a/pw_env_setup/py/pw_env_setup/env_setup.py
+++ b/pw_env_setup/py/pw_env_setup/env_setup.py
@@ -54,7 +54,9 @@
filename = __file__
else:
# Try introspection in environments where __file__ is not populated.
- filename = inspect.getfile(inspect.currentframe())
+ frame = inspect.currentframe()
+ if frame is not None:
+ filename = inspect.getfile(frame)
# If none of our strategies worked, the imports are going to fail.
if filename is None:
raise
diff --git a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
index 0ce326d..497a6b2 100755
--- a/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
+++ b/pw_presubmit/py/pw_presubmit/pigweed_presubmit.py
@@ -1,6 +1,6 @@
#!/usr/bin/env python3
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
@@ -244,14 +244,26 @@
@filter_paths(endswith='.py', exclude=r'(?:.+/)?setup\.py')
def mypy(ctx: PresubmitContext):
- run_python_module('mypy', *ctx.paths)
+ env = os.environ.copy()
+ # Use this environment variable to force mypy to colorize output.
+ # See https://github.com/python/mypy/issues/7771
+ env['MYPY_FORCE_COLOR'] = '1'
+
+ run_python_module(
+ 'mypy',
+ *ctx.paths,
+ '--pretty',
+ '--color-output',
+ # TODO(pwbug/146): Some imports from installed packages fail. These
+ # imports should be fixed and this option removed.
+ '--ignore-missing-imports',
+ env=env)
PYTHON = (
test_python_packages,
pylint,
- # TODO(hepler): Enable mypy when it passes.
- # mypy,
+ mypy,
)
@@ -269,13 +281,12 @@
'CMakeLists.txt'))
def cmake_tests(ctx: PresubmitContext):
env = _env_with_clang_cc_vars()
- output = ctx.output_directory.joinpath('cmake-host')
call('cmake',
- '-B', output,
+ '-B', ctx.output_directory,
'-S', ctx.repository_root,
'-G', 'Ninja',
env=env) # yapf: disable
- call('ninja', '-C', output, 'pw_run_tests.modules', env=env)
+ ninja('pw_run_tests.modules', ctx=ctx)
CMAKE: Tuple[Callable, ...] = ()
@@ -492,7 +503,7 @@
'quick': QUICK_PRESUBMIT,
}
-ALL_STEPS = frozenset(itertools.chain(*PROGRAMS.values()))
+ALL_STEPS = {c.__name__: c for c in itertools.chain(*PROGRAMS.values())}
def argument_parser(parser=None) -> argparse.ArgumentParser:
@@ -534,7 +545,8 @@
exclusive.add_argument(
'--step',
- choices=sorted(x.__name__ for x in itertools.chain(ALL_STEPS)),
+ dest='steps',
+ choices=sorted(ALL_STEPS),
action='append',
help='Provide explicit steps instead of running a predefined program.',
)
@@ -551,7 +563,7 @@
install: bool,
repository: Path,
output_directory: Path,
- step: Sequence[str],
+ steps: Sequence[str],
**presubmit_args,
) -> int:
"""Entry point for presubmit."""
@@ -577,8 +589,8 @@
return 0
program = PROGRAMS[program_name]
- if step:
- program = [x for x in ALL_STEPS if x.__name__ in step]
+ if steps:
+ program = [ALL_STEPS[name] for name in steps]
if pw_presubmit.run_presubmit(program,
repository=repository,
diff --git a/pw_presubmit/py/pw_presubmit/tools.py b/pw_presubmit/py/pw_presubmit/tools.py
index 75114cd..3d05bf0 100644
--- a/pw_presubmit/py/pw_presubmit/tools.py
+++ b/pw_presubmit/py/pw_presubmit/tools.py
@@ -1,4 +1,4 @@
-# Copyright 2019 The Pigweed Authors
+# Copyright 2020 The Pigweed Authors
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
@@ -668,7 +668,7 @@
**kwargs)
logfunc = _LOG.warning if process.returncode else _LOG.debug
- logfunc('[FINISHED] %s\n%s', attributes, command)
+ logfunc('[FINISHED]\n%s', command)
logfunc('[RESULT] %s with return code %d',
'Failed' if process.returncode else 'Passed', process.returncode)
diff --git a/pw_protobuf/py/pw_protobuf/proto_structures.py b/pw_protobuf/py/pw_protobuf/proto_structures.py
index b0bcaf6..f9bcdf0 100644
--- a/pw_protobuf/py/pw_protobuf/proto_structures.py
+++ b/pw_protobuf/py/pw_protobuf/proto_structures.py
@@ -49,7 +49,7 @@
"""The type of the node."""
def children(self) -> List['ProtoNode']:
- return self._children.values()
+ return list(self._children.values())
def name(self) -> str:
return self._name
@@ -81,6 +81,7 @@
second = self
while diff > 0:
+ assert second is not None
second = second.parent()
diff -= 1
@@ -116,7 +117,7 @@
(child.type(), self.type()))
# pylint: disable=protected-access
- if child.parent() is not None:
+ if child._parent is not None:
del child._parent._children[child.name()]
child._parent = self
@@ -129,7 +130,7 @@
# pylint: disable=protected-access
for section in path.split('.'):
- node = node._children.get(section)
+ node = node._children[section]
if node is None:
return None
# pylint: enable=protected-access
diff --git a/pw_tokenizer/py/pw_tokenizer/database.py b/pw_tokenizer/py/pw_tokenizer/database.py
index 4e35a9d..52d102f 100755
--- a/pw_tokenizer/py/pw_tokenizer/database.py
+++ b/pw_tokenizer/py/pw_tokenizer/database.py
@@ -29,7 +29,7 @@
from typing import Dict, Iterable
try:
- from pw_presubmit import elf_reader, tokens
+ from pw_tokenizer import elf_reader, tokens
except ImportError:
# Append this path to the module search path to allow running this module
# without installing the pw_tokenizer package.
diff --git a/pw_tokenizer/py/pw_tokenizer/elf_reader.py b/pw_tokenizer/py/pw_tokenizer/elf_reader.py
index 76f51e3..6eec96c 100755
--- a/pw_tokenizer/py/pw_tokenizer/elf_reader.py
+++ b/pw_tokenizer/py/pw_tokenizer/elf_reader.py
@@ -364,11 +364,12 @@
section_parser = subparsers.add_parser('section')
section_parser.set_defaults(handler=_dump_sections)
- section_parser.add_argument('sections',
- metavar='section_regex',
- nargs='*',
- type=re.compile,
- help='section name regular expression')
+ section_parser.add_argument(
+ 'sections',
+ metavar='section_regex',
+ nargs='*',
+ type=re.compile, # type: ignore
+ help='section name regular expression')
address_parser = subparsers.add_parser('address')
address_parser.set_defaults(handler=_read_addresses)
diff --git a/pw_unit_test/py/pw_unit_test/test_runner.py b/pw_unit_test/py/pw_unit_test/test_runner.py
index 46f9987..53eeeb7 100644
--- a/pw_unit_test/py/pw_unit_test/test_runner.py
+++ b/pw_unit_test/py/pw_unit_test/test_runner.py
@@ -138,7 +138,7 @@
_LOG.info('%s: [ RUN] %s', test_counter, test.name)
command = [self._executable, test.file_path, *self._args]
try:
- status = await pw_cli.process.run_async(command)
+ status = await pw_cli.process.run_async(*command)
if status == 0:
test.status = TestResult.SUCCESS
test_result = 'PASS'
diff --git a/pw_watch/py/pw_watch/watch.py b/pw_watch/py/pw_watch/watch.py
index 0613aa8..05817b5 100755
--- a/pw_watch/py/pw_watch/watch.py
+++ b/pw_watch/py/pw_watch/watch.py
@@ -130,7 +130,7 @@
# Track state of a build. These need to be members instead of locals
# due to the split between dispatch(), run(), and on_complete().
self.matching_path = None
- self.builds_succeeded = []
+ self.builds_succeeded: List[bool] = []
self.wait_for_keypress_thread = threading.Thread(
None, self._wait_for_enter)
diff --git a/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_runner.py b/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_runner.py
index 6ea239f..ce80e29 100755
--- a/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_runner.py
+++ b/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_runner.py
@@ -31,8 +31,9 @@
_OPENOCD_CONFIG = os.path.join(_DIR, 'openocd_stm32f4xx.cfg')
# Path to scripts provided by openocd.
-_OPENOCD_SCRIPTS_DIR = os.path.join(os.getenv('PW_PIGWEED_CIPD_INSTALL_DIR'),
- 'share', 'openocd', 'scripts')
+_OPENOCD_SCRIPTS_DIR = os.path.join(
+ os.getenv('PW_PIGWEED_CIPD_INSTALL_DIR', ''), 'share', 'openocd',
+ 'scripts')
_LOG = logging.getLogger('unit_test_runner')
diff --git a/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_server.py b/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_server.py
index 564928f..a7e6a5f 100644
--- a/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_server.py
+++ b/targets/stm32f429i-disc1/py/stm32f429i_disc1_utils/unit_test_server.py
@@ -18,7 +18,7 @@
import logging
import sys
import tempfile
-from typing import List, Optional, TextIO
+from typing import IO, List, Optional
import pw_cli.process
import pw_cli.log
@@ -63,7 +63,7 @@
return '\n'.join(runner)
-def generate_server_config() -> TextIO:
+def generate_server_config() -> IO[bytes]:
"""Returns a temporary generated file for use as the server config."""
boards = stm32f429i_detector.detect_boards()
if not boards:
@@ -83,7 +83,7 @@
return config_file
-def launch_server(server_config: Optional[TextIO],
+def launch_server(server_config: Optional[IO[bytes]],
server_port: Optional[int]) -> int:
"""Launch a device test server with the provided arguments."""
if server_config is None: