blob: 50d247fb4ff38cf631c29ace2eddec34ad6fd843 [file] [log] [blame]
Wyatt Hepleraf853ea2021-03-09 19:09:54 -08001# Copyright 2021 The Pigweed Authors
Wyatt Heplerb16dfd32020-10-12 08:46:38 -07002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14
15import("//build_overrides/pigweed.gni")
16
Wyatt Heplerb8e13602020-10-19 11:20:36 -070017import("$dir_pw_build/input_group.gni")
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080018import("$dir_pw_build/mirror_tree.gni")
Wyatt Heplerb8e13602020-10-19 11:20:36 -070019import("$dir_pw_build/python_action.gni")
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070020import("$dir_pw_protobuf_compiler/toolchain.gni")
21
22declare_args() {
23 # Python tasks, such as running tests and Pylint, are done in a single GN
24 # toolchain to avoid unnecessary duplication in the build.
25 pw_build_PYTHON_TOOLCHAIN = "$dir_pw_build/python_toolchain:python"
26}
Wyatt Heplerb16dfd32020-10-12 08:46:38 -070027
Wyatt Hepler438caa02021-01-15 17:13:11 -080028# Python packages provide the following targets as $target_name.$subtarget.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080029pw_python_package_subtargets = [
Wyatt Hepler438caa02021-01-15 17:13:11 -080030 "tests",
31 "lint",
32 "lint.mypy",
33 "lint.pylint",
34 "install",
35 "wheel",
Wyatt Hepler446f24c2021-03-24 09:29:35 -070036
37 # Internal targets that directly depend on one another.
38 "_run_pip_install",
39 "_build_wheel",
Wyatt Hepler438caa02021-01-15 17:13:11 -080040]
41
Wyatt Heplerb85cda42021-04-07 13:13:15 -070042# Create aliases for subsargets when the target name matches the directory name.
43# This allows //foo:foo.tests to be accessed as //foo:tests, for example.
44template("_pw_create_aliases_if_name_matches_directory") {
45 not_needed([ "invoker" ])
46
47 if (get_label_info(":$target_name", "name") ==
48 get_path_info(get_label_info(":$target_name", "dir"), "name")) {
49 foreach(subtarget, pw_python_package_subtargets) {
50 group(subtarget) {
51 public_deps = [ ":${invoker.target_name}.$subtarget" ]
52 }
53 }
54 }
55}
56
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070057# Internal template that runs Mypy.
58template("_pw_python_static_analysis_mypy") {
59 pw_python_action(target_name) {
60 module = "mypy"
61 args = [
62 "--pretty",
63 "--show-error-codes",
64 ]
65
66 if (defined(invoker.mypy_ini)) {
67 args += [ "--config-file=" + rebase_path(invoker.mypy_ini) ]
68 inputs = [ invoker.mypy_ini ]
69 }
70
71 args += rebase_path(invoker.sources)
72
73 # Use this environment variable to force mypy to colorize output.
74 # See https://github.com/python/mypy/issues/7771
75 environment = [ "MYPY_FORCE_COLOR=1" ]
76
77 directory = invoker.directory
78 stamp = true
79
80 deps = invoker.deps
81
82 foreach(dep, invoker.python_deps) {
83 deps += [ string_replace(dep, "(", ".lint.mypy(") ]
84 }
85 }
86}
87
88# Internal template that runs Pylint.
89template("_pw_python_static_analysis_pylint") {
90 # Create a target to run pylint on each of the Python files in this
91 # package and its dependencies.
92 pw_python_action_foreach(target_name) {
93 module = "pylint"
94 args = [
95 rebase_path(".") + "/{{source_target_relative}}",
96 "--jobs=1",
97 "--output-format=colorized",
98 ]
99
100 if (defined(invoker.pylintrc)) {
101 args += [ "--rcfile=" + rebase_path(invoker.pylintrc) ]
102 inputs = [ invoker.pylintrc ]
103 }
104
105 if (host_os == "win") {
106 # Allow CRLF on Windows, in case Git is set to switch line endings.
107 args += [ "--disable=unexpected-line-ending-format" ]
108 }
109
110 sources = invoker.sources
111 directory = invoker.directory
112
113 stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"
114
115 public_deps = invoker.deps
116
117 foreach(dep, invoker.python_deps) {
118 public_deps += [ string_replace(dep, "(", ".lint.pylint(") ]
119 }
120 }
121}
122
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700123# Defines a Python package. GN Python packages contain several GN targets:
124#
125# - $name - Provides the Python files in the build, but does not take any
126# actions. All subtargets depend on this target.
127# - $name.lint - Runs static analyis tools on the Python code. This is a group
128# of two subtargets:
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800129# - $name.lint.mypy - Runs mypy (if enabled).
130# - $name.lint.pylint - Runs pylint (if enabled).
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700131# - $name.tests - Runs all tests for this package.
Rob Mohrfcf60372020-11-04 11:41:36 -0800132# - $name.install - Installs the package in a venv.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800133# - $name.wheel - Builds a Python wheel for the package.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700134#
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700135# All Python packages are instantiated with in pw_build_PYTHON_TOOLCHAIN,
136# regardless of the current toolchain. This prevents Python-specific work, like
137# running Pylint, from occurring multiple times in a build.
Wyatt Hepler438caa02021-01-15 17:13:11 -0800138#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700139# Args:
140# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
141# which must all be in the same directory.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800142# generate_setup: As an alternative to 'setup', generate setup files with the
143# keywords in this scope. 'name' is required.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700144# sources: Python sources files in the package.
145# tests: Test files for this Python package.
146# python_deps: Dependencies on other pw_python_packages in the GN build.
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800147# python_test_deps: Test-only pw_python_package dependencies.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700148# other_deps: Dependencies on GN targets that are not pw_python_packages.
149# inputs: Other files to track, such as package_data.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800150# proto_library: A pw_proto_library target to embed in this Python package.
151# generate_setup is required in place of setup if proto_library is used.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700152# static_analysis: List of static analysis tools to run; "*" (default) runs
153# all tools. The supported tools are "mypy" and "pylint".
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800154# pylintrc: Optional path to a pylintrc configuration file to use. If not
155# provided, Pylint's default rcfile search is used. Pylint is executed
156# from the package's setup directory, so pylintrc files in that directory
157# will take precedence over others.
158# mypy_ini: Optional path to a mypy configuration file to use. If not
159# provided, mypy's default configuration file search is used. mypy is
160# executed from the package's setup directory, so mypy.ini files in that
161# directory will take precedence over others.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800162#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700163template("pw_python_package") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700164 # The Python targets are always instantiated in pw_build_PYTHON_TOOLCHAIN. Use
Wyatt Hepler752d7d32021-03-02 09:02:23 -0800165 # fully qualified labels so that the toolchain is not lost.
166 _other_deps = []
167 if (defined(invoker.other_deps)) {
168 foreach(dep, invoker.other_deps) {
169 _other_deps += [ get_label_info(dep, "label_with_toolchain") ]
170 }
171 }
172
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800173 _python_deps = []
174 if (defined(invoker.python_deps)) {
175 foreach(dep, invoker.python_deps) {
176 _python_deps += [ get_label_info(dep, "label_with_toolchain") ]
177 }
178 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700179
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700180 # pw_python_script uses pw_python_package, but with a limited set of features.
181 # _pw_standalone signals that this target is actually a pw_python_script.
182 _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone)
183
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800184 _generate_package = false
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800185
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800186 # Check the generate_setup and import_protos args to determine if this package
187 # is generated.
188 if (_is_package) {
189 assert(defined(invoker.generate_setup) != defined(invoker.setup),
190 "Either 'setup' or 'generate_setup' (but not both) must provided")
191
192 if (defined(invoker.proto_library)) {
193 assert(invoker.proto_library != "", "'proto_library' cannot be empty")
194 assert(defined(invoker.generate_setup),
195 "Python packages that import protos with 'proto_library' must " +
196 "use 'generate_setup' instead of 'setup'")
197
198 _import_protos = [ invoker.proto_library ]
199 } else if (defined(invoker.generate_setup)) {
200 _import_protos = []
201 }
202
203 if (defined(invoker.generate_setup)) {
204 _generate_package = true
205 _setup_dir = "$target_gen_dir/$target_name.generated_python_package"
206
207 if (defined(invoker.strip_prefix)) {
208 _source_root = invoker.strip_prefix
209 } else {
210 _source_root = "."
211 }
212 } else {
213 # Non-generated packages with sources provided need an __init__.py.
214 assert(!defined(invoker.sources) || invoker.sources == [] ||
215 filter_include(invoker.sources, [ "*\b__init__.py" ]) != [],
216 "Python packages must have at least one __init__.py file")
217
218 # Get the directories of the setup files. All must be in the same dir.
219 _setup_dirs = get_path_info(invoker.setup, "dir")
220 _setup_dir = _setup_dirs[0]
221
222 foreach(dir, _setup_dirs) {
223 assert(dir == _setup_dir,
224 "All files in 'setup' must be in the same directory")
225 }
226
227 assert(!defined(invoker.strip_prefix),
228 "'strip_prefix' may only be given if 'generate_setup' is provided")
229 }
230 }
231
232 # Process arguments defaults and set defaults.
233
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700234 _supported_static_analysis_tools = [
235 "mypy",
236 "pylint",
237 ]
238 not_needed([ "_supported_static_analysis_tools" ])
239
240 # Argument: static_analysis (list of tool names or "*"); default = "*" (all)
241 if (!defined(invoker.static_analysis) || invoker.static_analysis == "*") {
242 _static_analysis = _supported_static_analysis_tools
Keir Mierlee65ddc82020-12-02 17:59:52 -0800243 } else {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700244 _static_analysis = invoker.static_analysis
245 }
246
247 # TODO(hepler): Remove support for the lint option.
248 if (defined(invoker.lint)) {
249 assert(!defined(invoker.static_analysis),
250 "'lint' is deprecated; use 'static_analysis' instead")
251
252 # Only allow 'lint = false', for backwards compatibility.
253 assert(invoker.lint == false, "'lint' is deprecated; use 'static_analysis'")
254 print("WARNING:",
255 "The 'lint' option for pw_python_package is deprecated.",
256 "Instead, use 'static_analysis = []' to disable linting.")
257 _static_analysis = []
258 }
259
260 foreach(_tool, _static_analysis) {
261 assert(_supported_static_analysis_tools + [ _tool ] - [ _tool ] !=
262 _supported_static_analysis_tools,
263 "'$_tool' is not a supported static analysis tool")
Keir Mierlee65ddc82020-12-02 17:59:52 -0800264 }
265
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800266 # Argument: sources (list)
267 _sources = []
268 if (defined(invoker.sources)) {
269 if (_generate_package) {
270 foreach(source, rebase_path(invoker.sources, _source_root)) {
271 _sources += [ "$_setup_dir/$source" ]
272 }
273 } else {
274 _sources += invoker.sources
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700275 }
276 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700277
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800278 # Argument: tests (list)
279 _test_sources = []
280 if (defined(invoker.tests)) {
281 if (_generate_package) {
282 foreach(source, rebase_path(invoker.tests, _source_root)) {
283 _test_sources += [ "$_setup_dir/$source" ]
284 }
285 } else {
286 _test_sources += invoker.tests
Wyatt Heplerd7dc6552020-10-21 16:16:21 -0700287 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700288 }
289
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800290 # Argument: setup (list)
291 _setup_sources = []
292 if (defined(invoker.setup)) {
293 _setup_sources = invoker.setup
294 } else if (_generate_package) {
295 _setup_sources = [ "$_setup_dir/setup.py" ]
296 }
297
298 # Argument: python_test_deps (list)
299 _python_test_deps = _python_deps # include all deps in test deps
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800300 if (defined(invoker.python_test_deps)) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800301 foreach(dep, invoker.python_test_deps) {
302 _python_test_deps += [ get_label_info(dep, "label_with_toolchain") ]
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800303 }
304 }
305
306 if (_test_sources == []) {
307 assert(!defined(invoker.python_test_deps),
308 "python_test_deps was provided, but there are no tests in " +
309 get_label_info(":$target_name", "label_no_toolchain"))
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800310 not_needed([ "_python_test_deps" ])
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800311 }
312
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800313 _all_py_files = _sources + _test_sources + _setup_sources
Wyatt Hepler438caa02021-01-15 17:13:11 -0800314
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700315 # The pw_python_package subtargets are only instantiated in
316 # pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the
317 # targets in this toolchain.
318 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800319 # Declare the main Python package group. This represents the Python files,
320 # but does not take any actions. GN targets can depend on the package name
321 # to run when any files in the package change.
322 if (_generate_package) {
323 # If this package is generated, mirror the sources to the final directory.
324 pw_mirror_tree("$target_name._mirror_sources_to_out_dir") {
325 directory = _setup_dir
Wyatt Hepler438caa02021-01-15 17:13:11 -0800326
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800327 sources = []
328 if (defined(invoker.sources)) {
329 sources += invoker.sources
330 }
331 if (defined(invoker.tests)) {
332 sources += invoker.tests
333 }
Wyatt Hepler438caa02021-01-15 17:13:11 -0800334
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800335 source_root = _source_root
336 public_deps = _python_deps + _other_deps
Wyatt Hepler285636f2020-12-15 13:13:11 -0800337 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700338
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800339 # Depend on the proto's _gen targets (from the default toolchain).
340 _gen_protos = []
341 foreach(proto, _import_protos) {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700342 _gen_protos += [ get_label_info(proto, "label_no_toolchain") +
343 ".python._gen($pw_protobuf_compiler_TOOLCHAIN)" ]
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700344 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700345
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800346 generated_file("$target_name._protos") {
347 deps = _gen_protos
348 data_keys = [ "protoc_outputs" ]
349 outputs = [ "$_setup_dir/protos.txt" ]
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700350 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800351
352 _protos_file = get_target_outputs(":${invoker.target_name}._protos")
353
354 generated_file("$target_name._protos_root") {
355 deps = _gen_protos
356 data_keys = [ "root" ]
357 outputs = [ "$_setup_dir/proto_root.txt" ]
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700358 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700359
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800360 _root_file = get_target_outputs(":${invoker.target_name}._protos_root")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700361
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800362 # Get generated_setup scope and write it to disk ask JSON.
363 _gen_setup = invoker.generate_setup
364 assert(defined(_gen_setup.name), "'name' is required in generate_package")
365 assert(!defined(_gen_setup.packages) && !defined(_gen_setup.package_data),
366 "'packages' and 'package_data' may not be provided " +
367 "in 'generate_package'")
368 write_file("$_setup_dir/setup.json", _gen_setup, "json")
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800369
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800370 # Generate the setup.py, py.typed, and __init__.py files as needed.
371 action(target_name) {
372 script = "$dir_pw_build/py/pw_build/generate_python_package.py"
373 args = [
374 "--label",
375 get_label_info(":$target_name", "label_no_toolchain"),
376 "--root",
377 rebase_path(_setup_dir),
378 "--setup-json",
379 rebase_path("$_setup_dir/setup.json"),
380 "--file-list",
381 rebase_path(_protos_file[0]),
382 "--file-list-root",
383 rebase_path(_root_file[0]),
384 ] + rebase_path(_sources)
385
386 if (defined(invoker._pw_module_as_package) &&
387 invoker._pw_module_as_package) {
388 args += [ "--module-as-package" ]
389 }
390
Wyatt Hepler0e1f5e42021-03-16 22:51:57 -0700391 inputs = [ "$_setup_dir/setup.json" ]
392
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800393 public_deps = [
394 ":$target_name._mirror_sources_to_out_dir",
395 ":$target_name._protos",
396 ":$target_name._protos_root",
397 ]
398
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700399 # Each pw_proto_library generates a file that indicates which Python
400 # package it is nested in, if any. Locate those files.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800401 foreach(proto, _import_protos) {
402 _tgt = get_label_info(proto, "label_no_toolchain")
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700403 _path = get_label_info("$_tgt($pw_protobuf_compiler_TOOLCHAIN)",
404 "target_gen_dir")
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800405 _name = get_label_info(_tgt, "name")
406
407 args += [
408 "--proto-library=$_tgt",
409 "--proto-library-file",
410 rebase_path("$_path/$_name.proto_library/python_package.txt"),
411 ]
412
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700413 public_deps +=
414 [ "$_tgt.python._gen($pw_protobuf_compiler_TOOLCHAIN)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800415 }
416
417 outputs = _setup_sources
418 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800419 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800420 # If the package is not generated, use an input group for the sources.
421 pw_input_group(target_name) {
422 inputs = _all_py_files
423 if (defined(invoker.inputs)) {
424 inputs += invoker.inputs
425 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800426
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800427 deps = _python_deps + _other_deps
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800428 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700429 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700430
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800431 if (_is_package) {
432 # Install this Python package and its dependencies in the current Python
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700433 # environment using pip.
434 pw_python_action("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800435 module = "pip"
436 public_deps = []
437
438 args = [ "install" ]
439
440 # For generated packages, reinstall when any files change. For regular
441 # packages, only reinstall when setup.py changes.
442 if (_generate_package) {
443 public_deps += [ ":${invoker.target_name}" ]
444 } else {
445 inputs = invoker.setup
446
447 # Install with --editable since the complete package is in source.
448 args += [ "--editable" ]
449 }
450
451 args += [ rebase_path(_setup_dir) ]
452
453 stamp = true
454
455 # Parallel pip installations don't work, so serialize pip invocations.
456 pool = "$dir_pw_build:pip_pool"
457
458 foreach(dep, _python_deps) {
459 # We need to add a suffix to the target name, but the label is
460 # formatted as "//path/to:target(toolchain)", so we can't just append
461 # ".subtarget". Instead, we replace the opening parenthesis of the
462 # toolchain with ".suffix(".
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700463 public_deps += [ string_replace(dep, "(", "._run_pip_install(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800464 }
465 }
466
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800467 # Builds a Python wheel for this package. Records the output directory
468 # in the pw_python_package_wheels metadata key.
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700469 pw_python_action("$target_name._build_wheel") {
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800470 metadata = {
471 pw_python_package_wheels = [ "$target_out_dir/$target_name" ]
472 }
473
474 module = "build"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800475
476 args = [
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800477 rebase_path(_setup_dir),
478 "--wheel",
479 "--no-isolation",
480 "--outdir",
481 ] + rebase_path(metadata.pw_python_package_wheels)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800482
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800483 deps = [ ":${invoker.target_name}" ]
484 foreach(dep, _python_deps) {
485 deps += [ string_replace(dep, "(", ".wheel(") ]
486 }
487
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800488 stamp = true
489 }
490 } else {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700491 # Stubs for non-package targets.
492 group("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800493 }
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700494 group("$target_name._build_wheel") {
495 }
496 }
497
498 # Create the .install and .wheel targets. To limit unnecessary pip
499 # executions, non-generated packages are only reinstalled when their
500 # setup.py changes. However, targets that depend on the .install subtarget
501 # re-run whenever any source files change.
502 #
503 # These targets just represent the source files if this isn't a package.
504 group("$target_name.install") {
505 public_deps = [ ":${invoker.target_name}" ]
506
507 if (_is_package) {
508 public_deps += [ ":${invoker.target_name}._run_pip_install" ]
509 }
510
511 foreach(dep, _python_deps) {
512 public_deps += [ string_replace(dep, "(", ".install(") ]
513 }
514 }
515
516 group("$target_name.wheel") {
517 public_deps = [ ":${invoker.target_name}.install" ]
518
519 if (_is_package) {
520 public_deps += [ ":${invoker.target_name}._build_wheel" ]
521 }
522
523 foreach(dep, _python_deps) {
524 public_deps += [ string_replace(dep, "(", ".wheel(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800525 }
526 }
527
528 # Define the static analysis targets for this package.
529 group("$target_name.lint") {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700530 deps = []
531 foreach(_tool, _supported_static_analysis_tools) {
532 deps += [ ":${invoker.target_name}.lint.$_tool" ]
533 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800534 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700535
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700536 if (_static_analysis != [] || _test_sources != []) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800537 # All packages to install for either general use or test running.
538 _test_install_deps = [ ":$target_name.install" ]
539 foreach(dep, _python_test_deps + [ "$dir_pw_build:python_lint" ]) {
540 _test_install_deps += [ string_replace(dep, "(", ".install(") ]
541 }
542 }
543
544 # For packages that are not generated, create targets to run mypy and pylint.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700545 foreach(_tool, _static_analysis) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800546 # Run lint tools from the setup or target directory so that the tools detect
547 # config files (e.g. pylintrc or mypy.ini) in that directory. Config files
548 # may be explicitly specified with the pylintrc or mypy_ini arguments.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700549 target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800550 sources = _all_py_files
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700551 deps = _test_install_deps
552 python_deps = _python_deps
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800553
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700554 if (defined(_setup_dir)) {
555 directory = rebase_path(_setup_dir)
556 } else {
557 directory = rebase_path(".")
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800558 }
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700559
560 _optional_variables = [
561 "mypy_ini",
562 "pylintrc",
563 ]
564 forward_variables_from(invoker, _optional_variables)
565 not_needed(_optional_variables)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800566 }
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700567 }
568
569 foreach(_unused_tool, _supported_static_analysis_tools - _static_analysis) {
570 pw_input_group("$target_name.lint.$_unused_tool") {
571 inputs = []
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800572 if (defined(invoker.pylintrc)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700573 inputs += [ invoker.pylintrc ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800574 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800575 if (defined(invoker.mypy_ini)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700576 inputs += [ invoker.mypy_ini ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800577 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800578 }
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800579
580 # Generated packages with linting disabled never need the whole file list.
581 not_needed([ "_all_py_files" ])
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700582 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800583 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800584 # Create groups with the public target names ($target_name, $target_name.lint,
585 # $target_name.install, etc.). These are actually wrappers around internal
586 # Python actions instantiated with the default toolchain. This ensures there
587 # is only a single copy of each Python action in the build.
588 #
589 # The $target_name.tests group is created separately below.
590 group("$target_name") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700591 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800592 }
593
594 foreach(subtarget, pw_python_package_subtargets - [ "tests" ]) {
595 group("$target_name.$subtarget") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700596 deps =
597 [ ":${invoker.target_name}.$subtarget($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800598 }
Wyatt Heplerb8e13602020-10-19 11:20:36 -0700599 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800600
601 # Everything Python-related is only instantiated in the default toolchain.
602 # Silence not-needed warnings except for in the default toolchain.
603 not_needed("*")
604 not_needed(invoker, "*")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700605 }
606
607 # Create a target for each test file.
608 _test_targets = []
609
610 foreach(test, _test_sources) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800611 if (_is_package) {
612 _name = rebase_path(test, _setup_dir)
613 } else {
614 _name = test
615 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700616
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800617 _test_target = "$target_name.tests." + string_replace(_name, "/", "_")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700618
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700619 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800620 pw_python_action(_test_target) {
621 script = test
622 stamp = true
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800623
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800624 deps = _test_install_deps
625
626 foreach(dep, _python_test_deps) {
627 deps += [ string_replace(dep, "(", ".tests(") ]
628 }
629 }
630 } else {
631 # Create a public version of each test target, so tests can be executed as
632 # //path/to:package.tests.foo.py.
633 group(_test_target) {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700634 deps = [ ":$_test_target($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700635 }
636 }
637
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800638 _test_targets += [ ":$_test_target" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700639 }
640
641 group("$target_name.tests") {
642 deps = _test_targets
643 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700644
645 _pw_create_aliases_if_name_matches_directory(target_name) {
646 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700647}
648
649# Declares a group of Python packages or other Python groups. pw_python_groups
650# expose the same set of subtargets as pw_python_package (e.g.
651# "$group_name.lint" and "$group_name.tests"), but these apply to all packages
652# in deps and their dependencies.
653template("pw_python_group") {
654 if (defined(invoker.python_deps)) {
655 _python_deps = invoker.python_deps
656 } else {
657 _python_deps = []
658 }
659
660 group(target_name) {
661 deps = _python_deps
662 }
663
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800664 foreach(subtarget, pw_python_package_subtargets) {
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700665 group("$target_name.$subtarget") {
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700666 public_deps = []
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700667 foreach(dep, _python_deps) {
668 # Split out the toolchain to support deps with a toolchain specified.
669 _target = get_label_info(dep, "label_no_toolchain")
670 _toolchain = get_label_info(dep, "toolchain")
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700671 public_deps += [ "$_target.$subtarget($_toolchain)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700672 }
673 }
674 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700675
676 _pw_create_aliases_if_name_matches_directory(target_name) {
677 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700678}
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700679
680# Declares Python scripts or tests that are not part of a Python package.
681# Similar to pw_python_package, but only supports a subset of its features.
682#
683# pw_python_script accepts the same arguments as pw_python_package, except
684# `setup` cannot be provided.
685#
686# pw_python_script provides the same subtargets as pw_python_package, but
687# $target_name.install and $target_name.wheel only affect the python_deps of
688# this GN target, not the target itself.
Wyatt Hepler473efe02021-05-04 14:15:16 -0700689#
690# pw_python_script allows creating a pw_python_action associated with the
691# script. This is provided by passing an 'action' scope to pw_python_script.
692# This functions like a normal action, with a few additions: the action uses the
693# pw_python_script's python_deps and defaults to using the source file as its
694# 'script' argument, if there is only a single source file.
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700695template("pw_python_script") {
Wyatt Hepler473efe02021-05-04 14:15:16 -0700696 _package_variables = [
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700697 "sources",
698 "tests",
699 "python_deps",
700 "other_deps",
701 "inputs",
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800702 "pylintrc",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700703 "mypy_ini",
704 "static_analysis",
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700705 ]
706
707 pw_python_package(target_name) {
708 _pw_standalone = true
Wyatt Hepler473efe02021-05-04 14:15:16 -0700709 forward_variables_from(invoker, _package_variables)
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700710 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700711
712 _pw_create_aliases_if_name_matches_directory(target_name) {
713 }
Wyatt Hepler473efe02021-05-04 14:15:16 -0700714
715 if (defined(invoker.action)) {
716 pw_python_action("$target_name.action") {
717 forward_variables_from(invoker.action, "*", [ "python_deps" ])
718 python_deps = [ ":${invoker.target_name}" ]
719
720 if (!defined(script) && !defined(module) && defined(invoker.sources)) {
721 _sources = invoker.sources
722 assert(_sources != [] && _sources == [ _sources[0] ],
723 "'script' must be specified unless there is only one source " +
724 "in 'sources'")
725 script = _sources[0]
726 }
727 }
728 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700729}
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800730
731# Represents a list of Python requirements, as in a requirements.txt.
732#
733# Args:
734# files: One or more requirements.txt files.
735# requirements: A list of requirements.txt-style requirements.
736template("pw_python_requirements") {
737 assert(defined(invoker.files) || defined(invoker.requirements),
738 "pw_python_requirements requires a list of requirements.txt files " +
739 "in the 'files' arg or requirements in 'requirements'")
740
741 _requirements_files = []
742
743 if (defined(invoker.files)) {
744 _requirements_files += invoker.files
745 }
746
747 if (defined(invoker.requirements)) {
748 _requirements_file = "$target_gen_dir/$target_name.requirements.txt"
749 write_file(_requirements_file, invoker.requirements)
750 _requirements_files += [ _requirements_file ]
751 }
752
753 # The default target represents the requirements themselves.
754 pw_input_group(target_name) {
755 inputs = _requirements_files
756 }
757
758 # Use the same subtargets as pw_python_package so these targets can be listed
759 # as python_deps of pw_python_packages.
760 pw_python_action("$target_name.install") {
761 inputs = _requirements_files
762
763 module = "pip"
764 args = [ "install" ]
765
766 foreach(_requirements_file, inputs) {
767 args += [
768 "--requirement",
769 rebase_path(_requirements_file),
770 ]
771 }
772
773 pool = "$dir_pw_build:pip_pool"
774 stamp = true
775 }
776
777 # Create stubs for the unused subtargets so that pw_python_requirements can be
778 # used as python_deps.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800779 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) {
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800780 group("$target_name.$subtarget") {
781 }
782 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700783
784 _pw_create_aliases_if_name_matches_directory(target_name) {
785 }
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800786}