blob: 0f8de8fcea17421bc1d5c44d2af4160db9f70c53 [file] [log] [blame]
Wyatt Hepleraf853ea2021-03-09 19:09:54 -08001# Copyright 2021 The Pigweed Authors
Wyatt Heplerb16dfd32020-10-12 08:46:38 -07002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14
15import("//build_overrides/pigweed.gni")
16
Wyatt Heplerb8e13602020-10-19 11:20:36 -070017import("$dir_pw_build/input_group.gni")
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080018import("$dir_pw_build/mirror_tree.gni")
Wyatt Heplerb8e13602020-10-19 11:20:36 -070019import("$dir_pw_build/python_action.gni")
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070020import("$dir_pw_protobuf_compiler/toolchain.gni")
21
22declare_args() {
23 # Python tasks, such as running tests and Pylint, are done in a single GN
24 # toolchain to avoid unnecessary duplication in the build.
25 pw_build_PYTHON_TOOLCHAIN = "$dir_pw_build/python_toolchain:python"
Michael Spang01faaf72021-06-09 22:38:40 -040026
27 # Constraints file selection (arguments to pip install --constraint).
28 # See pip help install.
Ted Pudlik961706b2021-11-22 22:30:59 +000029 pw_build_PIP_CONSTRAINTS =
30 [ "$dir_pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list" ]
Anthony DiGirolamo6ab145b2022-01-20 20:52:50 -080031
32 # If true, GN will run each Python test using the coverage command. A separate
33 # coverage data file for each test will be saved. To generate reports from
34 # this information run: pw presubmit --step gn_python_test_coverage
35 pw_build_PYTHON_TEST_COVERAGE = false
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070036}
Wyatt Heplerb16dfd32020-10-12 08:46:38 -070037
Wyatt Hepler438caa02021-01-15 17:13:11 -080038# Python packages provide the following targets as $target_name.$subtarget.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080039pw_python_package_subtargets = [
Wyatt Hepler438caa02021-01-15 17:13:11 -080040 "tests",
41 "lint",
42 "lint.mypy",
43 "lint.pylint",
44 "install",
45 "wheel",
Wyatt Hepler446f24c2021-03-24 09:29:35 -070046
47 # Internal targets that directly depend on one another.
48 "_run_pip_install",
49 "_build_wheel",
Wyatt Hepler438caa02021-01-15 17:13:11 -080050]
51
Wyatt Heplerb85cda42021-04-07 13:13:15 -070052# Create aliases for subsargets when the target name matches the directory name.
53# This allows //foo:foo.tests to be accessed as //foo:tests, for example.
54template("_pw_create_aliases_if_name_matches_directory") {
55 not_needed([ "invoker" ])
56
57 if (get_label_info(":$target_name", "name") ==
58 get_path_info(get_label_info(":$target_name", "dir"), "name")) {
59 foreach(subtarget, pw_python_package_subtargets) {
60 group(subtarget) {
61 public_deps = [ ":${invoker.target_name}.$subtarget" ]
62 }
63 }
64 }
65}
66
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070067# Internal template that runs Mypy.
68template("_pw_python_static_analysis_mypy") {
69 pw_python_action(target_name) {
70 module = "mypy"
71 args = [
72 "--pretty",
73 "--show-error-codes",
74 ]
75
76 if (defined(invoker.mypy_ini)) {
Michael Spangc8b93902021-05-30 15:53:56 -040077 args +=
78 [ "--config-file=" + rebase_path(invoker.mypy_ini, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070079 inputs = [ invoker.mypy_ini ]
80 }
81
Michael Spangc8b93902021-05-30 15:53:56 -040082 args += rebase_path(invoker.sources, root_build_dir)
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070083
84 # Use this environment variable to force mypy to colorize output.
85 # See https://github.com/python/mypy/issues/7771
86 environment = [ "MYPY_FORCE_COLOR=1" ]
87
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070088 stamp = true
89
90 deps = invoker.deps
91
92 foreach(dep, invoker.python_deps) {
93 deps += [ string_replace(dep, "(", ".lint.mypy(") ]
94 }
95 }
96}
97
98# Internal template that runs Pylint.
99template("_pw_python_static_analysis_pylint") {
100 # Create a target to run pylint on each of the Python files in this
101 # package and its dependencies.
102 pw_python_action_foreach(target_name) {
103 module = "pylint"
104 args = [
Michael Spangc8b93902021-05-30 15:53:56 -0400105 rebase_path(".", root_build_dir) + "/{{source_target_relative}}",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700106 "--jobs=1",
107 "--output-format=colorized",
108 ]
109
110 if (defined(invoker.pylintrc)) {
Michael Spangc8b93902021-05-30 15:53:56 -0400111 args += [ "--rcfile=" + rebase_path(invoker.pylintrc, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700112 inputs = [ invoker.pylintrc ]
113 }
114
115 if (host_os == "win") {
116 # Allow CRLF on Windows, in case Git is set to switch line endings.
117 args += [ "--disable=unexpected-line-ending-format" ]
118 }
119
120 sources = invoker.sources
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700121
122 stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"
123
124 public_deps = invoker.deps
125
126 foreach(dep, invoker.python_deps) {
127 public_deps += [ string_replace(dep, "(", ".lint.pylint(") ]
128 }
129 }
130}
131
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700132# Defines a Python package. GN Python packages contain several GN targets:
133#
134# - $name - Provides the Python files in the build, but does not take any
135# actions. All subtargets depend on this target.
136# - $name.lint - Runs static analyis tools on the Python code. This is a group
137# of two subtargets:
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800138# - $name.lint.mypy - Runs mypy (if enabled).
139# - $name.lint.pylint - Runs pylint (if enabled).
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700140# - $name.tests - Runs all tests for this package.
Rob Mohrfcf60372020-11-04 11:41:36 -0800141# - $name.install - Installs the package in a venv.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800142# - $name.wheel - Builds a Python wheel for the package.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700143#
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700144# All Python packages are instantiated with in pw_build_PYTHON_TOOLCHAIN,
145# regardless of the current toolchain. This prevents Python-specific work, like
146# running Pylint, from occurring multiple times in a build.
Wyatt Hepler438caa02021-01-15 17:13:11 -0800147#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700148# Args:
149# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
150# which must all be in the same directory.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800151# generate_setup: As an alternative to 'setup', generate setup files with the
152# keywords in this scope. 'name' is required.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700153# sources: Python sources files in the package.
154# tests: Test files for this Python package.
155# python_deps: Dependencies on other pw_python_packages in the GN build.
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800156# python_test_deps: Test-only pw_python_package dependencies.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700157# other_deps: Dependencies on GN targets that are not pw_python_packages.
158# inputs: Other files to track, such as package_data.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800159# proto_library: A pw_proto_library target to embed in this Python package.
160# generate_setup is required in place of setup if proto_library is used.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700161# static_analysis: List of static analysis tools to run; "*" (default) runs
162# all tools. The supported tools are "mypy" and "pylint".
Michael Spangc8b93902021-05-30 15:53:56 -0400163# pylintrc: Path to a pylintrc configuration file to use. If not
164# provided, Pylint's default rcfile search is used. As this may
165# use the the local user's configuration file, it is highly
166# recommended to pass this option to specify the rcfile explicitly.
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800167# mypy_ini: Optional path to a mypy configuration file to use. If not
168# provided, mypy's default configuration file search is used. mypy is
169# executed from the package's setup directory, so mypy.ini files in that
170# directory will take precedence over others.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800171#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700172template("pw_python_package") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700173 # The Python targets are always instantiated in pw_build_PYTHON_TOOLCHAIN. Use
Wyatt Hepler752d7d32021-03-02 09:02:23 -0800174 # fully qualified labels so that the toolchain is not lost.
175 _other_deps = []
176 if (defined(invoker.other_deps)) {
177 foreach(dep, invoker.other_deps) {
178 _other_deps += [ get_label_info(dep, "label_with_toolchain") ]
179 }
180 }
181
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800182 _python_deps = []
183 if (defined(invoker.python_deps)) {
184 foreach(dep, invoker.python_deps) {
185 _python_deps += [ get_label_info(dep, "label_with_toolchain") ]
186 }
187 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700188
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700189 # pw_python_script uses pw_python_package, but with a limited set of features.
190 # _pw_standalone signals that this target is actually a pw_python_script.
191 _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone)
192
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800193 _generate_package = false
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800194
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800195 # Check the generate_setup and import_protos args to determine if this package
196 # is generated.
197 if (_is_package) {
198 assert(defined(invoker.generate_setup) != defined(invoker.setup),
199 "Either 'setup' or 'generate_setup' (but not both) must provided")
200
201 if (defined(invoker.proto_library)) {
202 assert(invoker.proto_library != "", "'proto_library' cannot be empty")
203 assert(defined(invoker.generate_setup),
204 "Python packages that import protos with 'proto_library' must " +
205 "use 'generate_setup' instead of 'setup'")
206
207 _import_protos = [ invoker.proto_library ]
Wyatt Heplercf184e82021-05-11 18:15:35 -0700208
209 # Depend on the dependencies of the proto library.
210 _proto = get_label_info(invoker.proto_library, "label_no_toolchain")
211 _toolchain = get_label_info(invoker.proto_library, "toolchain")
212 _python_deps += [ "$_proto.python._deps($_toolchain)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800213 } else if (defined(invoker.generate_setup)) {
214 _import_protos = []
215 }
216
217 if (defined(invoker.generate_setup)) {
218 _generate_package = true
219 _setup_dir = "$target_gen_dir/$target_name.generated_python_package"
220
221 if (defined(invoker.strip_prefix)) {
222 _source_root = invoker.strip_prefix
223 } else {
224 _source_root = "."
225 }
226 } else {
227 # Non-generated packages with sources provided need an __init__.py.
228 assert(!defined(invoker.sources) || invoker.sources == [] ||
229 filter_include(invoker.sources, [ "*\b__init__.py" ]) != [],
230 "Python packages must have at least one __init__.py file")
231
232 # Get the directories of the setup files. All must be in the same dir.
233 _setup_dirs = get_path_info(invoker.setup, "dir")
234 _setup_dir = _setup_dirs[0]
235
236 foreach(dir, _setup_dirs) {
237 assert(dir == _setup_dir,
238 "All files in 'setup' must be in the same directory")
239 }
240
241 assert(!defined(invoker.strip_prefix),
242 "'strip_prefix' may only be given if 'generate_setup' is provided")
243 }
244 }
245
246 # Process arguments defaults and set defaults.
247
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700248 _supported_static_analysis_tools = [
249 "mypy",
250 "pylint",
251 ]
252 not_needed([ "_supported_static_analysis_tools" ])
253
254 # Argument: static_analysis (list of tool names or "*"); default = "*" (all)
255 if (!defined(invoker.static_analysis) || invoker.static_analysis == "*") {
256 _static_analysis = _supported_static_analysis_tools
Keir Mierlee65ddc82020-12-02 17:59:52 -0800257 } else {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700258 _static_analysis = invoker.static_analysis
259 }
260
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700261 foreach(_tool, _static_analysis) {
262 assert(_supported_static_analysis_tools + [ _tool ] - [ _tool ] !=
263 _supported_static_analysis_tools,
264 "'$_tool' is not a supported static analysis tool")
Keir Mierlee65ddc82020-12-02 17:59:52 -0800265 }
266
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800267 # Argument: sources (list)
268 _sources = []
269 if (defined(invoker.sources)) {
270 if (_generate_package) {
271 foreach(source, rebase_path(invoker.sources, _source_root)) {
272 _sources += [ "$_setup_dir/$source" ]
273 }
274 } else {
275 _sources += invoker.sources
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700276 }
277 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700278
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800279 # Argument: tests (list)
280 _test_sources = []
281 if (defined(invoker.tests)) {
282 if (_generate_package) {
283 foreach(source, rebase_path(invoker.tests, _source_root)) {
284 _test_sources += [ "$_setup_dir/$source" ]
285 }
286 } else {
287 _test_sources += invoker.tests
Wyatt Heplerd7dc6552020-10-21 16:16:21 -0700288 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700289 }
290
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800291 # Argument: setup (list)
292 _setup_sources = []
293 if (defined(invoker.setup)) {
294 _setup_sources = invoker.setup
295 } else if (_generate_package) {
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700296 _setup_sources = [
297 "$_setup_dir/pyproject.toml",
298 "$_setup_dir/setup.cfg",
299 ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800300 }
301
302 # Argument: python_test_deps (list)
303 _python_test_deps = _python_deps # include all deps in test deps
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800304 if (defined(invoker.python_test_deps)) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800305 foreach(dep, invoker.python_test_deps) {
306 _python_test_deps += [ get_label_info(dep, "label_with_toolchain") ]
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800307 }
308 }
309
310 if (_test_sources == []) {
311 assert(!defined(invoker.python_test_deps),
312 "python_test_deps was provided, but there are no tests in " +
313 get_label_info(":$target_name", "label_no_toolchain"))
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800314 not_needed([ "_python_test_deps" ])
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800315 }
316
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700317 _all_py_files =
318 _sources + _test_sources + filter_include(_setup_sources, [ "*.py" ])
Wyatt Hepler438caa02021-01-15 17:13:11 -0800319
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700320 # The pw_python_package subtargets are only instantiated in
321 # pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the
322 # targets in this toolchain.
323 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700324 # Create the package_metadata.json file. This is used by the
325 # pw_create_python_source_tree template.
326 _package_metadata_json_file =
327 "$target_gen_dir/$target_name/package_metadata.json"
328
329 # Get Python package metadata and write to disk as JSON.
330 _package_metadata = {
331 gn_target_name = get_label_info(invoker.target_name, "label_no_toolchain")
332
333 # Get package source files
334 sources = rebase_path(_sources, root_build_dir)
335
336 # Get setup.cfg, pyproject.toml, or setup.py file
337 setup_sources = rebase_path(_setup_sources, root_build_dir)
338
339 # Get test source files
340 tests = rebase_path(_test_sources, root_build_dir)
341
342 # Get package input files (package data)
343 inputs = []
344 if (defined(invoker.inputs)) {
345 inputs = rebase_path(invoker.inputs, root_build_dir)
346 }
347
348 # Get generate_setup
349 if (defined(invoker.generate_setup)) {
350 generate_setup = invoker.generate_setup
351 }
352 }
353
354 # Finally, write out the json
355 write_file(_package_metadata_json_file, _package_metadata, "json")
356
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800357 # Declare the main Python package group. This represents the Python files,
358 # but does not take any actions. GN targets can depend on the package name
359 # to run when any files in the package change.
360 if (_generate_package) {
361 # If this package is generated, mirror the sources to the final directory.
362 pw_mirror_tree("$target_name._mirror_sources_to_out_dir") {
363 directory = _setup_dir
Wyatt Hepler438caa02021-01-15 17:13:11 -0800364
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800365 sources = []
366 if (defined(invoker.sources)) {
367 sources += invoker.sources
368 }
369 if (defined(invoker.tests)) {
370 sources += invoker.tests
371 }
Wyatt Heplerde20d742021-06-02 23:34:14 -0700372 if (defined(invoker.inputs)) {
373 sources += invoker.inputs
374 }
Wyatt Hepler438caa02021-01-15 17:13:11 -0800375
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800376 source_root = _source_root
377 public_deps = _python_deps + _other_deps
Wyatt Hepler285636f2020-12-15 13:13:11 -0800378 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700379
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700380 # Get generated_setup scope and write it to disk as JSON.
381
382 # Expected setup.cfg structure:
383 # https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800384 _gen_setup = invoker.generate_setup
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700385 assert(defined(_gen_setup.metadata),
386 "'metadata = {}' is required in generate_package")
387
388 # Get metadata which should contain at least name.
389 _gen_metadata = {
390 }
391 _gen_metadata = _gen_setup.metadata
392 assert(
393 defined(_gen_metadata.name),
394 "metadata = { name = 'package_name' } is required in generate_package")
395
396 # Get options which should not have packages or package_data.
397 if (defined(_gen_setup.options)) {
398 _gen_options = {
399 }
400 _gen_options = _gen_setup.options
401 assert(!defined(_gen_options.packages) &&
402 !defined(_gen_options.package_data),
403 "'packages' and 'package_data' may not be provided " +
404 "in 'generate_package' options.")
405 }
406
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800407 write_file("$_setup_dir/setup.json", _gen_setup, "json")
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800408
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800409 # Generate the setup.py, py.typed, and __init__.py files as needed.
410 action(target_name) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700411 metadata = {
412 pw_python_package_metadata_json = [ _package_metadata_json_file ]
413 }
414
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800415 script = "$dir_pw_build/py/pw_build/generate_python_package.py"
416 args = [
417 "--label",
418 get_label_info(":$target_name", "label_no_toolchain"),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700419 "--generated-root",
Michael Spangc8b93902021-05-30 15:53:56 -0400420 rebase_path(_setup_dir, root_build_dir),
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800421 "--setup-json",
Michael Spangc8b93902021-05-30 15:53:56 -0400422 rebase_path("$_setup_dir/setup.json", root_build_dir),
423 ] + rebase_path(_sources, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800424
Wyatt Heplercf184e82021-05-11 18:15:35 -0700425 # Pass in the .json information files for the imported proto libraries.
426 foreach(proto, _import_protos) {
427 _label = get_label_info(proto, "label_no_toolchain") +
428 ".python($pw_protobuf_compiler_TOOLCHAIN)"
429 _file = get_label_info(_label, "target_gen_dir") + "/" +
430 get_label_info(_label, "name") + ".json"
431 args += [
432 "--proto-library",
Michael Spangc8b93902021-05-30 15:53:56 -0400433 rebase_path(_file, root_build_dir),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700434 ]
435 }
436
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800437 if (defined(invoker._pw_module_as_package) &&
438 invoker._pw_module_as_package) {
439 args += [ "--module-as-package" ]
440 }
441
Wyatt Hepler0e1f5e42021-03-16 22:51:57 -0700442 inputs = [ "$_setup_dir/setup.json" ]
443
Wyatt Heplercf184e82021-05-11 18:15:35 -0700444 public_deps = [ ":$target_name._mirror_sources_to_out_dir" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800445
446 outputs = _setup_sources
447 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800448 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800449 # If the package is not generated, use an input group for the sources.
450 pw_input_group(target_name) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700451 metadata = {
452 pw_python_package_metadata_json = [ _package_metadata_json_file ]
453 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800454 inputs = _all_py_files
455 if (defined(invoker.inputs)) {
456 inputs += invoker.inputs
457 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800458
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800459 deps = _python_deps + _other_deps
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800460 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700461 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700462
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800463 if (_is_package) {
464 # Install this Python package and its dependencies in the current Python
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700465 # environment using pip.
466 pw_python_action("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800467 module = "pip"
468 public_deps = []
Joe Ethiere93ccb12021-08-20 15:59:53 -0700469 if (defined(invoker.public_deps)) {
470 public_deps += invoker.public_deps
471 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800472
Anthony DiGirolamocd405772021-10-18 16:29:26 -0700473 args = [
474 "install",
475
476 # This speeds up pip installs. At this point in the gn build the
477 # virtualenv is already activated so build isolation isn't required.
478 # This requires that pip, setuptools, and wheel packages are
479 # installed.
480 "--no-build-isolation",
481 ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800482
Michael Spang01faaf72021-06-09 22:38:40 -0400483 inputs = pw_build_PIP_CONSTRAINTS
484 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
485 args += [
486 "--constraint",
487 rebase_path(_constraints_file, root_build_dir),
488 ]
489 }
490
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800491 # For generated packages, reinstall when any files change. For regular
492 # packages, only reinstall when setup.py changes.
493 if (_generate_package) {
494 public_deps += [ ":${invoker.target_name}" ]
495 } else {
Michael Spang01faaf72021-06-09 22:38:40 -0400496 inputs += invoker.setup
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800497
498 # Install with --editable since the complete package is in source.
499 args += [ "--editable" ]
500 }
501
Michael Spangc8b93902021-05-30 15:53:56 -0400502 args += [ rebase_path(_setup_dir, root_build_dir) ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800503
504 stamp = true
505
506 # Parallel pip installations don't work, so serialize pip invocations.
Michael Spangda057f72021-06-10 00:50:36 -0400507 pool = "$dir_pw_build/pool:pip($default_toolchain)"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800508
509 foreach(dep, _python_deps) {
510 # We need to add a suffix to the target name, but the label is
511 # formatted as "//path/to:target(toolchain)", so we can't just append
512 # ".subtarget". Instead, we replace the opening parenthesis of the
513 # toolchain with ".suffix(".
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700514 public_deps += [ string_replace(dep, "(", "._run_pip_install(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800515 }
516 }
517
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800518 # Builds a Python wheel for this package. Records the output directory
519 # in the pw_python_package_wheels metadata key.
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700520 pw_python_action("$target_name._build_wheel") {
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800521 metadata = {
522 pw_python_package_wheels = [ "$target_out_dir/$target_name" ]
523 }
524
525 module = "build"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800526
Michael Spangc8b93902021-05-30 15:53:56 -0400527 args =
528 [
529 rebase_path(_setup_dir, root_build_dir),
530 "--wheel",
531 "--no-isolation",
532 "--outdir",
533 ] + rebase_path(metadata.pw_python_package_wheels, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800534
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800535 deps = [ ":${invoker.target_name}" ]
536 foreach(dep, _python_deps) {
537 deps += [ string_replace(dep, "(", ".wheel(") ]
538 }
539
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800540 stamp = true
541 }
542 } else {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700543 # Stubs for non-package targets.
544 group("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800545 }
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700546 group("$target_name._build_wheel") {
547 }
548 }
549
550 # Create the .install and .wheel targets. To limit unnecessary pip
551 # executions, non-generated packages are only reinstalled when their
552 # setup.py changes. However, targets that depend on the .install subtarget
553 # re-run whenever any source files change.
554 #
555 # These targets just represent the source files if this isn't a package.
556 group("$target_name.install") {
557 public_deps = [ ":${invoker.target_name}" ]
558
559 if (_is_package) {
560 public_deps += [ ":${invoker.target_name}._run_pip_install" ]
561 }
562
563 foreach(dep, _python_deps) {
564 public_deps += [ string_replace(dep, "(", ".install(") ]
565 }
566 }
567
568 group("$target_name.wheel") {
569 public_deps = [ ":${invoker.target_name}.install" ]
570
571 if (_is_package) {
572 public_deps += [ ":${invoker.target_name}._build_wheel" ]
573 }
574
575 foreach(dep, _python_deps) {
576 public_deps += [ string_replace(dep, "(", ".wheel(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800577 }
578 }
579
580 # Define the static analysis targets for this package.
581 group("$target_name.lint") {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700582 deps = []
583 foreach(_tool, _supported_static_analysis_tools) {
584 deps += [ ":${invoker.target_name}.lint.$_tool" ]
585 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800586 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700587
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700588 if (_static_analysis != [] || _test_sources != []) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800589 # All packages to install for either general use or test running.
590 _test_install_deps = [ ":$target_name.install" ]
591 foreach(dep, _python_test_deps + [ "$dir_pw_build:python_lint" ]) {
592 _test_install_deps += [ string_replace(dep, "(", ".install(") ]
593 }
594 }
595
596 # For packages that are not generated, create targets to run mypy and pylint.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700597 foreach(_tool, _static_analysis) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800598 # Run lint tools from the setup or target directory so that the tools detect
599 # config files (e.g. pylintrc or mypy.ini) in that directory. Config files
600 # may be explicitly specified with the pylintrc or mypy_ini arguments.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700601 target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800602 sources = _all_py_files
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700603 deps = _test_install_deps
604 python_deps = _python_deps
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800605
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700606 _optional_variables = [
607 "mypy_ini",
608 "pylintrc",
609 ]
610 forward_variables_from(invoker, _optional_variables)
611 not_needed(_optional_variables)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800612 }
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700613 }
614
615 foreach(_unused_tool, _supported_static_analysis_tools - _static_analysis) {
616 pw_input_group("$target_name.lint.$_unused_tool") {
617 inputs = []
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800618 if (defined(invoker.pylintrc)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700619 inputs += [ invoker.pylintrc ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800620 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800621 if (defined(invoker.mypy_ini)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700622 inputs += [ invoker.mypy_ini ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800623 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800624 }
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800625
626 # Generated packages with linting disabled never need the whole file list.
627 not_needed([ "_all_py_files" ])
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700628 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800629 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800630 # Create groups with the public target names ($target_name, $target_name.lint,
631 # $target_name.install, etc.). These are actually wrappers around internal
632 # Python actions instantiated with the default toolchain. This ensures there
633 # is only a single copy of each Python action in the build.
634 #
635 # The $target_name.tests group is created separately below.
636 group("$target_name") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700637 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800638 }
639
640 foreach(subtarget, pw_python_package_subtargets - [ "tests" ]) {
641 group("$target_name.$subtarget") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700642 deps =
643 [ ":${invoker.target_name}.$subtarget($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800644 }
Wyatt Heplerb8e13602020-10-19 11:20:36 -0700645 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800646
647 # Everything Python-related is only instantiated in the default toolchain.
648 # Silence not-needed warnings except for in the default toolchain.
649 not_needed("*")
650 not_needed(invoker, "*")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700651 }
652
653 # Create a target for each test file.
654 _test_targets = []
655
656 foreach(test, _test_sources) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800657 if (_is_package) {
658 _name = rebase_path(test, _setup_dir)
659 } else {
660 _name = test
661 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700662
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800663 _test_target = "$target_name.tests." + string_replace(_name, "/", "_")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700664
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700665 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800666 pw_python_action(_test_target) {
Anthony DiGirolamo6ab145b2022-01-20 20:52:50 -0800667 if (pw_build_PYTHON_TEST_COVERAGE) {
668 module = "coverage"
669 working_directory =
670 rebase_path(get_path_info(test, "dir"), root_build_dir)
671 args = [
672 "run",
673 "--branch",
674
675 # Include all source files in the working_directory when calculating coverage.
676 "--source=.",
677
678 # Test file to run.
679 get_path_info(test, "file"),
680 ]
681
682 # Set the coverage file to a location in out/python/gen/
683 _coverage_data_file = "$target_gen_dir/$target_name.coverage"
684 outputs = [ _coverage_data_file ]
685
686 # The coverage tool only allows setting the output with an environment variable.
687 environment =
688 [ "COVERAGE_FILE=" +
689 rebase_path(_coverage_data_file, get_path_info(test, "dir")) ]
690 } else {
691 script = test
692 }
693
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800694 stamp = true
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800695
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800696 deps = _test_install_deps
697
698 foreach(dep, _python_test_deps) {
699 deps += [ string_replace(dep, "(", ".tests(") ]
700 }
701 }
702 } else {
703 # Create a public version of each test target, so tests can be executed as
704 # //path/to:package.tests.foo.py.
705 group(_test_target) {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700706 deps = [ ":$_test_target($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700707 }
708 }
709
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800710 _test_targets += [ ":$_test_target" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700711 }
712
713 group("$target_name.tests") {
714 deps = _test_targets
715 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700716
717 _pw_create_aliases_if_name_matches_directory(target_name) {
718 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700719}
720
721# Declares a group of Python packages or other Python groups. pw_python_groups
722# expose the same set of subtargets as pw_python_package (e.g.
723# "$group_name.lint" and "$group_name.tests"), but these apply to all packages
724# in deps and their dependencies.
725template("pw_python_group") {
726 if (defined(invoker.python_deps)) {
727 _python_deps = invoker.python_deps
728 } else {
729 _python_deps = []
Wyatt Heplera3ca62a2021-05-04 16:21:43 -0700730 not_needed([ "invoker" ]) # Allow empty groups.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700731 }
732
733 group(target_name) {
734 deps = _python_deps
Wyatt Heplercf184e82021-05-11 18:15:35 -0700735
736 if (defined(invoker.other_deps)) {
737 deps += invoker.other_deps
738 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700739 }
740
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800741 foreach(subtarget, pw_python_package_subtargets) {
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700742 group("$target_name.$subtarget") {
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700743 public_deps = []
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700744 foreach(dep, _python_deps) {
745 # Split out the toolchain to support deps with a toolchain specified.
746 _target = get_label_info(dep, "label_no_toolchain")
747 _toolchain = get_label_info(dep, "toolchain")
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700748 public_deps += [ "$_target.$subtarget($_toolchain)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700749 }
750 }
751 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700752
753 _pw_create_aliases_if_name_matches_directory(target_name) {
754 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700755}
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700756
757# Declares Python scripts or tests that are not part of a Python package.
758# Similar to pw_python_package, but only supports a subset of its features.
759#
760# pw_python_script accepts the same arguments as pw_python_package, except
761# `setup` cannot be provided.
762#
763# pw_python_script provides the same subtargets as pw_python_package, but
764# $target_name.install and $target_name.wheel only affect the python_deps of
765# this GN target, not the target itself.
Wyatt Hepler473efe02021-05-04 14:15:16 -0700766#
767# pw_python_script allows creating a pw_python_action associated with the
768# script. This is provided by passing an 'action' scope to pw_python_script.
769# This functions like a normal action, with a few additions: the action uses the
770# pw_python_script's python_deps and defaults to using the source file as its
771# 'script' argument, if there is only a single source file.
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700772template("pw_python_script") {
Wyatt Hepler473efe02021-05-04 14:15:16 -0700773 _package_variables = [
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700774 "sources",
775 "tests",
776 "python_deps",
777 "other_deps",
778 "inputs",
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800779 "pylintrc",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700780 "mypy_ini",
781 "static_analysis",
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700782 ]
783
784 pw_python_package(target_name) {
785 _pw_standalone = true
Wyatt Hepler473efe02021-05-04 14:15:16 -0700786 forward_variables_from(invoker, _package_variables)
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700787 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700788
789 _pw_create_aliases_if_name_matches_directory(target_name) {
790 }
Wyatt Hepler473efe02021-05-04 14:15:16 -0700791
792 if (defined(invoker.action)) {
793 pw_python_action("$target_name.action") {
794 forward_variables_from(invoker.action, "*", [ "python_deps" ])
795 python_deps = [ ":${invoker.target_name}" ]
796
797 if (!defined(script) && !defined(module) && defined(invoker.sources)) {
798 _sources = invoker.sources
799 assert(_sources != [] && _sources == [ _sources[0] ],
800 "'script' must be specified unless there is only one source " +
801 "in 'sources'")
802 script = _sources[0]
803 }
804 }
805 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700806}
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800807
808# Represents a list of Python requirements, as in a requirements.txt.
809#
810# Args:
811# files: One or more requirements.txt files.
812# requirements: A list of requirements.txt-style requirements.
813template("pw_python_requirements") {
814 assert(defined(invoker.files) || defined(invoker.requirements),
815 "pw_python_requirements requires a list of requirements.txt files " +
816 "in the 'files' arg or requirements in 'requirements'")
817
818 _requirements_files = []
819
820 if (defined(invoker.files)) {
821 _requirements_files += invoker.files
822 }
823
824 if (defined(invoker.requirements)) {
825 _requirements_file = "$target_gen_dir/$target_name.requirements.txt"
826 write_file(_requirements_file, invoker.requirements)
827 _requirements_files += [ _requirements_file ]
828 }
829
830 # The default target represents the requirements themselves.
831 pw_input_group(target_name) {
832 inputs = _requirements_files
833 }
834
835 # Use the same subtargets as pw_python_package so these targets can be listed
836 # as python_deps of pw_python_packages.
837 pw_python_action("$target_name.install") {
838 inputs = _requirements_files
839
840 module = "pip"
841 args = [ "install" ]
842
843 foreach(_requirements_file, inputs) {
844 args += [
845 "--requirement",
Michael Spangc8b93902021-05-30 15:53:56 -0400846 rebase_path(_requirements_file, root_build_dir),
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800847 ]
848 }
849
Michael Spang01faaf72021-06-09 22:38:40 -0400850 inputs += pw_build_PIP_CONSTRAINTS
851 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
852 args += [
853 "--constraint",
854 rebase_path(_constraints_file, root_build_dir),
855 ]
856 }
857
Michael Spangda057f72021-06-10 00:50:36 -0400858 pool = "$dir_pw_build/pool:pip($default_toolchain)"
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800859 stamp = true
860 }
861
862 # Create stubs for the unused subtargets so that pw_python_requirements can be
863 # used as python_deps.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800864 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) {
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800865 group("$target_name.$subtarget") {
866 }
867 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700868
869 _pw_create_aliases_if_name_matches_directory(target_name) {
870 }
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800871}