blob: 3170d3c0ae9491011067ec6060ec4291915cad64 [file] [log] [blame]
Wyatt Hepleraf853ea2021-03-09 19:09:54 -08001# Copyright 2021 The Pigweed Authors
Wyatt Heplerb16dfd32020-10-12 08:46:38 -07002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14
15import("//build_overrides/pigweed.gni")
16
Wyatt Heplerb8e13602020-10-19 11:20:36 -070017import("$dir_pw_build/input_group.gni")
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080018import("$dir_pw_build/mirror_tree.gni")
Wyatt Heplerb8e13602020-10-19 11:20:36 -070019import("$dir_pw_build/python_action.gni")
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070020import("$dir_pw_protobuf_compiler/toolchain.gni")
21
22declare_args() {
23 # Python tasks, such as running tests and Pylint, are done in a single GN
24 # toolchain to avoid unnecessary duplication in the build.
25 pw_build_PYTHON_TOOLCHAIN = "$dir_pw_build/python_toolchain:python"
Michael Spang01faaf72021-06-09 22:38:40 -040026
27 # Constraints file selection (arguments to pip install --constraint).
28 # See pip help install.
29 pw_build_PIP_CONSTRAINTS = []
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070030}
Wyatt Heplerb16dfd32020-10-12 08:46:38 -070031
Wyatt Hepler438caa02021-01-15 17:13:11 -080032# Python packages provide the following targets as $target_name.$subtarget.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080033pw_python_package_subtargets = [
Wyatt Hepler438caa02021-01-15 17:13:11 -080034 "tests",
35 "lint",
36 "lint.mypy",
37 "lint.pylint",
38 "install",
39 "wheel",
Wyatt Hepler446f24c2021-03-24 09:29:35 -070040
41 # Internal targets that directly depend on one another.
42 "_run_pip_install",
43 "_build_wheel",
Wyatt Hepler438caa02021-01-15 17:13:11 -080044]
45
Wyatt Heplerb85cda42021-04-07 13:13:15 -070046# Create aliases for subsargets when the target name matches the directory name.
47# This allows //foo:foo.tests to be accessed as //foo:tests, for example.
48template("_pw_create_aliases_if_name_matches_directory") {
49 not_needed([ "invoker" ])
50
51 if (get_label_info(":$target_name", "name") ==
52 get_path_info(get_label_info(":$target_name", "dir"), "name")) {
53 foreach(subtarget, pw_python_package_subtargets) {
54 group(subtarget) {
55 public_deps = [ ":${invoker.target_name}.$subtarget" ]
56 }
57 }
58 }
59}
60
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070061# Internal template that runs Mypy.
62template("_pw_python_static_analysis_mypy") {
63 pw_python_action(target_name) {
64 module = "mypy"
65 args = [
66 "--pretty",
67 "--show-error-codes",
68 ]
69
70 if (defined(invoker.mypy_ini)) {
Michael Spangc8b93902021-05-30 15:53:56 -040071 args +=
72 [ "--config-file=" + rebase_path(invoker.mypy_ini, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070073 inputs = [ invoker.mypy_ini ]
74 }
75
Michael Spangc8b93902021-05-30 15:53:56 -040076 args += rebase_path(invoker.sources, root_build_dir)
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070077
78 # Use this environment variable to force mypy to colorize output.
79 # See https://github.com/python/mypy/issues/7771
80 environment = [ "MYPY_FORCE_COLOR=1" ]
81
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070082 stamp = true
83
84 deps = invoker.deps
85
86 foreach(dep, invoker.python_deps) {
87 deps += [ string_replace(dep, "(", ".lint.mypy(") ]
88 }
89 }
90}
91
92# Internal template that runs Pylint.
93template("_pw_python_static_analysis_pylint") {
94 # Create a target to run pylint on each of the Python files in this
95 # package and its dependencies.
96 pw_python_action_foreach(target_name) {
97 module = "pylint"
98 args = [
Michael Spangc8b93902021-05-30 15:53:56 -040099 rebase_path(".", root_build_dir) + "/{{source_target_relative}}",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700100 "--jobs=1",
101 "--output-format=colorized",
102 ]
103
104 if (defined(invoker.pylintrc)) {
Michael Spangc8b93902021-05-30 15:53:56 -0400105 args += [ "--rcfile=" + rebase_path(invoker.pylintrc, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700106 inputs = [ invoker.pylintrc ]
107 }
108
109 if (host_os == "win") {
110 # Allow CRLF on Windows, in case Git is set to switch line endings.
111 args += [ "--disable=unexpected-line-ending-format" ]
112 }
113
114 sources = invoker.sources
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700115
116 stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"
117
118 public_deps = invoker.deps
119
120 foreach(dep, invoker.python_deps) {
121 public_deps += [ string_replace(dep, "(", ".lint.pylint(") ]
122 }
123 }
124}
125
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700126# Defines a Python package. GN Python packages contain several GN targets:
127#
128# - $name - Provides the Python files in the build, but does not take any
129# actions. All subtargets depend on this target.
130# - $name.lint - Runs static analyis tools on the Python code. This is a group
131# of two subtargets:
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800132# - $name.lint.mypy - Runs mypy (if enabled).
133# - $name.lint.pylint - Runs pylint (if enabled).
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700134# - $name.tests - Runs all tests for this package.
Rob Mohrfcf60372020-11-04 11:41:36 -0800135# - $name.install - Installs the package in a venv.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800136# - $name.wheel - Builds a Python wheel for the package.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700137#
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700138# All Python packages are instantiated with in pw_build_PYTHON_TOOLCHAIN,
139# regardless of the current toolchain. This prevents Python-specific work, like
140# running Pylint, from occurring multiple times in a build.
Wyatt Hepler438caa02021-01-15 17:13:11 -0800141#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700142# Args:
143# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
144# which must all be in the same directory.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800145# generate_setup: As an alternative to 'setup', generate setup files with the
146# keywords in this scope. 'name' is required.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700147# sources: Python sources files in the package.
148# tests: Test files for this Python package.
149# python_deps: Dependencies on other pw_python_packages in the GN build.
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800150# python_test_deps: Test-only pw_python_package dependencies.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700151# other_deps: Dependencies on GN targets that are not pw_python_packages.
152# inputs: Other files to track, such as package_data.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800153# proto_library: A pw_proto_library target to embed in this Python package.
154# generate_setup is required in place of setup if proto_library is used.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700155# static_analysis: List of static analysis tools to run; "*" (default) runs
156# all tools. The supported tools are "mypy" and "pylint".
Michael Spangc8b93902021-05-30 15:53:56 -0400157# pylintrc: Path to a pylintrc configuration file to use. If not
158# provided, Pylint's default rcfile search is used. As this may
159# use the the local user's configuration file, it is highly
160# recommended to pass this option to specify the rcfile explicitly.
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800161# mypy_ini: Optional path to a mypy configuration file to use. If not
162# provided, mypy's default configuration file search is used. mypy is
163# executed from the package's setup directory, so mypy.ini files in that
164# directory will take precedence over others.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800165#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700166template("pw_python_package") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700167 # The Python targets are always instantiated in pw_build_PYTHON_TOOLCHAIN. Use
Wyatt Hepler752d7d32021-03-02 09:02:23 -0800168 # fully qualified labels so that the toolchain is not lost.
169 _other_deps = []
170 if (defined(invoker.other_deps)) {
171 foreach(dep, invoker.other_deps) {
172 _other_deps += [ get_label_info(dep, "label_with_toolchain") ]
173 }
174 }
175
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800176 _python_deps = []
177 if (defined(invoker.python_deps)) {
178 foreach(dep, invoker.python_deps) {
179 _python_deps += [ get_label_info(dep, "label_with_toolchain") ]
180 }
181 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700182
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700183 # pw_python_script uses pw_python_package, but with a limited set of features.
184 # _pw_standalone signals that this target is actually a pw_python_script.
185 _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone)
186
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800187 _generate_package = false
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800188
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800189 # Check the generate_setup and import_protos args to determine if this package
190 # is generated.
191 if (_is_package) {
192 assert(defined(invoker.generate_setup) != defined(invoker.setup),
193 "Either 'setup' or 'generate_setup' (but not both) must provided")
194
195 if (defined(invoker.proto_library)) {
196 assert(invoker.proto_library != "", "'proto_library' cannot be empty")
197 assert(defined(invoker.generate_setup),
198 "Python packages that import protos with 'proto_library' must " +
199 "use 'generate_setup' instead of 'setup'")
200
201 _import_protos = [ invoker.proto_library ]
Wyatt Heplercf184e82021-05-11 18:15:35 -0700202
203 # Depend on the dependencies of the proto library.
204 _proto = get_label_info(invoker.proto_library, "label_no_toolchain")
205 _toolchain = get_label_info(invoker.proto_library, "toolchain")
206 _python_deps += [ "$_proto.python._deps($_toolchain)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800207 } else if (defined(invoker.generate_setup)) {
208 _import_protos = []
209 }
210
211 if (defined(invoker.generate_setup)) {
212 _generate_package = true
213 _setup_dir = "$target_gen_dir/$target_name.generated_python_package"
214
215 if (defined(invoker.strip_prefix)) {
216 _source_root = invoker.strip_prefix
217 } else {
218 _source_root = "."
219 }
220 } else {
221 # Non-generated packages with sources provided need an __init__.py.
222 assert(!defined(invoker.sources) || invoker.sources == [] ||
223 filter_include(invoker.sources, [ "*\b__init__.py" ]) != [],
224 "Python packages must have at least one __init__.py file")
225
226 # Get the directories of the setup files. All must be in the same dir.
227 _setup_dirs = get_path_info(invoker.setup, "dir")
228 _setup_dir = _setup_dirs[0]
229
230 foreach(dir, _setup_dirs) {
231 assert(dir == _setup_dir,
232 "All files in 'setup' must be in the same directory")
233 }
234
235 assert(!defined(invoker.strip_prefix),
236 "'strip_prefix' may only be given if 'generate_setup' is provided")
237 }
238 }
239
240 # Process arguments defaults and set defaults.
241
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700242 _supported_static_analysis_tools = [
243 "mypy",
244 "pylint",
245 ]
246 not_needed([ "_supported_static_analysis_tools" ])
247
248 # Argument: static_analysis (list of tool names or "*"); default = "*" (all)
249 if (!defined(invoker.static_analysis) || invoker.static_analysis == "*") {
250 _static_analysis = _supported_static_analysis_tools
Keir Mierlee65ddc82020-12-02 17:59:52 -0800251 } else {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700252 _static_analysis = invoker.static_analysis
253 }
254
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700255 foreach(_tool, _static_analysis) {
256 assert(_supported_static_analysis_tools + [ _tool ] - [ _tool ] !=
257 _supported_static_analysis_tools,
258 "'$_tool' is not a supported static analysis tool")
Keir Mierlee65ddc82020-12-02 17:59:52 -0800259 }
260
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800261 # Argument: sources (list)
262 _sources = []
263 if (defined(invoker.sources)) {
264 if (_generate_package) {
265 foreach(source, rebase_path(invoker.sources, _source_root)) {
266 _sources += [ "$_setup_dir/$source" ]
267 }
268 } else {
269 _sources += invoker.sources
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700270 }
271 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700272
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800273 # Argument: tests (list)
274 _test_sources = []
275 if (defined(invoker.tests)) {
276 if (_generate_package) {
277 foreach(source, rebase_path(invoker.tests, _source_root)) {
278 _test_sources += [ "$_setup_dir/$source" ]
279 }
280 } else {
281 _test_sources += invoker.tests
Wyatt Heplerd7dc6552020-10-21 16:16:21 -0700282 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700283 }
284
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800285 # Argument: setup (list)
286 _setup_sources = []
287 if (defined(invoker.setup)) {
288 _setup_sources = invoker.setup
289 } else if (_generate_package) {
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700290 _setup_sources = [
291 "$_setup_dir/pyproject.toml",
292 "$_setup_dir/setup.cfg",
293 ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800294 }
295
296 # Argument: python_test_deps (list)
297 _python_test_deps = _python_deps # include all deps in test deps
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800298 if (defined(invoker.python_test_deps)) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800299 foreach(dep, invoker.python_test_deps) {
300 _python_test_deps += [ get_label_info(dep, "label_with_toolchain") ]
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800301 }
302 }
303
304 if (_test_sources == []) {
305 assert(!defined(invoker.python_test_deps),
306 "python_test_deps was provided, but there are no tests in " +
307 get_label_info(":$target_name", "label_no_toolchain"))
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800308 not_needed([ "_python_test_deps" ])
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800309 }
310
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700311 _all_py_files =
312 _sources + _test_sources + filter_include(_setup_sources, [ "*.py" ])
Wyatt Hepler438caa02021-01-15 17:13:11 -0800313
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700314 # The pw_python_package subtargets are only instantiated in
315 # pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the
316 # targets in this toolchain.
317 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700318 # Create the package_metadata.json file. This is used by the
319 # pw_create_python_source_tree template.
320 _package_metadata_json_file =
321 "$target_gen_dir/$target_name/package_metadata.json"
322
323 # Get Python package metadata and write to disk as JSON.
324 _package_metadata = {
325 gn_target_name = get_label_info(invoker.target_name, "label_no_toolchain")
326
327 # Get package source files
328 sources = rebase_path(_sources, root_build_dir)
329
330 # Get setup.cfg, pyproject.toml, or setup.py file
331 setup_sources = rebase_path(_setup_sources, root_build_dir)
332
333 # Get test source files
334 tests = rebase_path(_test_sources, root_build_dir)
335
336 # Get package input files (package data)
337 inputs = []
338 if (defined(invoker.inputs)) {
339 inputs = rebase_path(invoker.inputs, root_build_dir)
340 }
341
342 # Get generate_setup
343 if (defined(invoker.generate_setup)) {
344 generate_setup = invoker.generate_setup
345 }
346 }
347
348 # Finally, write out the json
349 write_file(_package_metadata_json_file, _package_metadata, "json")
350
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800351 # Declare the main Python package group. This represents the Python files,
352 # but does not take any actions. GN targets can depend on the package name
353 # to run when any files in the package change.
354 if (_generate_package) {
355 # If this package is generated, mirror the sources to the final directory.
356 pw_mirror_tree("$target_name._mirror_sources_to_out_dir") {
357 directory = _setup_dir
Wyatt Hepler438caa02021-01-15 17:13:11 -0800358
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800359 sources = []
360 if (defined(invoker.sources)) {
361 sources += invoker.sources
362 }
363 if (defined(invoker.tests)) {
364 sources += invoker.tests
365 }
Wyatt Heplerde20d742021-06-02 23:34:14 -0700366 if (defined(invoker.inputs)) {
367 sources += invoker.inputs
368 }
Wyatt Hepler438caa02021-01-15 17:13:11 -0800369
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800370 source_root = _source_root
371 public_deps = _python_deps + _other_deps
Wyatt Hepler285636f2020-12-15 13:13:11 -0800372 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700373
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700374 # Get generated_setup scope and write it to disk as JSON.
375
376 # Expected setup.cfg structure:
377 # https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800378 _gen_setup = invoker.generate_setup
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700379 assert(defined(_gen_setup.metadata),
380 "'metadata = {}' is required in generate_package")
381
382 # Get metadata which should contain at least name.
383 _gen_metadata = {
384 }
385 _gen_metadata = _gen_setup.metadata
386 assert(
387 defined(_gen_metadata.name),
388 "metadata = { name = 'package_name' } is required in generate_package")
389
390 # Get options which should not have packages or package_data.
391 if (defined(_gen_setup.options)) {
392 _gen_options = {
393 }
394 _gen_options = _gen_setup.options
395 assert(!defined(_gen_options.packages) &&
396 !defined(_gen_options.package_data),
397 "'packages' and 'package_data' may not be provided " +
398 "in 'generate_package' options.")
399 }
400
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800401 write_file("$_setup_dir/setup.json", _gen_setup, "json")
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800402
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800403 # Generate the setup.py, py.typed, and __init__.py files as needed.
404 action(target_name) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700405 metadata = {
406 pw_python_package_metadata_json = [ _package_metadata_json_file ]
407 }
408
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800409 script = "$dir_pw_build/py/pw_build/generate_python_package.py"
410 args = [
411 "--label",
412 get_label_info(":$target_name", "label_no_toolchain"),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700413 "--generated-root",
Michael Spangc8b93902021-05-30 15:53:56 -0400414 rebase_path(_setup_dir, root_build_dir),
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800415 "--setup-json",
Michael Spangc8b93902021-05-30 15:53:56 -0400416 rebase_path("$_setup_dir/setup.json", root_build_dir),
417 ] + rebase_path(_sources, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800418
Wyatt Heplercf184e82021-05-11 18:15:35 -0700419 # Pass in the .json information files for the imported proto libraries.
420 foreach(proto, _import_protos) {
421 _label = get_label_info(proto, "label_no_toolchain") +
422 ".python($pw_protobuf_compiler_TOOLCHAIN)"
423 _file = get_label_info(_label, "target_gen_dir") + "/" +
424 get_label_info(_label, "name") + ".json"
425 args += [
426 "--proto-library",
Michael Spangc8b93902021-05-30 15:53:56 -0400427 rebase_path(_file, root_build_dir),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700428 ]
429 }
430
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800431 if (defined(invoker._pw_module_as_package) &&
432 invoker._pw_module_as_package) {
433 args += [ "--module-as-package" ]
434 }
435
Wyatt Hepler0e1f5e42021-03-16 22:51:57 -0700436 inputs = [ "$_setup_dir/setup.json" ]
437
Wyatt Heplercf184e82021-05-11 18:15:35 -0700438 public_deps = [ ":$target_name._mirror_sources_to_out_dir" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800439
440 outputs = _setup_sources
441 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800442 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800443 # If the package is not generated, use an input group for the sources.
444 pw_input_group(target_name) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700445 metadata = {
446 pw_python_package_metadata_json = [ _package_metadata_json_file ]
447 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800448 inputs = _all_py_files
449 if (defined(invoker.inputs)) {
450 inputs += invoker.inputs
451 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800452
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800453 deps = _python_deps + _other_deps
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800454 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700455 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700456
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800457 if (_is_package) {
458 # Install this Python package and its dependencies in the current Python
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700459 # environment using pip.
460 pw_python_action("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800461 module = "pip"
462 public_deps = []
Joe Ethiere93ccb12021-08-20 15:59:53 -0700463 if (defined(invoker.public_deps)) {
464 public_deps += invoker.public_deps
465 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800466
Anthony DiGirolamocd405772021-10-18 16:29:26 -0700467 args = [
468 "install",
469
470 # This speeds up pip installs. At this point in the gn build the
471 # virtualenv is already activated so build isolation isn't required.
472 # This requires that pip, setuptools, and wheel packages are
473 # installed.
474 "--no-build-isolation",
475 ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800476
Michael Spang01faaf72021-06-09 22:38:40 -0400477 inputs = pw_build_PIP_CONSTRAINTS
478 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
479 args += [
480 "--constraint",
481 rebase_path(_constraints_file, root_build_dir),
482 ]
483 }
484
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800485 # For generated packages, reinstall when any files change. For regular
486 # packages, only reinstall when setup.py changes.
487 if (_generate_package) {
488 public_deps += [ ":${invoker.target_name}" ]
489 } else {
Michael Spang01faaf72021-06-09 22:38:40 -0400490 inputs += invoker.setup
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800491
492 # Install with --editable since the complete package is in source.
493 args += [ "--editable" ]
494 }
495
Michael Spangc8b93902021-05-30 15:53:56 -0400496 args += [ rebase_path(_setup_dir, root_build_dir) ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800497
498 stamp = true
499
500 # Parallel pip installations don't work, so serialize pip invocations.
Michael Spangda057f72021-06-10 00:50:36 -0400501 pool = "$dir_pw_build/pool:pip($default_toolchain)"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800502
503 foreach(dep, _python_deps) {
504 # We need to add a suffix to the target name, but the label is
505 # formatted as "//path/to:target(toolchain)", so we can't just append
506 # ".subtarget". Instead, we replace the opening parenthesis of the
507 # toolchain with ".suffix(".
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700508 public_deps += [ string_replace(dep, "(", "._run_pip_install(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800509 }
510 }
511
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800512 # Builds a Python wheel for this package. Records the output directory
513 # in the pw_python_package_wheels metadata key.
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700514 pw_python_action("$target_name._build_wheel") {
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800515 metadata = {
516 pw_python_package_wheels = [ "$target_out_dir/$target_name" ]
517 }
518
519 module = "build"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800520
Michael Spangc8b93902021-05-30 15:53:56 -0400521 args =
522 [
523 rebase_path(_setup_dir, root_build_dir),
524 "--wheel",
525 "--no-isolation",
526 "--outdir",
527 ] + rebase_path(metadata.pw_python_package_wheels, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800528
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800529 deps = [ ":${invoker.target_name}" ]
530 foreach(dep, _python_deps) {
531 deps += [ string_replace(dep, "(", ".wheel(") ]
532 }
533
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800534 stamp = true
535 }
536 } else {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700537 # Stubs for non-package targets.
538 group("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800539 }
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700540 group("$target_name._build_wheel") {
541 }
542 }
543
544 # Create the .install and .wheel targets. To limit unnecessary pip
545 # executions, non-generated packages are only reinstalled when their
546 # setup.py changes. However, targets that depend on the .install subtarget
547 # re-run whenever any source files change.
548 #
549 # These targets just represent the source files if this isn't a package.
550 group("$target_name.install") {
551 public_deps = [ ":${invoker.target_name}" ]
552
553 if (_is_package) {
554 public_deps += [ ":${invoker.target_name}._run_pip_install" ]
555 }
556
557 foreach(dep, _python_deps) {
558 public_deps += [ string_replace(dep, "(", ".install(") ]
559 }
560 }
561
562 group("$target_name.wheel") {
563 public_deps = [ ":${invoker.target_name}.install" ]
564
565 if (_is_package) {
566 public_deps += [ ":${invoker.target_name}._build_wheel" ]
567 }
568
569 foreach(dep, _python_deps) {
570 public_deps += [ string_replace(dep, "(", ".wheel(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800571 }
572 }
573
574 # Define the static analysis targets for this package.
575 group("$target_name.lint") {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700576 deps = []
577 foreach(_tool, _supported_static_analysis_tools) {
578 deps += [ ":${invoker.target_name}.lint.$_tool" ]
579 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800580 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700581
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700582 if (_static_analysis != [] || _test_sources != []) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800583 # All packages to install for either general use or test running.
584 _test_install_deps = [ ":$target_name.install" ]
585 foreach(dep, _python_test_deps + [ "$dir_pw_build:python_lint" ]) {
586 _test_install_deps += [ string_replace(dep, "(", ".install(") ]
587 }
588 }
589
590 # For packages that are not generated, create targets to run mypy and pylint.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700591 foreach(_tool, _static_analysis) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800592 # Run lint tools from the setup or target directory so that the tools detect
593 # config files (e.g. pylintrc or mypy.ini) in that directory. Config files
594 # may be explicitly specified with the pylintrc or mypy_ini arguments.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700595 target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800596 sources = _all_py_files
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700597 deps = _test_install_deps
598 python_deps = _python_deps
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800599
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700600 _optional_variables = [
601 "mypy_ini",
602 "pylintrc",
603 ]
604 forward_variables_from(invoker, _optional_variables)
605 not_needed(_optional_variables)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800606 }
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700607 }
608
609 foreach(_unused_tool, _supported_static_analysis_tools - _static_analysis) {
610 pw_input_group("$target_name.lint.$_unused_tool") {
611 inputs = []
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800612 if (defined(invoker.pylintrc)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700613 inputs += [ invoker.pylintrc ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800614 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800615 if (defined(invoker.mypy_ini)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700616 inputs += [ invoker.mypy_ini ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800617 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800618 }
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800619
620 # Generated packages with linting disabled never need the whole file list.
621 not_needed([ "_all_py_files" ])
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700622 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800623 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800624 # Create groups with the public target names ($target_name, $target_name.lint,
625 # $target_name.install, etc.). These are actually wrappers around internal
626 # Python actions instantiated with the default toolchain. This ensures there
627 # is only a single copy of each Python action in the build.
628 #
629 # The $target_name.tests group is created separately below.
630 group("$target_name") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700631 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800632 }
633
634 foreach(subtarget, pw_python_package_subtargets - [ "tests" ]) {
635 group("$target_name.$subtarget") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700636 deps =
637 [ ":${invoker.target_name}.$subtarget($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800638 }
Wyatt Heplerb8e13602020-10-19 11:20:36 -0700639 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800640
641 # Everything Python-related is only instantiated in the default toolchain.
642 # Silence not-needed warnings except for in the default toolchain.
643 not_needed("*")
644 not_needed(invoker, "*")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700645 }
646
647 # Create a target for each test file.
648 _test_targets = []
649
650 foreach(test, _test_sources) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800651 if (_is_package) {
652 _name = rebase_path(test, _setup_dir)
653 } else {
654 _name = test
655 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700656
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800657 _test_target = "$target_name.tests." + string_replace(_name, "/", "_")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700658
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700659 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800660 pw_python_action(_test_target) {
661 script = test
662 stamp = true
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800663
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800664 deps = _test_install_deps
665
666 foreach(dep, _python_test_deps) {
667 deps += [ string_replace(dep, "(", ".tests(") ]
668 }
669 }
670 } else {
671 # Create a public version of each test target, so tests can be executed as
672 # //path/to:package.tests.foo.py.
673 group(_test_target) {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700674 deps = [ ":$_test_target($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700675 }
676 }
677
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800678 _test_targets += [ ":$_test_target" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700679 }
680
681 group("$target_name.tests") {
682 deps = _test_targets
683 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700684
685 _pw_create_aliases_if_name_matches_directory(target_name) {
686 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700687}
688
689# Declares a group of Python packages or other Python groups. pw_python_groups
690# expose the same set of subtargets as pw_python_package (e.g.
691# "$group_name.lint" and "$group_name.tests"), but these apply to all packages
692# in deps and their dependencies.
693template("pw_python_group") {
694 if (defined(invoker.python_deps)) {
695 _python_deps = invoker.python_deps
696 } else {
697 _python_deps = []
Wyatt Heplera3ca62a2021-05-04 16:21:43 -0700698 not_needed([ "invoker" ]) # Allow empty groups.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700699 }
700
701 group(target_name) {
702 deps = _python_deps
Wyatt Heplercf184e82021-05-11 18:15:35 -0700703
704 if (defined(invoker.other_deps)) {
705 deps += invoker.other_deps
706 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700707 }
708
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800709 foreach(subtarget, pw_python_package_subtargets) {
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700710 group("$target_name.$subtarget") {
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700711 public_deps = []
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700712 foreach(dep, _python_deps) {
713 # Split out the toolchain to support deps with a toolchain specified.
714 _target = get_label_info(dep, "label_no_toolchain")
715 _toolchain = get_label_info(dep, "toolchain")
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700716 public_deps += [ "$_target.$subtarget($_toolchain)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700717 }
718 }
719 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700720
721 _pw_create_aliases_if_name_matches_directory(target_name) {
722 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700723}
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700724
725# Declares Python scripts or tests that are not part of a Python package.
726# Similar to pw_python_package, but only supports a subset of its features.
727#
728# pw_python_script accepts the same arguments as pw_python_package, except
729# `setup` cannot be provided.
730#
731# pw_python_script provides the same subtargets as pw_python_package, but
732# $target_name.install and $target_name.wheel only affect the python_deps of
733# this GN target, not the target itself.
Wyatt Hepler473efe02021-05-04 14:15:16 -0700734#
735# pw_python_script allows creating a pw_python_action associated with the
736# script. This is provided by passing an 'action' scope to pw_python_script.
737# This functions like a normal action, with a few additions: the action uses the
738# pw_python_script's python_deps and defaults to using the source file as its
739# 'script' argument, if there is only a single source file.
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700740template("pw_python_script") {
Wyatt Hepler473efe02021-05-04 14:15:16 -0700741 _package_variables = [
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700742 "sources",
743 "tests",
744 "python_deps",
745 "other_deps",
746 "inputs",
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800747 "pylintrc",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700748 "mypy_ini",
749 "static_analysis",
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700750 ]
751
752 pw_python_package(target_name) {
753 _pw_standalone = true
Wyatt Hepler473efe02021-05-04 14:15:16 -0700754 forward_variables_from(invoker, _package_variables)
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700755 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700756
757 _pw_create_aliases_if_name_matches_directory(target_name) {
758 }
Wyatt Hepler473efe02021-05-04 14:15:16 -0700759
760 if (defined(invoker.action)) {
761 pw_python_action("$target_name.action") {
762 forward_variables_from(invoker.action, "*", [ "python_deps" ])
763 python_deps = [ ":${invoker.target_name}" ]
764
765 if (!defined(script) && !defined(module) && defined(invoker.sources)) {
766 _sources = invoker.sources
767 assert(_sources != [] && _sources == [ _sources[0] ],
768 "'script' must be specified unless there is only one source " +
769 "in 'sources'")
770 script = _sources[0]
771 }
772 }
773 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700774}
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800775
776# Represents a list of Python requirements, as in a requirements.txt.
777#
778# Args:
779# files: One or more requirements.txt files.
780# requirements: A list of requirements.txt-style requirements.
781template("pw_python_requirements") {
782 assert(defined(invoker.files) || defined(invoker.requirements),
783 "pw_python_requirements requires a list of requirements.txt files " +
784 "in the 'files' arg or requirements in 'requirements'")
785
786 _requirements_files = []
787
788 if (defined(invoker.files)) {
789 _requirements_files += invoker.files
790 }
791
792 if (defined(invoker.requirements)) {
793 _requirements_file = "$target_gen_dir/$target_name.requirements.txt"
794 write_file(_requirements_file, invoker.requirements)
795 _requirements_files += [ _requirements_file ]
796 }
797
798 # The default target represents the requirements themselves.
799 pw_input_group(target_name) {
800 inputs = _requirements_files
801 }
802
803 # Use the same subtargets as pw_python_package so these targets can be listed
804 # as python_deps of pw_python_packages.
805 pw_python_action("$target_name.install") {
806 inputs = _requirements_files
807
808 module = "pip"
809 args = [ "install" ]
810
811 foreach(_requirements_file, inputs) {
812 args += [
813 "--requirement",
Michael Spangc8b93902021-05-30 15:53:56 -0400814 rebase_path(_requirements_file, root_build_dir),
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800815 ]
816 }
817
Michael Spang01faaf72021-06-09 22:38:40 -0400818 inputs += pw_build_PIP_CONSTRAINTS
819 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
820 args += [
821 "--constraint",
822 rebase_path(_constraints_file, root_build_dir),
823 ]
824 }
825
Michael Spangda057f72021-06-10 00:50:36 -0400826 pool = "$dir_pw_build/pool:pip($default_toolchain)"
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800827 stamp = true
828 }
829
830 # Create stubs for the unused subtargets so that pw_python_requirements can be
831 # used as python_deps.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800832 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) {
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800833 group("$target_name.$subtarget") {
834 }
835 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700836
837 _pw_create_aliases_if_name_matches_directory(target_name) {
838 }
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800839}