blob: cb5c1e3d8f952a3003ea6a3835023b89aa0df9ec [file] [log] [blame]
Wyatt Hepleraf853ea2021-03-09 19:09:54 -08001# Copyright 2021 The Pigweed Authors
Wyatt Heplerb16dfd32020-10-12 08:46:38 -07002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14
15import("//build_overrides/pigweed.gni")
16
Wyatt Heplerb8e13602020-10-19 11:20:36 -070017import("$dir_pw_build/input_group.gni")
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080018import("$dir_pw_build/mirror_tree.gni")
Wyatt Heplerb8e13602020-10-19 11:20:36 -070019import("$dir_pw_build/python_action.gni")
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070020import("$dir_pw_protobuf_compiler/toolchain.gni")
21
22declare_args() {
23 # Python tasks, such as running tests and Pylint, are done in a single GN
24 # toolchain to avoid unnecessary duplication in the build.
25 pw_build_PYTHON_TOOLCHAIN = "$dir_pw_build/python_toolchain:python"
Michael Spang01faaf72021-06-09 22:38:40 -040026
27 # Constraints file selection (arguments to pip install --constraint).
28 # See pip help install.
29 pw_build_PIP_CONSTRAINTS = []
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070030}
Wyatt Heplerb16dfd32020-10-12 08:46:38 -070031
Wyatt Hepler438caa02021-01-15 17:13:11 -080032# Python packages provide the following targets as $target_name.$subtarget.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080033pw_python_package_subtargets = [
Wyatt Hepler438caa02021-01-15 17:13:11 -080034 "tests",
35 "lint",
36 "lint.mypy",
37 "lint.pylint",
38 "install",
39 "wheel",
Wyatt Hepler446f24c2021-03-24 09:29:35 -070040
41 # Internal targets that directly depend on one another.
42 "_run_pip_install",
43 "_build_wheel",
Wyatt Hepler438caa02021-01-15 17:13:11 -080044]
45
Wyatt Heplerb85cda42021-04-07 13:13:15 -070046# Create aliases for subsargets when the target name matches the directory name.
47# This allows //foo:foo.tests to be accessed as //foo:tests, for example.
48template("_pw_create_aliases_if_name_matches_directory") {
49 not_needed([ "invoker" ])
50
51 if (get_label_info(":$target_name", "name") ==
52 get_path_info(get_label_info(":$target_name", "dir"), "name")) {
53 foreach(subtarget, pw_python_package_subtargets) {
54 group(subtarget) {
55 public_deps = [ ":${invoker.target_name}.$subtarget" ]
56 }
57 }
58 }
59}
60
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070061# Internal template that runs Mypy.
62template("_pw_python_static_analysis_mypy") {
63 pw_python_action(target_name) {
64 module = "mypy"
65 args = [
66 "--pretty",
67 "--show-error-codes",
68 ]
69
70 if (defined(invoker.mypy_ini)) {
Michael Spangc8b93902021-05-30 15:53:56 -040071 args +=
72 [ "--config-file=" + rebase_path(invoker.mypy_ini, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070073 inputs = [ invoker.mypy_ini ]
74 }
75
Michael Spangc8b93902021-05-30 15:53:56 -040076 args += rebase_path(invoker.sources, root_build_dir)
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070077
78 # Use this environment variable to force mypy to colorize output.
79 # See https://github.com/python/mypy/issues/7771
80 environment = [ "MYPY_FORCE_COLOR=1" ]
81
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070082 stamp = true
83
84 deps = invoker.deps
85
86 foreach(dep, invoker.python_deps) {
87 deps += [ string_replace(dep, "(", ".lint.mypy(") ]
88 }
89 }
90}
91
92# Internal template that runs Pylint.
93template("_pw_python_static_analysis_pylint") {
94 # Create a target to run pylint on each of the Python files in this
95 # package and its dependencies.
96 pw_python_action_foreach(target_name) {
97 module = "pylint"
98 args = [
Michael Spangc8b93902021-05-30 15:53:56 -040099 rebase_path(".", root_build_dir) + "/{{source_target_relative}}",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700100 "--jobs=1",
101 "--output-format=colorized",
102 ]
103
104 if (defined(invoker.pylintrc)) {
Michael Spangc8b93902021-05-30 15:53:56 -0400105 args += [ "--rcfile=" + rebase_path(invoker.pylintrc, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700106 inputs = [ invoker.pylintrc ]
107 }
108
109 if (host_os == "win") {
110 # Allow CRLF on Windows, in case Git is set to switch line endings.
111 args += [ "--disable=unexpected-line-ending-format" ]
112 }
113
114 sources = invoker.sources
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700115
116 stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"
117
118 public_deps = invoker.deps
119
120 foreach(dep, invoker.python_deps) {
121 public_deps += [ string_replace(dep, "(", ".lint.pylint(") ]
122 }
123 }
124}
125
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700126# Defines a Python package. GN Python packages contain several GN targets:
127#
128# - $name - Provides the Python files in the build, but does not take any
129# actions. All subtargets depend on this target.
130# - $name.lint - Runs static analyis tools on the Python code. This is a group
131# of two subtargets:
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800132# - $name.lint.mypy - Runs mypy (if enabled).
133# - $name.lint.pylint - Runs pylint (if enabled).
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700134# - $name.tests - Runs all tests for this package.
Rob Mohrfcf60372020-11-04 11:41:36 -0800135# - $name.install - Installs the package in a venv.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800136# - $name.wheel - Builds a Python wheel for the package.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700137#
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700138# All Python packages are instantiated with in pw_build_PYTHON_TOOLCHAIN,
139# regardless of the current toolchain. This prevents Python-specific work, like
140# running Pylint, from occurring multiple times in a build.
Wyatt Hepler438caa02021-01-15 17:13:11 -0800141#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700142# Args:
143# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
144# which must all be in the same directory.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800145# generate_setup: As an alternative to 'setup', generate setup files with the
146# keywords in this scope. 'name' is required.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700147# sources: Python sources files in the package.
148# tests: Test files for this Python package.
149# python_deps: Dependencies on other pw_python_packages in the GN build.
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800150# python_test_deps: Test-only pw_python_package dependencies.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700151# other_deps: Dependencies on GN targets that are not pw_python_packages.
152# inputs: Other files to track, such as package_data.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800153# proto_library: A pw_proto_library target to embed in this Python package.
154# generate_setup is required in place of setup if proto_library is used.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700155# static_analysis: List of static analysis tools to run; "*" (default) runs
156# all tools. The supported tools are "mypy" and "pylint".
Michael Spangc8b93902021-05-30 15:53:56 -0400157# pylintrc: Path to a pylintrc configuration file to use. If not
158# provided, Pylint's default rcfile search is used. As this may
159# use the the local user's configuration file, it is highly
160# recommended to pass this option to specify the rcfile explicitly.
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800161# mypy_ini: Optional path to a mypy configuration file to use. If not
162# provided, mypy's default configuration file search is used. mypy is
163# executed from the package's setup directory, so mypy.ini files in that
164# directory will take precedence over others.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800165#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700166template("pw_python_package") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700167 # The Python targets are always instantiated in pw_build_PYTHON_TOOLCHAIN. Use
Wyatt Hepler752d7d32021-03-02 09:02:23 -0800168 # fully qualified labels so that the toolchain is not lost.
169 _other_deps = []
170 if (defined(invoker.other_deps)) {
171 foreach(dep, invoker.other_deps) {
172 _other_deps += [ get_label_info(dep, "label_with_toolchain") ]
173 }
174 }
175
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800176 _python_deps = []
177 if (defined(invoker.python_deps)) {
178 foreach(dep, invoker.python_deps) {
179 _python_deps += [ get_label_info(dep, "label_with_toolchain") ]
180 }
181 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700182
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700183 # pw_python_script uses pw_python_package, but with a limited set of features.
184 # _pw_standalone signals that this target is actually a pw_python_script.
185 _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone)
186
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800187 _generate_package = false
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800188
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800189 # Check the generate_setup and import_protos args to determine if this package
190 # is generated.
191 if (_is_package) {
192 assert(defined(invoker.generate_setup) != defined(invoker.setup),
193 "Either 'setup' or 'generate_setup' (but not both) must provided")
194
195 if (defined(invoker.proto_library)) {
196 assert(invoker.proto_library != "", "'proto_library' cannot be empty")
197 assert(defined(invoker.generate_setup),
198 "Python packages that import protos with 'proto_library' must " +
199 "use 'generate_setup' instead of 'setup'")
200
201 _import_protos = [ invoker.proto_library ]
Wyatt Heplercf184e82021-05-11 18:15:35 -0700202
203 # Depend on the dependencies of the proto library.
204 _proto = get_label_info(invoker.proto_library, "label_no_toolchain")
205 _toolchain = get_label_info(invoker.proto_library, "toolchain")
206 _python_deps += [ "$_proto.python._deps($_toolchain)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800207 } else if (defined(invoker.generate_setup)) {
208 _import_protos = []
209 }
210
211 if (defined(invoker.generate_setup)) {
212 _generate_package = true
213 _setup_dir = "$target_gen_dir/$target_name.generated_python_package"
214
215 if (defined(invoker.strip_prefix)) {
216 _source_root = invoker.strip_prefix
217 } else {
218 _source_root = "."
219 }
220 } else {
221 # Non-generated packages with sources provided need an __init__.py.
222 assert(!defined(invoker.sources) || invoker.sources == [] ||
223 filter_include(invoker.sources, [ "*\b__init__.py" ]) != [],
224 "Python packages must have at least one __init__.py file")
225
226 # Get the directories of the setup files. All must be in the same dir.
227 _setup_dirs = get_path_info(invoker.setup, "dir")
228 _setup_dir = _setup_dirs[0]
229
230 foreach(dir, _setup_dirs) {
231 assert(dir == _setup_dir,
232 "All files in 'setup' must be in the same directory")
233 }
234
235 assert(!defined(invoker.strip_prefix),
236 "'strip_prefix' may only be given if 'generate_setup' is provided")
237 }
238 }
239
240 # Process arguments defaults and set defaults.
241
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700242 _supported_static_analysis_tools = [
243 "mypy",
244 "pylint",
245 ]
246 not_needed([ "_supported_static_analysis_tools" ])
247
248 # Argument: static_analysis (list of tool names or "*"); default = "*" (all)
249 if (!defined(invoker.static_analysis) || invoker.static_analysis == "*") {
250 _static_analysis = _supported_static_analysis_tools
Keir Mierlee65ddc82020-12-02 17:59:52 -0800251 } else {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700252 _static_analysis = invoker.static_analysis
253 }
254
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700255 foreach(_tool, _static_analysis) {
256 assert(_supported_static_analysis_tools + [ _tool ] - [ _tool ] !=
257 _supported_static_analysis_tools,
258 "'$_tool' is not a supported static analysis tool")
Keir Mierlee65ddc82020-12-02 17:59:52 -0800259 }
260
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800261 # Argument: sources (list)
262 _sources = []
263 if (defined(invoker.sources)) {
264 if (_generate_package) {
265 foreach(source, rebase_path(invoker.sources, _source_root)) {
266 _sources += [ "$_setup_dir/$source" ]
267 }
268 } else {
269 _sources += invoker.sources
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700270 }
271 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700272
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800273 # Argument: tests (list)
274 _test_sources = []
275 if (defined(invoker.tests)) {
276 if (_generate_package) {
277 foreach(source, rebase_path(invoker.tests, _source_root)) {
278 _test_sources += [ "$_setup_dir/$source" ]
279 }
280 } else {
281 _test_sources += invoker.tests
Wyatt Heplerd7dc6552020-10-21 16:16:21 -0700282 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700283 }
284
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800285 # Argument: setup (list)
286 _setup_sources = []
287 if (defined(invoker.setup)) {
288 _setup_sources = invoker.setup
289 } else if (_generate_package) {
290 _setup_sources = [ "$_setup_dir/setup.py" ]
291 }
292
293 # Argument: python_test_deps (list)
294 _python_test_deps = _python_deps # include all deps in test deps
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800295 if (defined(invoker.python_test_deps)) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800296 foreach(dep, invoker.python_test_deps) {
297 _python_test_deps += [ get_label_info(dep, "label_with_toolchain") ]
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800298 }
299 }
300
301 if (_test_sources == []) {
302 assert(!defined(invoker.python_test_deps),
303 "python_test_deps was provided, but there are no tests in " +
304 get_label_info(":$target_name", "label_no_toolchain"))
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800305 not_needed([ "_python_test_deps" ])
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800306 }
307
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800308 _all_py_files = _sources + _test_sources + _setup_sources
Wyatt Hepler438caa02021-01-15 17:13:11 -0800309
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700310 # The pw_python_package subtargets are only instantiated in
311 # pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the
312 # targets in this toolchain.
313 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800314 # Declare the main Python package group. This represents the Python files,
315 # but does not take any actions. GN targets can depend on the package name
316 # to run when any files in the package change.
317 if (_generate_package) {
318 # If this package is generated, mirror the sources to the final directory.
319 pw_mirror_tree("$target_name._mirror_sources_to_out_dir") {
320 directory = _setup_dir
Wyatt Hepler438caa02021-01-15 17:13:11 -0800321
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800322 sources = []
323 if (defined(invoker.sources)) {
324 sources += invoker.sources
325 }
326 if (defined(invoker.tests)) {
327 sources += invoker.tests
328 }
Wyatt Heplerde20d742021-06-02 23:34:14 -0700329 if (defined(invoker.inputs)) {
330 sources += invoker.inputs
331 }
Wyatt Hepler438caa02021-01-15 17:13:11 -0800332
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800333 source_root = _source_root
334 public_deps = _python_deps + _other_deps
Wyatt Hepler285636f2020-12-15 13:13:11 -0800335 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700336
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800337 # Get generated_setup scope and write it to disk ask JSON.
338 _gen_setup = invoker.generate_setup
339 assert(defined(_gen_setup.name), "'name' is required in generate_package")
340 assert(!defined(_gen_setup.packages) && !defined(_gen_setup.package_data),
341 "'packages' and 'package_data' may not be provided " +
342 "in 'generate_package'")
343 write_file("$_setup_dir/setup.json", _gen_setup, "json")
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800344
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800345 # Generate the setup.py, py.typed, and __init__.py files as needed.
346 action(target_name) {
347 script = "$dir_pw_build/py/pw_build/generate_python_package.py"
348 args = [
349 "--label",
350 get_label_info(":$target_name", "label_no_toolchain"),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700351 "--generated-root",
Michael Spangc8b93902021-05-30 15:53:56 -0400352 rebase_path(_setup_dir, root_build_dir),
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800353 "--setup-json",
Michael Spangc8b93902021-05-30 15:53:56 -0400354 rebase_path("$_setup_dir/setup.json", root_build_dir),
355 ] + rebase_path(_sources, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800356
Wyatt Heplercf184e82021-05-11 18:15:35 -0700357 # Pass in the .json information files for the imported proto libraries.
358 foreach(proto, _import_protos) {
359 _label = get_label_info(proto, "label_no_toolchain") +
360 ".python($pw_protobuf_compiler_TOOLCHAIN)"
361 _file = get_label_info(_label, "target_gen_dir") + "/" +
362 get_label_info(_label, "name") + ".json"
363 args += [
364 "--proto-library",
Michael Spangc8b93902021-05-30 15:53:56 -0400365 rebase_path(_file, root_build_dir),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700366 ]
367 }
368
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800369 if (defined(invoker._pw_module_as_package) &&
370 invoker._pw_module_as_package) {
371 args += [ "--module-as-package" ]
372 }
373
Wyatt Hepler0e1f5e42021-03-16 22:51:57 -0700374 inputs = [ "$_setup_dir/setup.json" ]
375
Wyatt Heplercf184e82021-05-11 18:15:35 -0700376 public_deps = [ ":$target_name._mirror_sources_to_out_dir" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800377
378 outputs = _setup_sources
379 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800380 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800381 # If the package is not generated, use an input group for the sources.
382 pw_input_group(target_name) {
383 inputs = _all_py_files
384 if (defined(invoker.inputs)) {
385 inputs += invoker.inputs
386 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800387
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800388 deps = _python_deps + _other_deps
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800389 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700390 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700391
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800392 if (_is_package) {
393 # Install this Python package and its dependencies in the current Python
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700394 # environment using pip.
395 pw_python_action("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800396 module = "pip"
397 public_deps = []
398
399 args = [ "install" ]
400
Michael Spang01faaf72021-06-09 22:38:40 -0400401 inputs = pw_build_PIP_CONSTRAINTS
402 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
403 args += [
404 "--constraint",
405 rebase_path(_constraints_file, root_build_dir),
406 ]
407 }
408
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800409 # For generated packages, reinstall when any files change. For regular
410 # packages, only reinstall when setup.py changes.
411 if (_generate_package) {
412 public_deps += [ ":${invoker.target_name}" ]
413 } else {
Michael Spang01faaf72021-06-09 22:38:40 -0400414 inputs += invoker.setup
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800415
416 # Install with --editable since the complete package is in source.
417 args += [ "--editable" ]
418 }
419
Michael Spangc8b93902021-05-30 15:53:56 -0400420 args += [ rebase_path(_setup_dir, root_build_dir) ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800421
422 stamp = true
423
424 # Parallel pip installations don't work, so serialize pip invocations.
Michael Spangda057f72021-06-10 00:50:36 -0400425 pool = "$dir_pw_build/pool:pip($default_toolchain)"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800426
427 foreach(dep, _python_deps) {
428 # We need to add a suffix to the target name, but the label is
429 # formatted as "//path/to:target(toolchain)", so we can't just append
430 # ".subtarget". Instead, we replace the opening parenthesis of the
431 # toolchain with ".suffix(".
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700432 public_deps += [ string_replace(dep, "(", "._run_pip_install(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800433 }
434 }
435
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800436 # Builds a Python wheel for this package. Records the output directory
437 # in the pw_python_package_wheels metadata key.
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700438 pw_python_action("$target_name._build_wheel") {
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800439 metadata = {
440 pw_python_package_wheels = [ "$target_out_dir/$target_name" ]
441 }
442
443 module = "build"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800444
Michael Spangc8b93902021-05-30 15:53:56 -0400445 args =
446 [
447 rebase_path(_setup_dir, root_build_dir),
448 "--wheel",
449 "--no-isolation",
450 "--outdir",
451 ] + rebase_path(metadata.pw_python_package_wheels, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800452
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800453 deps = [ ":${invoker.target_name}" ]
454 foreach(dep, _python_deps) {
455 deps += [ string_replace(dep, "(", ".wheel(") ]
456 }
457
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800458 stamp = true
459 }
460 } else {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700461 # Stubs for non-package targets.
462 group("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800463 }
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700464 group("$target_name._build_wheel") {
465 }
466 }
467
468 # Create the .install and .wheel targets. To limit unnecessary pip
469 # executions, non-generated packages are only reinstalled when their
470 # setup.py changes. However, targets that depend on the .install subtarget
471 # re-run whenever any source files change.
472 #
473 # These targets just represent the source files if this isn't a package.
474 group("$target_name.install") {
475 public_deps = [ ":${invoker.target_name}" ]
476
477 if (_is_package) {
478 public_deps += [ ":${invoker.target_name}._run_pip_install" ]
479 }
480
481 foreach(dep, _python_deps) {
482 public_deps += [ string_replace(dep, "(", ".install(") ]
483 }
484 }
485
486 group("$target_name.wheel") {
487 public_deps = [ ":${invoker.target_name}.install" ]
488
489 if (_is_package) {
490 public_deps += [ ":${invoker.target_name}._build_wheel" ]
491 }
492
493 foreach(dep, _python_deps) {
494 public_deps += [ string_replace(dep, "(", ".wheel(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800495 }
496 }
497
498 # Define the static analysis targets for this package.
499 group("$target_name.lint") {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700500 deps = []
501 foreach(_tool, _supported_static_analysis_tools) {
502 deps += [ ":${invoker.target_name}.lint.$_tool" ]
503 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800504 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700505
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700506 if (_static_analysis != [] || _test_sources != []) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800507 # All packages to install for either general use or test running.
508 _test_install_deps = [ ":$target_name.install" ]
509 foreach(dep, _python_test_deps + [ "$dir_pw_build:python_lint" ]) {
510 _test_install_deps += [ string_replace(dep, "(", ".install(") ]
511 }
512 }
513
514 # For packages that are not generated, create targets to run mypy and pylint.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700515 foreach(_tool, _static_analysis) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800516 # Run lint tools from the setup or target directory so that the tools detect
517 # config files (e.g. pylintrc or mypy.ini) in that directory. Config files
518 # may be explicitly specified with the pylintrc or mypy_ini arguments.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700519 target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800520 sources = _all_py_files
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700521 deps = _test_install_deps
522 python_deps = _python_deps
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800523
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700524 _optional_variables = [
525 "mypy_ini",
526 "pylintrc",
527 ]
528 forward_variables_from(invoker, _optional_variables)
529 not_needed(_optional_variables)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800530 }
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700531 }
532
533 foreach(_unused_tool, _supported_static_analysis_tools - _static_analysis) {
534 pw_input_group("$target_name.lint.$_unused_tool") {
535 inputs = []
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800536 if (defined(invoker.pylintrc)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700537 inputs += [ invoker.pylintrc ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800538 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800539 if (defined(invoker.mypy_ini)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700540 inputs += [ invoker.mypy_ini ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800541 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800542 }
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800543
544 # Generated packages with linting disabled never need the whole file list.
545 not_needed([ "_all_py_files" ])
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700546 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800547 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800548 # Create groups with the public target names ($target_name, $target_name.lint,
549 # $target_name.install, etc.). These are actually wrappers around internal
550 # Python actions instantiated with the default toolchain. This ensures there
551 # is only a single copy of each Python action in the build.
552 #
553 # The $target_name.tests group is created separately below.
554 group("$target_name") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700555 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800556 }
557
558 foreach(subtarget, pw_python_package_subtargets - [ "tests" ]) {
559 group("$target_name.$subtarget") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700560 deps =
561 [ ":${invoker.target_name}.$subtarget($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800562 }
Wyatt Heplerb8e13602020-10-19 11:20:36 -0700563 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800564
565 # Everything Python-related is only instantiated in the default toolchain.
566 # Silence not-needed warnings except for in the default toolchain.
567 not_needed("*")
568 not_needed(invoker, "*")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700569 }
570
571 # Create a target for each test file.
572 _test_targets = []
573
574 foreach(test, _test_sources) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800575 if (_is_package) {
576 _name = rebase_path(test, _setup_dir)
577 } else {
578 _name = test
579 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700580
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800581 _test_target = "$target_name.tests." + string_replace(_name, "/", "_")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700582
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700583 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800584 pw_python_action(_test_target) {
585 script = test
586 stamp = true
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800587
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800588 deps = _test_install_deps
589
590 foreach(dep, _python_test_deps) {
591 deps += [ string_replace(dep, "(", ".tests(") ]
592 }
593 }
594 } else {
595 # Create a public version of each test target, so tests can be executed as
596 # //path/to:package.tests.foo.py.
597 group(_test_target) {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700598 deps = [ ":$_test_target($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700599 }
600 }
601
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800602 _test_targets += [ ":$_test_target" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700603 }
604
605 group("$target_name.tests") {
606 deps = _test_targets
607 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700608
609 _pw_create_aliases_if_name_matches_directory(target_name) {
610 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700611}
612
613# Declares a group of Python packages or other Python groups. pw_python_groups
614# expose the same set of subtargets as pw_python_package (e.g.
615# "$group_name.lint" and "$group_name.tests"), but these apply to all packages
616# in deps and their dependencies.
617template("pw_python_group") {
618 if (defined(invoker.python_deps)) {
619 _python_deps = invoker.python_deps
620 } else {
621 _python_deps = []
Wyatt Heplera3ca62a2021-05-04 16:21:43 -0700622 not_needed([ "invoker" ]) # Allow empty groups.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700623 }
624
625 group(target_name) {
626 deps = _python_deps
Wyatt Heplercf184e82021-05-11 18:15:35 -0700627
628 if (defined(invoker.other_deps)) {
629 deps += invoker.other_deps
630 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700631 }
632
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800633 foreach(subtarget, pw_python_package_subtargets) {
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700634 group("$target_name.$subtarget") {
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700635 public_deps = []
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700636 foreach(dep, _python_deps) {
637 # Split out the toolchain to support deps with a toolchain specified.
638 _target = get_label_info(dep, "label_no_toolchain")
639 _toolchain = get_label_info(dep, "toolchain")
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700640 public_deps += [ "$_target.$subtarget($_toolchain)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700641 }
642 }
643 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700644
645 _pw_create_aliases_if_name_matches_directory(target_name) {
646 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700647}
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700648
649# Declares Python scripts or tests that are not part of a Python package.
650# Similar to pw_python_package, but only supports a subset of its features.
651#
652# pw_python_script accepts the same arguments as pw_python_package, except
653# `setup` cannot be provided.
654#
655# pw_python_script provides the same subtargets as pw_python_package, but
656# $target_name.install and $target_name.wheel only affect the python_deps of
657# this GN target, not the target itself.
Wyatt Hepler473efe02021-05-04 14:15:16 -0700658#
659# pw_python_script allows creating a pw_python_action associated with the
660# script. This is provided by passing an 'action' scope to pw_python_script.
661# This functions like a normal action, with a few additions: the action uses the
662# pw_python_script's python_deps and defaults to using the source file as its
663# 'script' argument, if there is only a single source file.
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700664template("pw_python_script") {
Wyatt Hepler473efe02021-05-04 14:15:16 -0700665 _package_variables = [
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700666 "sources",
667 "tests",
668 "python_deps",
669 "other_deps",
670 "inputs",
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800671 "pylintrc",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700672 "mypy_ini",
673 "static_analysis",
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700674 ]
675
676 pw_python_package(target_name) {
677 _pw_standalone = true
Wyatt Hepler473efe02021-05-04 14:15:16 -0700678 forward_variables_from(invoker, _package_variables)
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700679 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700680
681 _pw_create_aliases_if_name_matches_directory(target_name) {
682 }
Wyatt Hepler473efe02021-05-04 14:15:16 -0700683
684 if (defined(invoker.action)) {
685 pw_python_action("$target_name.action") {
686 forward_variables_from(invoker.action, "*", [ "python_deps" ])
687 python_deps = [ ":${invoker.target_name}" ]
688
689 if (!defined(script) && !defined(module) && defined(invoker.sources)) {
690 _sources = invoker.sources
691 assert(_sources != [] && _sources == [ _sources[0] ],
692 "'script' must be specified unless there is only one source " +
693 "in 'sources'")
694 script = _sources[0]
695 }
696 }
697 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700698}
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800699
700# Represents a list of Python requirements, as in a requirements.txt.
701#
702# Args:
703# files: One or more requirements.txt files.
704# requirements: A list of requirements.txt-style requirements.
705template("pw_python_requirements") {
706 assert(defined(invoker.files) || defined(invoker.requirements),
707 "pw_python_requirements requires a list of requirements.txt files " +
708 "in the 'files' arg or requirements in 'requirements'")
709
710 _requirements_files = []
711
712 if (defined(invoker.files)) {
713 _requirements_files += invoker.files
714 }
715
716 if (defined(invoker.requirements)) {
717 _requirements_file = "$target_gen_dir/$target_name.requirements.txt"
718 write_file(_requirements_file, invoker.requirements)
719 _requirements_files += [ _requirements_file ]
720 }
721
722 # The default target represents the requirements themselves.
723 pw_input_group(target_name) {
724 inputs = _requirements_files
725 }
726
727 # Use the same subtargets as pw_python_package so these targets can be listed
728 # as python_deps of pw_python_packages.
729 pw_python_action("$target_name.install") {
730 inputs = _requirements_files
731
732 module = "pip"
733 args = [ "install" ]
734
735 foreach(_requirements_file, inputs) {
736 args += [
737 "--requirement",
Michael Spangc8b93902021-05-30 15:53:56 -0400738 rebase_path(_requirements_file, root_build_dir),
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800739 ]
740 }
741
Michael Spang01faaf72021-06-09 22:38:40 -0400742 inputs += pw_build_PIP_CONSTRAINTS
743 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
744 args += [
745 "--constraint",
746 rebase_path(_constraints_file, root_build_dir),
747 ]
748 }
749
Michael Spangda057f72021-06-10 00:50:36 -0400750 pool = "$dir_pw_build/pool:pip($default_toolchain)"
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800751 stamp = true
752 }
753
754 # Create stubs for the unused subtargets so that pw_python_requirements can be
755 # used as python_deps.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800756 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) {
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800757 group("$target_name.$subtarget") {
758 }
759 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700760
761 _pw_create_aliases_if_name_matches_directory(target_name) {
762 }
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800763}