blob: 217a7f8f3638ad2058c4b44516aa81318369ed0a [file] [log] [blame]
Wyatt Hepleraf853ea2021-03-09 19:09:54 -08001# Copyright 2021 The Pigweed Authors
Wyatt Heplerb16dfd32020-10-12 08:46:38 -07002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14
15import("//build_overrides/pigweed.gni")
16
Wyatt Heplerb8e13602020-10-19 11:20:36 -070017import("$dir_pw_build/input_group.gni")
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080018import("$dir_pw_build/mirror_tree.gni")
Wyatt Heplerb8e13602020-10-19 11:20:36 -070019import("$dir_pw_build/python_action.gni")
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070020import("$dir_pw_protobuf_compiler/toolchain.gni")
21
22declare_args() {
23 # Python tasks, such as running tests and Pylint, are done in a single GN
24 # toolchain to avoid unnecessary duplication in the build.
25 pw_build_PYTHON_TOOLCHAIN = "$dir_pw_build/python_toolchain:python"
26}
Wyatt Heplerb16dfd32020-10-12 08:46:38 -070027
Wyatt Hepler438caa02021-01-15 17:13:11 -080028# Python packages provide the following targets as $target_name.$subtarget.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080029pw_python_package_subtargets = [
Wyatt Hepler438caa02021-01-15 17:13:11 -080030 "tests",
31 "lint",
32 "lint.mypy",
33 "lint.pylint",
34 "install",
35 "wheel",
Wyatt Hepler446f24c2021-03-24 09:29:35 -070036
37 # Internal targets that directly depend on one another.
38 "_run_pip_install",
39 "_build_wheel",
Wyatt Hepler438caa02021-01-15 17:13:11 -080040]
41
Wyatt Heplerb85cda42021-04-07 13:13:15 -070042# Create aliases for subsargets when the target name matches the directory name.
43# This allows //foo:foo.tests to be accessed as //foo:tests, for example.
44template("_pw_create_aliases_if_name_matches_directory") {
45 not_needed([ "invoker" ])
46
47 if (get_label_info(":$target_name", "name") ==
48 get_path_info(get_label_info(":$target_name", "dir"), "name")) {
49 foreach(subtarget, pw_python_package_subtargets) {
50 group(subtarget) {
51 public_deps = [ ":${invoker.target_name}.$subtarget" ]
52 }
53 }
54 }
55}
56
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070057# Internal template that runs Mypy.
58template("_pw_python_static_analysis_mypy") {
59 pw_python_action(target_name) {
60 module = "mypy"
61 args = [
62 "--pretty",
63 "--show-error-codes",
64 ]
65
66 if (defined(invoker.mypy_ini)) {
Michael Spangc8b93902021-05-30 15:53:56 -040067 args +=
68 [ "--config-file=" + rebase_path(invoker.mypy_ini, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070069 inputs = [ invoker.mypy_ini ]
70 }
71
Michael Spangc8b93902021-05-30 15:53:56 -040072 args += rebase_path(invoker.sources, root_build_dir)
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070073
74 # Use this environment variable to force mypy to colorize output.
75 # See https://github.com/python/mypy/issues/7771
76 environment = [ "MYPY_FORCE_COLOR=1" ]
77
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070078 stamp = true
79
80 deps = invoker.deps
81
82 foreach(dep, invoker.python_deps) {
83 deps += [ string_replace(dep, "(", ".lint.mypy(") ]
84 }
85 }
86}
87
88# Internal template that runs Pylint.
89template("_pw_python_static_analysis_pylint") {
90 # Create a target to run pylint on each of the Python files in this
91 # package and its dependencies.
92 pw_python_action_foreach(target_name) {
93 module = "pylint"
94 args = [
Michael Spangc8b93902021-05-30 15:53:56 -040095 rebase_path(".", root_build_dir) + "/{{source_target_relative}}",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070096 "--jobs=1",
97 "--output-format=colorized",
98 ]
99
100 if (defined(invoker.pylintrc)) {
Michael Spangc8b93902021-05-30 15:53:56 -0400101 args += [ "--rcfile=" + rebase_path(invoker.pylintrc, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700102 inputs = [ invoker.pylintrc ]
103 }
104
105 if (host_os == "win") {
106 # Allow CRLF on Windows, in case Git is set to switch line endings.
107 args += [ "--disable=unexpected-line-ending-format" ]
108 }
109
110 sources = invoker.sources
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700111
112 stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"
113
114 public_deps = invoker.deps
115
116 foreach(dep, invoker.python_deps) {
117 public_deps += [ string_replace(dep, "(", ".lint.pylint(") ]
118 }
119 }
120}
121
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700122# Defines a Python package. GN Python packages contain several GN targets:
123#
124# - $name - Provides the Python files in the build, but does not take any
125# actions. All subtargets depend on this target.
126# - $name.lint - Runs static analyis tools on the Python code. This is a group
127# of two subtargets:
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800128# - $name.lint.mypy - Runs mypy (if enabled).
129# - $name.lint.pylint - Runs pylint (if enabled).
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700130# - $name.tests - Runs all tests for this package.
Rob Mohrfcf60372020-11-04 11:41:36 -0800131# - $name.install - Installs the package in a venv.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800132# - $name.wheel - Builds a Python wheel for the package.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700133#
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700134# All Python packages are instantiated with in pw_build_PYTHON_TOOLCHAIN,
135# regardless of the current toolchain. This prevents Python-specific work, like
136# running Pylint, from occurring multiple times in a build.
Wyatt Hepler438caa02021-01-15 17:13:11 -0800137#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700138# Args:
139# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
140# which must all be in the same directory.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800141# generate_setup: As an alternative to 'setup', generate setup files with the
142# keywords in this scope. 'name' is required.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700143# sources: Python sources files in the package.
144# tests: Test files for this Python package.
145# python_deps: Dependencies on other pw_python_packages in the GN build.
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800146# python_test_deps: Test-only pw_python_package dependencies.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700147# other_deps: Dependencies on GN targets that are not pw_python_packages.
148# inputs: Other files to track, such as package_data.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800149# proto_library: A pw_proto_library target to embed in this Python package.
150# generate_setup is required in place of setup if proto_library is used.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700151# static_analysis: List of static analysis tools to run; "*" (default) runs
152# all tools. The supported tools are "mypy" and "pylint".
Michael Spangc8b93902021-05-30 15:53:56 -0400153# pylintrc: Path to a pylintrc configuration file to use. If not
154# provided, Pylint's default rcfile search is used. As this may
155# use the the local user's configuration file, it is highly
156# recommended to pass this option to specify the rcfile explicitly.
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800157# mypy_ini: Optional path to a mypy configuration file to use. If not
158# provided, mypy's default configuration file search is used. mypy is
159# executed from the package's setup directory, so mypy.ini files in that
160# directory will take precedence over others.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800161#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700162template("pw_python_package") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700163 # The Python targets are always instantiated in pw_build_PYTHON_TOOLCHAIN. Use
Wyatt Hepler752d7d32021-03-02 09:02:23 -0800164 # fully qualified labels so that the toolchain is not lost.
165 _other_deps = []
166 if (defined(invoker.other_deps)) {
167 foreach(dep, invoker.other_deps) {
168 _other_deps += [ get_label_info(dep, "label_with_toolchain") ]
169 }
170 }
171
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800172 _python_deps = []
173 if (defined(invoker.python_deps)) {
174 foreach(dep, invoker.python_deps) {
175 _python_deps += [ get_label_info(dep, "label_with_toolchain") ]
176 }
177 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700178
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700179 # pw_python_script uses pw_python_package, but with a limited set of features.
180 # _pw_standalone signals that this target is actually a pw_python_script.
181 _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone)
182
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800183 _generate_package = false
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800184
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800185 # Check the generate_setup and import_protos args to determine if this package
186 # is generated.
187 if (_is_package) {
188 assert(defined(invoker.generate_setup) != defined(invoker.setup),
189 "Either 'setup' or 'generate_setup' (but not both) must provided")
190
191 if (defined(invoker.proto_library)) {
192 assert(invoker.proto_library != "", "'proto_library' cannot be empty")
193 assert(defined(invoker.generate_setup),
194 "Python packages that import protos with 'proto_library' must " +
195 "use 'generate_setup' instead of 'setup'")
196
197 _import_protos = [ invoker.proto_library ]
Wyatt Heplercf184e82021-05-11 18:15:35 -0700198
199 # Depend on the dependencies of the proto library.
200 _proto = get_label_info(invoker.proto_library, "label_no_toolchain")
201 _toolchain = get_label_info(invoker.proto_library, "toolchain")
202 _python_deps += [ "$_proto.python._deps($_toolchain)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800203 } else if (defined(invoker.generate_setup)) {
204 _import_protos = []
205 }
206
207 if (defined(invoker.generate_setup)) {
208 _generate_package = true
209 _setup_dir = "$target_gen_dir/$target_name.generated_python_package"
210
211 if (defined(invoker.strip_prefix)) {
212 _source_root = invoker.strip_prefix
213 } else {
214 _source_root = "."
215 }
216 } else {
217 # Non-generated packages with sources provided need an __init__.py.
218 assert(!defined(invoker.sources) || invoker.sources == [] ||
219 filter_include(invoker.sources, [ "*\b__init__.py" ]) != [],
220 "Python packages must have at least one __init__.py file")
221
222 # Get the directories of the setup files. All must be in the same dir.
223 _setup_dirs = get_path_info(invoker.setup, "dir")
224 _setup_dir = _setup_dirs[0]
225
226 foreach(dir, _setup_dirs) {
227 assert(dir == _setup_dir,
228 "All files in 'setup' must be in the same directory")
229 }
230
231 assert(!defined(invoker.strip_prefix),
232 "'strip_prefix' may only be given if 'generate_setup' is provided")
233 }
234 }
235
236 # Process arguments defaults and set defaults.
237
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700238 _supported_static_analysis_tools = [
239 "mypy",
240 "pylint",
241 ]
242 not_needed([ "_supported_static_analysis_tools" ])
243
244 # Argument: static_analysis (list of tool names or "*"); default = "*" (all)
245 if (!defined(invoker.static_analysis) || invoker.static_analysis == "*") {
246 _static_analysis = _supported_static_analysis_tools
Keir Mierlee65ddc82020-12-02 17:59:52 -0800247 } else {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700248 _static_analysis = invoker.static_analysis
249 }
250
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700251 foreach(_tool, _static_analysis) {
252 assert(_supported_static_analysis_tools + [ _tool ] - [ _tool ] !=
253 _supported_static_analysis_tools,
254 "'$_tool' is not a supported static analysis tool")
Keir Mierlee65ddc82020-12-02 17:59:52 -0800255 }
256
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800257 # Argument: sources (list)
258 _sources = []
259 if (defined(invoker.sources)) {
260 if (_generate_package) {
261 foreach(source, rebase_path(invoker.sources, _source_root)) {
262 _sources += [ "$_setup_dir/$source" ]
263 }
264 } else {
265 _sources += invoker.sources
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700266 }
267 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700268
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800269 # Argument: tests (list)
270 _test_sources = []
271 if (defined(invoker.tests)) {
272 if (_generate_package) {
273 foreach(source, rebase_path(invoker.tests, _source_root)) {
274 _test_sources += [ "$_setup_dir/$source" ]
275 }
276 } else {
277 _test_sources += invoker.tests
Wyatt Heplerd7dc6552020-10-21 16:16:21 -0700278 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700279 }
280
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800281 # Argument: setup (list)
282 _setup_sources = []
283 if (defined(invoker.setup)) {
284 _setup_sources = invoker.setup
285 } else if (_generate_package) {
286 _setup_sources = [ "$_setup_dir/setup.py" ]
287 }
288
289 # Argument: python_test_deps (list)
290 _python_test_deps = _python_deps # include all deps in test deps
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800291 if (defined(invoker.python_test_deps)) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800292 foreach(dep, invoker.python_test_deps) {
293 _python_test_deps += [ get_label_info(dep, "label_with_toolchain") ]
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800294 }
295 }
296
297 if (_test_sources == []) {
298 assert(!defined(invoker.python_test_deps),
299 "python_test_deps was provided, but there are no tests in " +
300 get_label_info(":$target_name", "label_no_toolchain"))
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800301 not_needed([ "_python_test_deps" ])
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800302 }
303
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800304 _all_py_files = _sources + _test_sources + _setup_sources
Wyatt Hepler438caa02021-01-15 17:13:11 -0800305
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700306 # The pw_python_package subtargets are only instantiated in
307 # pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the
308 # targets in this toolchain.
309 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800310 # Declare the main Python package group. This represents the Python files,
311 # but does not take any actions. GN targets can depend on the package name
312 # to run when any files in the package change.
313 if (_generate_package) {
314 # If this package is generated, mirror the sources to the final directory.
315 pw_mirror_tree("$target_name._mirror_sources_to_out_dir") {
316 directory = _setup_dir
Wyatt Hepler438caa02021-01-15 17:13:11 -0800317
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800318 sources = []
319 if (defined(invoker.sources)) {
320 sources += invoker.sources
321 }
322 if (defined(invoker.tests)) {
323 sources += invoker.tests
324 }
Wyatt Heplerde20d742021-06-02 23:34:14 -0700325 if (defined(invoker.inputs)) {
326 sources += invoker.inputs
327 }
Wyatt Hepler438caa02021-01-15 17:13:11 -0800328
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800329 source_root = _source_root
330 public_deps = _python_deps + _other_deps
Wyatt Hepler285636f2020-12-15 13:13:11 -0800331 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700332
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800333 # Get generated_setup scope and write it to disk ask JSON.
334 _gen_setup = invoker.generate_setup
335 assert(defined(_gen_setup.name), "'name' is required in generate_package")
336 assert(!defined(_gen_setup.packages) && !defined(_gen_setup.package_data),
337 "'packages' and 'package_data' may not be provided " +
338 "in 'generate_package'")
339 write_file("$_setup_dir/setup.json", _gen_setup, "json")
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800340
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800341 # Generate the setup.py, py.typed, and __init__.py files as needed.
342 action(target_name) {
343 script = "$dir_pw_build/py/pw_build/generate_python_package.py"
344 args = [
345 "--label",
346 get_label_info(":$target_name", "label_no_toolchain"),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700347 "--generated-root",
Michael Spangc8b93902021-05-30 15:53:56 -0400348 rebase_path(_setup_dir, root_build_dir),
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800349 "--setup-json",
Michael Spangc8b93902021-05-30 15:53:56 -0400350 rebase_path("$_setup_dir/setup.json", root_build_dir),
351 ] + rebase_path(_sources, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800352
Wyatt Heplercf184e82021-05-11 18:15:35 -0700353 # Pass in the .json information files for the imported proto libraries.
354 foreach(proto, _import_protos) {
355 _label = get_label_info(proto, "label_no_toolchain") +
356 ".python($pw_protobuf_compiler_TOOLCHAIN)"
357 _file = get_label_info(_label, "target_gen_dir") + "/" +
358 get_label_info(_label, "name") + ".json"
359 args += [
360 "--proto-library",
Michael Spangc8b93902021-05-30 15:53:56 -0400361 rebase_path(_file, root_build_dir),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700362 ]
363 }
364
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800365 if (defined(invoker._pw_module_as_package) &&
366 invoker._pw_module_as_package) {
367 args += [ "--module-as-package" ]
368 }
369
Wyatt Hepler0e1f5e42021-03-16 22:51:57 -0700370 inputs = [ "$_setup_dir/setup.json" ]
371
Wyatt Heplercf184e82021-05-11 18:15:35 -0700372 public_deps = [ ":$target_name._mirror_sources_to_out_dir" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800373
374 outputs = _setup_sources
375 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800376 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800377 # If the package is not generated, use an input group for the sources.
378 pw_input_group(target_name) {
379 inputs = _all_py_files
380 if (defined(invoker.inputs)) {
381 inputs += invoker.inputs
382 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800383
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800384 deps = _python_deps + _other_deps
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800385 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700386 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700387
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800388 if (_is_package) {
389 # Install this Python package and its dependencies in the current Python
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700390 # environment using pip.
391 pw_python_action("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800392 module = "pip"
393 public_deps = []
394
395 args = [ "install" ]
396
397 # For generated packages, reinstall when any files change. For regular
398 # packages, only reinstall when setup.py changes.
399 if (_generate_package) {
400 public_deps += [ ":${invoker.target_name}" ]
401 } else {
402 inputs = invoker.setup
403
404 # Install with --editable since the complete package is in source.
405 args += [ "--editable" ]
406 }
407
Michael Spangc8b93902021-05-30 15:53:56 -0400408 args += [ rebase_path(_setup_dir, root_build_dir) ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800409
410 stamp = true
411
412 # Parallel pip installations don't work, so serialize pip invocations.
413 pool = "$dir_pw_build:pip_pool"
414
415 foreach(dep, _python_deps) {
416 # We need to add a suffix to the target name, but the label is
417 # formatted as "//path/to:target(toolchain)", so we can't just append
418 # ".subtarget". Instead, we replace the opening parenthesis of the
419 # toolchain with ".suffix(".
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700420 public_deps += [ string_replace(dep, "(", "._run_pip_install(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800421 }
422 }
423
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800424 # Builds a Python wheel for this package. Records the output directory
425 # in the pw_python_package_wheels metadata key.
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700426 pw_python_action("$target_name._build_wheel") {
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800427 metadata = {
428 pw_python_package_wheels = [ "$target_out_dir/$target_name" ]
429 }
430
431 module = "build"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800432
Michael Spangc8b93902021-05-30 15:53:56 -0400433 args =
434 [
435 rebase_path(_setup_dir, root_build_dir),
436 "--wheel",
437 "--no-isolation",
438 "--outdir",
439 ] + rebase_path(metadata.pw_python_package_wheels, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800440
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800441 deps = [ ":${invoker.target_name}" ]
442 foreach(dep, _python_deps) {
443 deps += [ string_replace(dep, "(", ".wheel(") ]
444 }
445
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800446 stamp = true
447 }
448 } else {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700449 # Stubs for non-package targets.
450 group("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800451 }
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700452 group("$target_name._build_wheel") {
453 }
454 }
455
456 # Create the .install and .wheel targets. To limit unnecessary pip
457 # executions, non-generated packages are only reinstalled when their
458 # setup.py changes. However, targets that depend on the .install subtarget
459 # re-run whenever any source files change.
460 #
461 # These targets just represent the source files if this isn't a package.
462 group("$target_name.install") {
463 public_deps = [ ":${invoker.target_name}" ]
464
465 if (_is_package) {
466 public_deps += [ ":${invoker.target_name}._run_pip_install" ]
467 }
468
469 foreach(dep, _python_deps) {
470 public_deps += [ string_replace(dep, "(", ".install(") ]
471 }
472 }
473
474 group("$target_name.wheel") {
475 public_deps = [ ":${invoker.target_name}.install" ]
476
477 if (_is_package) {
478 public_deps += [ ":${invoker.target_name}._build_wheel" ]
479 }
480
481 foreach(dep, _python_deps) {
482 public_deps += [ string_replace(dep, "(", ".wheel(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800483 }
484 }
485
486 # Define the static analysis targets for this package.
487 group("$target_name.lint") {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700488 deps = []
489 foreach(_tool, _supported_static_analysis_tools) {
490 deps += [ ":${invoker.target_name}.lint.$_tool" ]
491 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800492 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700493
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700494 if (_static_analysis != [] || _test_sources != []) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800495 # All packages to install for either general use or test running.
496 _test_install_deps = [ ":$target_name.install" ]
497 foreach(dep, _python_test_deps + [ "$dir_pw_build:python_lint" ]) {
498 _test_install_deps += [ string_replace(dep, "(", ".install(") ]
499 }
500 }
501
502 # For packages that are not generated, create targets to run mypy and pylint.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700503 foreach(_tool, _static_analysis) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800504 # Run lint tools from the setup or target directory so that the tools detect
505 # config files (e.g. pylintrc or mypy.ini) in that directory. Config files
506 # may be explicitly specified with the pylintrc or mypy_ini arguments.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700507 target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800508 sources = _all_py_files
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700509 deps = _test_install_deps
510 python_deps = _python_deps
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800511
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700512 _optional_variables = [
513 "mypy_ini",
514 "pylintrc",
515 ]
516 forward_variables_from(invoker, _optional_variables)
517 not_needed(_optional_variables)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800518 }
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700519 }
520
521 foreach(_unused_tool, _supported_static_analysis_tools - _static_analysis) {
522 pw_input_group("$target_name.lint.$_unused_tool") {
523 inputs = []
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800524 if (defined(invoker.pylintrc)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700525 inputs += [ invoker.pylintrc ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800526 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800527 if (defined(invoker.mypy_ini)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700528 inputs += [ invoker.mypy_ini ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800529 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800530 }
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800531
532 # Generated packages with linting disabled never need the whole file list.
533 not_needed([ "_all_py_files" ])
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700534 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800535 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800536 # Create groups with the public target names ($target_name, $target_name.lint,
537 # $target_name.install, etc.). These are actually wrappers around internal
538 # Python actions instantiated with the default toolchain. This ensures there
539 # is only a single copy of each Python action in the build.
540 #
541 # The $target_name.tests group is created separately below.
542 group("$target_name") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700543 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800544 }
545
546 foreach(subtarget, pw_python_package_subtargets - [ "tests" ]) {
547 group("$target_name.$subtarget") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700548 deps =
549 [ ":${invoker.target_name}.$subtarget($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800550 }
Wyatt Heplerb8e13602020-10-19 11:20:36 -0700551 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800552
553 # Everything Python-related is only instantiated in the default toolchain.
554 # Silence not-needed warnings except for in the default toolchain.
555 not_needed("*")
556 not_needed(invoker, "*")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700557 }
558
559 # Create a target for each test file.
560 _test_targets = []
561
562 foreach(test, _test_sources) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800563 if (_is_package) {
564 _name = rebase_path(test, _setup_dir)
565 } else {
566 _name = test
567 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700568
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800569 _test_target = "$target_name.tests." + string_replace(_name, "/", "_")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700570
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700571 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800572 pw_python_action(_test_target) {
573 script = test
574 stamp = true
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800575
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800576 deps = _test_install_deps
577
578 foreach(dep, _python_test_deps) {
579 deps += [ string_replace(dep, "(", ".tests(") ]
580 }
581 }
582 } else {
583 # Create a public version of each test target, so tests can be executed as
584 # //path/to:package.tests.foo.py.
585 group(_test_target) {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700586 deps = [ ":$_test_target($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700587 }
588 }
589
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800590 _test_targets += [ ":$_test_target" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700591 }
592
593 group("$target_name.tests") {
594 deps = _test_targets
595 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700596
597 _pw_create_aliases_if_name_matches_directory(target_name) {
598 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700599}
600
601# Declares a group of Python packages or other Python groups. pw_python_groups
602# expose the same set of subtargets as pw_python_package (e.g.
603# "$group_name.lint" and "$group_name.tests"), but these apply to all packages
604# in deps and their dependencies.
605template("pw_python_group") {
606 if (defined(invoker.python_deps)) {
607 _python_deps = invoker.python_deps
608 } else {
609 _python_deps = []
Wyatt Heplera3ca62a2021-05-04 16:21:43 -0700610 not_needed([ "invoker" ]) # Allow empty groups.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700611 }
612
613 group(target_name) {
614 deps = _python_deps
Wyatt Heplercf184e82021-05-11 18:15:35 -0700615
616 if (defined(invoker.other_deps)) {
617 deps += invoker.other_deps
618 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700619 }
620
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800621 foreach(subtarget, pw_python_package_subtargets) {
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700622 group("$target_name.$subtarget") {
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700623 public_deps = []
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700624 foreach(dep, _python_deps) {
625 # Split out the toolchain to support deps with a toolchain specified.
626 _target = get_label_info(dep, "label_no_toolchain")
627 _toolchain = get_label_info(dep, "toolchain")
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700628 public_deps += [ "$_target.$subtarget($_toolchain)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700629 }
630 }
631 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700632
633 _pw_create_aliases_if_name_matches_directory(target_name) {
634 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700635}
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700636
637# Declares Python scripts or tests that are not part of a Python package.
638# Similar to pw_python_package, but only supports a subset of its features.
639#
640# pw_python_script accepts the same arguments as pw_python_package, except
641# `setup` cannot be provided.
642#
643# pw_python_script provides the same subtargets as pw_python_package, but
644# $target_name.install and $target_name.wheel only affect the python_deps of
645# this GN target, not the target itself.
Wyatt Hepler473efe02021-05-04 14:15:16 -0700646#
647# pw_python_script allows creating a pw_python_action associated with the
648# script. This is provided by passing an 'action' scope to pw_python_script.
649# This functions like a normal action, with a few additions: the action uses the
650# pw_python_script's python_deps and defaults to using the source file as its
651# 'script' argument, if there is only a single source file.
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700652template("pw_python_script") {
Wyatt Hepler473efe02021-05-04 14:15:16 -0700653 _package_variables = [
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700654 "sources",
655 "tests",
656 "python_deps",
657 "other_deps",
658 "inputs",
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800659 "pylintrc",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700660 "mypy_ini",
661 "static_analysis",
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700662 ]
663
664 pw_python_package(target_name) {
665 _pw_standalone = true
Wyatt Hepler473efe02021-05-04 14:15:16 -0700666 forward_variables_from(invoker, _package_variables)
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700667 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700668
669 _pw_create_aliases_if_name_matches_directory(target_name) {
670 }
Wyatt Hepler473efe02021-05-04 14:15:16 -0700671
672 if (defined(invoker.action)) {
673 pw_python_action("$target_name.action") {
674 forward_variables_from(invoker.action, "*", [ "python_deps" ])
675 python_deps = [ ":${invoker.target_name}" ]
676
677 if (!defined(script) && !defined(module) && defined(invoker.sources)) {
678 _sources = invoker.sources
679 assert(_sources != [] && _sources == [ _sources[0] ],
680 "'script' must be specified unless there is only one source " +
681 "in 'sources'")
682 script = _sources[0]
683 }
684 }
685 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700686}
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800687
688# Represents a list of Python requirements, as in a requirements.txt.
689#
690# Args:
691# files: One or more requirements.txt files.
692# requirements: A list of requirements.txt-style requirements.
693template("pw_python_requirements") {
694 assert(defined(invoker.files) || defined(invoker.requirements),
695 "pw_python_requirements requires a list of requirements.txt files " +
696 "in the 'files' arg or requirements in 'requirements'")
697
698 _requirements_files = []
699
700 if (defined(invoker.files)) {
701 _requirements_files += invoker.files
702 }
703
704 if (defined(invoker.requirements)) {
705 _requirements_file = "$target_gen_dir/$target_name.requirements.txt"
706 write_file(_requirements_file, invoker.requirements)
707 _requirements_files += [ _requirements_file ]
708 }
709
710 # The default target represents the requirements themselves.
711 pw_input_group(target_name) {
712 inputs = _requirements_files
713 }
714
715 # Use the same subtargets as pw_python_package so these targets can be listed
716 # as python_deps of pw_python_packages.
717 pw_python_action("$target_name.install") {
718 inputs = _requirements_files
719
720 module = "pip"
721 args = [ "install" ]
722
723 foreach(_requirements_file, inputs) {
724 args += [
725 "--requirement",
Michael Spangc8b93902021-05-30 15:53:56 -0400726 rebase_path(_requirements_file, root_build_dir),
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800727 ]
728 }
729
730 pool = "$dir_pw_build:pip_pool"
731 stamp = true
732 }
733
734 # Create stubs for the unused subtargets so that pw_python_requirements can be
735 # used as python_deps.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800736 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) {
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800737 group("$target_name.$subtarget") {
738 }
739 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700740
741 _pw_create_aliases_if_name_matches_directory(target_name) {
742 }
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800743}