blob: eac49176ab487aee273b819c9e96d87b31f59d4a [file] [log] [blame]
Wyatt Hepleraf853ea2021-03-09 19:09:54 -08001# Copyright 2021 The Pigweed Authors
Wyatt Heplerb16dfd32020-10-12 08:46:38 -07002#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7# https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14
15import("//build_overrides/pigweed.gni")
16
Wyatt Heplerb8e13602020-10-19 11:20:36 -070017import("$dir_pw_build/input_group.gni")
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080018import("$dir_pw_build/mirror_tree.gni")
Wyatt Heplerb8e13602020-10-19 11:20:36 -070019import("$dir_pw_build/python_action.gni")
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070020import("$dir_pw_protobuf_compiler/toolchain.gni")
21
22declare_args() {
23 # Python tasks, such as running tests and Pylint, are done in a single GN
24 # toolchain to avoid unnecessary duplication in the build.
25 pw_build_PYTHON_TOOLCHAIN = "$dir_pw_build/python_toolchain:python"
Michael Spang01faaf72021-06-09 22:38:40 -040026
27 # Constraints file selection (arguments to pip install --constraint).
28 # See pip help install.
Ted Pudlik961706b2021-11-22 22:30:59 +000029 pw_build_PIP_CONSTRAINTS =
30 [ "$dir_pw_env_setup/py/pw_env_setup/virtualenv_setup/constraint.list" ]
Wyatt Heplerf6f74f42021-04-13 19:26:20 -070031}
Wyatt Heplerb16dfd32020-10-12 08:46:38 -070032
Wyatt Hepler438caa02021-01-15 17:13:11 -080033# Python packages provide the following targets as $target_name.$subtarget.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -080034pw_python_package_subtargets = [
Wyatt Hepler438caa02021-01-15 17:13:11 -080035 "tests",
36 "lint",
37 "lint.mypy",
38 "lint.pylint",
39 "install",
40 "wheel",
Wyatt Hepler446f24c2021-03-24 09:29:35 -070041
42 # Internal targets that directly depend on one another.
43 "_run_pip_install",
44 "_build_wheel",
Wyatt Hepler438caa02021-01-15 17:13:11 -080045]
46
Wyatt Heplerb85cda42021-04-07 13:13:15 -070047# Create aliases for subsargets when the target name matches the directory name.
48# This allows //foo:foo.tests to be accessed as //foo:tests, for example.
49template("_pw_create_aliases_if_name_matches_directory") {
50 not_needed([ "invoker" ])
51
52 if (get_label_info(":$target_name", "name") ==
53 get_path_info(get_label_info(":$target_name", "dir"), "name")) {
54 foreach(subtarget, pw_python_package_subtargets) {
55 group(subtarget) {
56 public_deps = [ ":${invoker.target_name}.$subtarget" ]
57 }
58 }
59 }
60}
61
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070062# Internal template that runs Mypy.
63template("_pw_python_static_analysis_mypy") {
64 pw_python_action(target_name) {
65 module = "mypy"
66 args = [
67 "--pretty",
68 "--show-error-codes",
69 ]
70
71 if (defined(invoker.mypy_ini)) {
Michael Spangc8b93902021-05-30 15:53:56 -040072 args +=
73 [ "--config-file=" + rebase_path(invoker.mypy_ini, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070074 inputs = [ invoker.mypy_ini ]
75 }
76
Michael Spangc8b93902021-05-30 15:53:56 -040077 args += rebase_path(invoker.sources, root_build_dir)
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070078
79 # Use this environment variable to force mypy to colorize output.
80 # See https://github.com/python/mypy/issues/7771
81 environment = [ "MYPY_FORCE_COLOR=1" ]
82
Wyatt Heplerc2ce5242021-03-17 08:42:14 -070083 stamp = true
84
85 deps = invoker.deps
86
87 foreach(dep, invoker.python_deps) {
88 deps += [ string_replace(dep, "(", ".lint.mypy(") ]
89 }
90 }
91}
92
93# Internal template that runs Pylint.
94template("_pw_python_static_analysis_pylint") {
95 # Create a target to run pylint on each of the Python files in this
96 # package and its dependencies.
97 pw_python_action_foreach(target_name) {
98 module = "pylint"
99 args = [
Michael Spangc8b93902021-05-30 15:53:56 -0400100 rebase_path(".", root_build_dir) + "/{{source_target_relative}}",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700101 "--jobs=1",
102 "--output-format=colorized",
103 ]
104
105 if (defined(invoker.pylintrc)) {
Michael Spangc8b93902021-05-30 15:53:56 -0400106 args += [ "--rcfile=" + rebase_path(invoker.pylintrc, root_build_dir) ]
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700107 inputs = [ invoker.pylintrc ]
108 }
109
110 if (host_os == "win") {
111 # Allow CRLF on Windows, in case Git is set to switch line endings.
112 args += [ "--disable=unexpected-line-ending-format" ]
113 }
114
115 sources = invoker.sources
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700116
117 stamp = "$target_gen_dir/{{source_target_relative}}.pylint.passed"
118
119 public_deps = invoker.deps
120
121 foreach(dep, invoker.python_deps) {
122 public_deps += [ string_replace(dep, "(", ".lint.pylint(") ]
123 }
124 }
125}
126
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700127# Defines a Python package. GN Python packages contain several GN targets:
128#
129# - $name - Provides the Python files in the build, but does not take any
130# actions. All subtargets depend on this target.
131# - $name.lint - Runs static analyis tools on the Python code. This is a group
132# of two subtargets:
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800133# - $name.lint.mypy - Runs mypy (if enabled).
134# - $name.lint.pylint - Runs pylint (if enabled).
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700135# - $name.tests - Runs all tests for this package.
Rob Mohrfcf60372020-11-04 11:41:36 -0800136# - $name.install - Installs the package in a venv.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800137# - $name.wheel - Builds a Python wheel for the package.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700138#
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700139# All Python packages are instantiated with in pw_build_PYTHON_TOOLCHAIN,
140# regardless of the current toolchain. This prevents Python-specific work, like
141# running Pylint, from occurring multiple times in a build.
Wyatt Hepler438caa02021-01-15 17:13:11 -0800142#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700143# Args:
144# setup: List of setup file paths (setup.py or pyproject.toml & setup.cfg),
145# which must all be in the same directory.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800146# generate_setup: As an alternative to 'setup', generate setup files with the
147# keywords in this scope. 'name' is required.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700148# sources: Python sources files in the package.
149# tests: Test files for this Python package.
150# python_deps: Dependencies on other pw_python_packages in the GN build.
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800151# python_test_deps: Test-only pw_python_package dependencies.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700152# other_deps: Dependencies on GN targets that are not pw_python_packages.
153# inputs: Other files to track, such as package_data.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800154# proto_library: A pw_proto_library target to embed in this Python package.
155# generate_setup is required in place of setup if proto_library is used.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700156# static_analysis: List of static analysis tools to run; "*" (default) runs
157# all tools. The supported tools are "mypy" and "pylint".
Michael Spangc8b93902021-05-30 15:53:56 -0400158# pylintrc: Path to a pylintrc configuration file to use. If not
159# provided, Pylint's default rcfile search is used. As this may
160# use the the local user's configuration file, it is highly
161# recommended to pass this option to specify the rcfile explicitly.
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800162# mypy_ini: Optional path to a mypy configuration file to use. If not
163# provided, mypy's default configuration file search is used. mypy is
164# executed from the package's setup directory, so mypy.ini files in that
165# directory will take precedence over others.
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800166#
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700167template("pw_python_package") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700168 # The Python targets are always instantiated in pw_build_PYTHON_TOOLCHAIN. Use
Wyatt Hepler752d7d32021-03-02 09:02:23 -0800169 # fully qualified labels so that the toolchain is not lost.
170 _other_deps = []
171 if (defined(invoker.other_deps)) {
172 foreach(dep, invoker.other_deps) {
173 _other_deps += [ get_label_info(dep, "label_with_toolchain") ]
174 }
175 }
176
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800177 _python_deps = []
178 if (defined(invoker.python_deps)) {
179 foreach(dep, invoker.python_deps) {
180 _python_deps += [ get_label_info(dep, "label_with_toolchain") ]
181 }
182 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700183
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700184 # pw_python_script uses pw_python_package, but with a limited set of features.
185 # _pw_standalone signals that this target is actually a pw_python_script.
186 _is_package = !(defined(invoker._pw_standalone) && invoker._pw_standalone)
187
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800188 _generate_package = false
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800189
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800190 # Check the generate_setup and import_protos args to determine if this package
191 # is generated.
192 if (_is_package) {
193 assert(defined(invoker.generate_setup) != defined(invoker.setup),
194 "Either 'setup' or 'generate_setup' (but not both) must provided")
195
196 if (defined(invoker.proto_library)) {
197 assert(invoker.proto_library != "", "'proto_library' cannot be empty")
198 assert(defined(invoker.generate_setup),
199 "Python packages that import protos with 'proto_library' must " +
200 "use 'generate_setup' instead of 'setup'")
201
202 _import_protos = [ invoker.proto_library ]
Wyatt Heplercf184e82021-05-11 18:15:35 -0700203
204 # Depend on the dependencies of the proto library.
205 _proto = get_label_info(invoker.proto_library, "label_no_toolchain")
206 _toolchain = get_label_info(invoker.proto_library, "toolchain")
207 _python_deps += [ "$_proto.python._deps($_toolchain)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800208 } else if (defined(invoker.generate_setup)) {
209 _import_protos = []
210 }
211
212 if (defined(invoker.generate_setup)) {
213 _generate_package = true
214 _setup_dir = "$target_gen_dir/$target_name.generated_python_package"
215
216 if (defined(invoker.strip_prefix)) {
217 _source_root = invoker.strip_prefix
218 } else {
219 _source_root = "."
220 }
221 } else {
222 # Non-generated packages with sources provided need an __init__.py.
223 assert(!defined(invoker.sources) || invoker.sources == [] ||
224 filter_include(invoker.sources, [ "*\b__init__.py" ]) != [],
225 "Python packages must have at least one __init__.py file")
226
227 # Get the directories of the setup files. All must be in the same dir.
228 _setup_dirs = get_path_info(invoker.setup, "dir")
229 _setup_dir = _setup_dirs[0]
230
231 foreach(dir, _setup_dirs) {
232 assert(dir == _setup_dir,
233 "All files in 'setup' must be in the same directory")
234 }
235
236 assert(!defined(invoker.strip_prefix),
237 "'strip_prefix' may only be given if 'generate_setup' is provided")
238 }
239 }
240
241 # Process arguments defaults and set defaults.
242
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700243 _supported_static_analysis_tools = [
244 "mypy",
245 "pylint",
246 ]
247 not_needed([ "_supported_static_analysis_tools" ])
248
249 # Argument: static_analysis (list of tool names or "*"); default = "*" (all)
250 if (!defined(invoker.static_analysis) || invoker.static_analysis == "*") {
251 _static_analysis = _supported_static_analysis_tools
Keir Mierlee65ddc82020-12-02 17:59:52 -0800252 } else {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700253 _static_analysis = invoker.static_analysis
254 }
255
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700256 foreach(_tool, _static_analysis) {
257 assert(_supported_static_analysis_tools + [ _tool ] - [ _tool ] !=
258 _supported_static_analysis_tools,
259 "'$_tool' is not a supported static analysis tool")
Keir Mierlee65ddc82020-12-02 17:59:52 -0800260 }
261
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800262 # Argument: sources (list)
263 _sources = []
264 if (defined(invoker.sources)) {
265 if (_generate_package) {
266 foreach(source, rebase_path(invoker.sources, _source_root)) {
267 _sources += [ "$_setup_dir/$source" ]
268 }
269 } else {
270 _sources += invoker.sources
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700271 }
272 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700273
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800274 # Argument: tests (list)
275 _test_sources = []
276 if (defined(invoker.tests)) {
277 if (_generate_package) {
278 foreach(source, rebase_path(invoker.tests, _source_root)) {
279 _test_sources += [ "$_setup_dir/$source" ]
280 }
281 } else {
282 _test_sources += invoker.tests
Wyatt Heplerd7dc6552020-10-21 16:16:21 -0700283 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700284 }
285
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800286 # Argument: setup (list)
287 _setup_sources = []
288 if (defined(invoker.setup)) {
289 _setup_sources = invoker.setup
290 } else if (_generate_package) {
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700291 _setup_sources = [
292 "$_setup_dir/pyproject.toml",
293 "$_setup_dir/setup.cfg",
294 ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800295 }
296
297 # Argument: python_test_deps (list)
298 _python_test_deps = _python_deps # include all deps in test deps
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800299 if (defined(invoker.python_test_deps)) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800300 foreach(dep, invoker.python_test_deps) {
301 _python_test_deps += [ get_label_info(dep, "label_with_toolchain") ]
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800302 }
303 }
304
305 if (_test_sources == []) {
306 assert(!defined(invoker.python_test_deps),
307 "python_test_deps was provided, but there are no tests in " +
308 get_label_info(":$target_name", "label_no_toolchain"))
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800309 not_needed([ "_python_test_deps" ])
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800310 }
311
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700312 _all_py_files =
313 _sources + _test_sources + filter_include(_setup_sources, [ "*.py" ])
Wyatt Hepler438caa02021-01-15 17:13:11 -0800314
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700315 # The pw_python_package subtargets are only instantiated in
316 # pw_build_PYTHON_TOOLCHAIN. Targets in other toolchains just refer to the
317 # targets in this toolchain.
318 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700319 # Create the package_metadata.json file. This is used by the
320 # pw_create_python_source_tree template.
321 _package_metadata_json_file =
322 "$target_gen_dir/$target_name/package_metadata.json"
323
324 # Get Python package metadata and write to disk as JSON.
325 _package_metadata = {
326 gn_target_name = get_label_info(invoker.target_name, "label_no_toolchain")
327
328 # Get package source files
329 sources = rebase_path(_sources, root_build_dir)
330
331 # Get setup.cfg, pyproject.toml, or setup.py file
332 setup_sources = rebase_path(_setup_sources, root_build_dir)
333
334 # Get test source files
335 tests = rebase_path(_test_sources, root_build_dir)
336
337 # Get package input files (package data)
338 inputs = []
339 if (defined(invoker.inputs)) {
340 inputs = rebase_path(invoker.inputs, root_build_dir)
341 }
342
343 # Get generate_setup
344 if (defined(invoker.generate_setup)) {
345 generate_setup = invoker.generate_setup
346 }
347 }
348
349 # Finally, write out the json
350 write_file(_package_metadata_json_file, _package_metadata, "json")
351
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800352 # Declare the main Python package group. This represents the Python files,
353 # but does not take any actions. GN targets can depend on the package name
354 # to run when any files in the package change.
355 if (_generate_package) {
356 # If this package is generated, mirror the sources to the final directory.
357 pw_mirror_tree("$target_name._mirror_sources_to_out_dir") {
358 directory = _setup_dir
Wyatt Hepler438caa02021-01-15 17:13:11 -0800359
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800360 sources = []
361 if (defined(invoker.sources)) {
362 sources += invoker.sources
363 }
364 if (defined(invoker.tests)) {
365 sources += invoker.tests
366 }
Wyatt Heplerde20d742021-06-02 23:34:14 -0700367 if (defined(invoker.inputs)) {
368 sources += invoker.inputs
369 }
Wyatt Hepler438caa02021-01-15 17:13:11 -0800370
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800371 source_root = _source_root
372 public_deps = _python_deps + _other_deps
Wyatt Hepler285636f2020-12-15 13:13:11 -0800373 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700374
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700375 # Get generated_setup scope and write it to disk as JSON.
376
377 # Expected setup.cfg structure:
378 # https://setuptools.readthedocs.io/en/latest/userguide/declarative_config.html
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800379 _gen_setup = invoker.generate_setup
Anthony DiGirolamoafb861c2021-08-12 16:48:04 -0700380 assert(defined(_gen_setup.metadata),
381 "'metadata = {}' is required in generate_package")
382
383 # Get metadata which should contain at least name.
384 _gen_metadata = {
385 }
386 _gen_metadata = _gen_setup.metadata
387 assert(
388 defined(_gen_metadata.name),
389 "metadata = { name = 'package_name' } is required in generate_package")
390
391 # Get options which should not have packages or package_data.
392 if (defined(_gen_setup.options)) {
393 _gen_options = {
394 }
395 _gen_options = _gen_setup.options
396 assert(!defined(_gen_options.packages) &&
397 !defined(_gen_options.package_data),
398 "'packages' and 'package_data' may not be provided " +
399 "in 'generate_package' options.")
400 }
401
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800402 write_file("$_setup_dir/setup.json", _gen_setup, "json")
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800403
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800404 # Generate the setup.py, py.typed, and __init__.py files as needed.
405 action(target_name) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700406 metadata = {
407 pw_python_package_metadata_json = [ _package_metadata_json_file ]
408 }
409
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800410 script = "$dir_pw_build/py/pw_build/generate_python_package.py"
411 args = [
412 "--label",
413 get_label_info(":$target_name", "label_no_toolchain"),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700414 "--generated-root",
Michael Spangc8b93902021-05-30 15:53:56 -0400415 rebase_path(_setup_dir, root_build_dir),
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800416 "--setup-json",
Michael Spangc8b93902021-05-30 15:53:56 -0400417 rebase_path("$_setup_dir/setup.json", root_build_dir),
418 ] + rebase_path(_sources, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800419
Wyatt Heplercf184e82021-05-11 18:15:35 -0700420 # Pass in the .json information files for the imported proto libraries.
421 foreach(proto, _import_protos) {
422 _label = get_label_info(proto, "label_no_toolchain") +
423 ".python($pw_protobuf_compiler_TOOLCHAIN)"
424 _file = get_label_info(_label, "target_gen_dir") + "/" +
425 get_label_info(_label, "name") + ".json"
426 args += [
427 "--proto-library",
Michael Spangc8b93902021-05-30 15:53:56 -0400428 rebase_path(_file, root_build_dir),
Wyatt Heplercf184e82021-05-11 18:15:35 -0700429 ]
430 }
431
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800432 if (defined(invoker._pw_module_as_package) &&
433 invoker._pw_module_as_package) {
434 args += [ "--module-as-package" ]
435 }
436
Wyatt Hepler0e1f5e42021-03-16 22:51:57 -0700437 inputs = [ "$_setup_dir/setup.json" ]
438
Wyatt Heplercf184e82021-05-11 18:15:35 -0700439 public_deps = [ ":$target_name._mirror_sources_to_out_dir" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800440
441 outputs = _setup_sources
442 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800443 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800444 # If the package is not generated, use an input group for the sources.
445 pw_input_group(target_name) {
Anthony DiGirolamo211dce42021-08-12 16:48:43 -0700446 metadata = {
447 pw_python_package_metadata_json = [ _package_metadata_json_file ]
448 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800449 inputs = _all_py_files
450 if (defined(invoker.inputs)) {
451 inputs += invoker.inputs
452 }
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800453
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800454 deps = _python_deps + _other_deps
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800455 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700456 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700457
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800458 if (_is_package) {
459 # Install this Python package and its dependencies in the current Python
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700460 # environment using pip.
461 pw_python_action("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800462 module = "pip"
463 public_deps = []
Joe Ethiere93ccb12021-08-20 15:59:53 -0700464 if (defined(invoker.public_deps)) {
465 public_deps += invoker.public_deps
466 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800467
Anthony DiGirolamocd405772021-10-18 16:29:26 -0700468 args = [
469 "install",
470
471 # This speeds up pip installs. At this point in the gn build the
472 # virtualenv is already activated so build isolation isn't required.
473 # This requires that pip, setuptools, and wheel packages are
474 # installed.
475 "--no-build-isolation",
476 ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800477
Michael Spang01faaf72021-06-09 22:38:40 -0400478 inputs = pw_build_PIP_CONSTRAINTS
479 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
480 args += [
481 "--constraint",
482 rebase_path(_constraints_file, root_build_dir),
483 ]
484 }
485
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800486 # For generated packages, reinstall when any files change. For regular
487 # packages, only reinstall when setup.py changes.
488 if (_generate_package) {
489 public_deps += [ ":${invoker.target_name}" ]
490 } else {
Michael Spang01faaf72021-06-09 22:38:40 -0400491 inputs += invoker.setup
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800492
493 # Install with --editable since the complete package is in source.
494 args += [ "--editable" ]
495 }
496
Michael Spangc8b93902021-05-30 15:53:56 -0400497 args += [ rebase_path(_setup_dir, root_build_dir) ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800498
499 stamp = true
500
501 # Parallel pip installations don't work, so serialize pip invocations.
Michael Spangda057f72021-06-10 00:50:36 -0400502 pool = "$dir_pw_build/pool:pip($default_toolchain)"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800503
504 foreach(dep, _python_deps) {
505 # We need to add a suffix to the target name, but the label is
506 # formatted as "//path/to:target(toolchain)", so we can't just append
507 # ".subtarget". Instead, we replace the opening parenthesis of the
508 # toolchain with ".suffix(".
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700509 public_deps += [ string_replace(dep, "(", "._run_pip_install(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800510 }
511 }
512
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800513 # Builds a Python wheel for this package. Records the output directory
514 # in the pw_python_package_wheels metadata key.
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700515 pw_python_action("$target_name._build_wheel") {
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800516 metadata = {
517 pw_python_package_wheels = [ "$target_out_dir/$target_name" ]
518 }
519
520 module = "build"
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800521
Michael Spangc8b93902021-05-30 15:53:56 -0400522 args =
523 [
524 rebase_path(_setup_dir, root_build_dir),
525 "--wheel",
526 "--no-isolation",
527 "--outdir",
528 ] + rebase_path(metadata.pw_python_package_wheels, root_build_dir)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800529
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800530 deps = [ ":${invoker.target_name}" ]
531 foreach(dep, _python_deps) {
532 deps += [ string_replace(dep, "(", ".wheel(") ]
533 }
534
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800535 stamp = true
536 }
537 } else {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700538 # Stubs for non-package targets.
539 group("$target_name._run_pip_install") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800540 }
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700541 group("$target_name._build_wheel") {
542 }
543 }
544
545 # Create the .install and .wheel targets. To limit unnecessary pip
546 # executions, non-generated packages are only reinstalled when their
547 # setup.py changes. However, targets that depend on the .install subtarget
548 # re-run whenever any source files change.
549 #
550 # These targets just represent the source files if this isn't a package.
551 group("$target_name.install") {
552 public_deps = [ ":${invoker.target_name}" ]
553
554 if (_is_package) {
555 public_deps += [ ":${invoker.target_name}._run_pip_install" ]
556 }
557
558 foreach(dep, _python_deps) {
559 public_deps += [ string_replace(dep, "(", ".install(") ]
560 }
561 }
562
563 group("$target_name.wheel") {
564 public_deps = [ ":${invoker.target_name}.install" ]
565
566 if (_is_package) {
567 public_deps += [ ":${invoker.target_name}._build_wheel" ]
568 }
569
570 foreach(dep, _python_deps) {
571 public_deps += [ string_replace(dep, "(", ".wheel(") ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800572 }
573 }
574
575 # Define the static analysis targets for this package.
576 group("$target_name.lint") {
Wyatt Hepler446f24c2021-03-24 09:29:35 -0700577 deps = []
578 foreach(_tool, _supported_static_analysis_tools) {
579 deps += [ ":${invoker.target_name}.lint.$_tool" ]
580 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800581 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700582
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700583 if (_static_analysis != [] || _test_sources != []) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800584 # All packages to install for either general use or test running.
585 _test_install_deps = [ ":$target_name.install" ]
586 foreach(dep, _python_test_deps + [ "$dir_pw_build:python_lint" ]) {
587 _test_install_deps += [ string_replace(dep, "(", ".install(") ]
588 }
589 }
590
591 # For packages that are not generated, create targets to run mypy and pylint.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700592 foreach(_tool, _static_analysis) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800593 # Run lint tools from the setup or target directory so that the tools detect
594 # config files (e.g. pylintrc or mypy.ini) in that directory. Config files
595 # may be explicitly specified with the pylintrc or mypy_ini arguments.
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700596 target("_pw_python_static_analysis_$_tool", "$target_name.lint.$_tool") {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800597 sources = _all_py_files
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700598 deps = _test_install_deps
599 python_deps = _python_deps
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800600
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700601 _optional_variables = [
602 "mypy_ini",
603 "pylintrc",
604 ]
605 forward_variables_from(invoker, _optional_variables)
606 not_needed(_optional_variables)
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800607 }
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700608 }
609
610 foreach(_unused_tool, _supported_static_analysis_tools - _static_analysis) {
611 pw_input_group("$target_name.lint.$_unused_tool") {
612 inputs = []
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800613 if (defined(invoker.pylintrc)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700614 inputs += [ invoker.pylintrc ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800615 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800616 if (defined(invoker.mypy_ini)) {
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700617 inputs += [ invoker.mypy_ini ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800618 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800619 }
Wyatt Hepleraf853ea2021-03-09 19:09:54 -0800620
621 # Generated packages with linting disabled never need the whole file list.
622 not_needed([ "_all_py_files" ])
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700623 }
Alexei Frolova4c0aee2020-12-01 13:48:48 -0800624 } else {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800625 # Create groups with the public target names ($target_name, $target_name.lint,
626 # $target_name.install, etc.). These are actually wrappers around internal
627 # Python actions instantiated with the default toolchain. This ensures there
628 # is only a single copy of each Python action in the build.
629 #
630 # The $target_name.tests group is created separately below.
631 group("$target_name") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700632 deps = [ ":$target_name($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800633 }
634
635 foreach(subtarget, pw_python_package_subtargets - [ "tests" ]) {
636 group("$target_name.$subtarget") {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700637 deps =
638 [ ":${invoker.target_name}.$subtarget($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800639 }
Wyatt Heplerb8e13602020-10-19 11:20:36 -0700640 }
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800641
642 # Everything Python-related is only instantiated in the default toolchain.
643 # Silence not-needed warnings except for in the default toolchain.
644 not_needed("*")
645 not_needed(invoker, "*")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700646 }
647
648 # Create a target for each test file.
649 _test_targets = []
650
651 foreach(test, _test_sources) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800652 if (_is_package) {
653 _name = rebase_path(test, _setup_dir)
654 } else {
655 _name = test
656 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700657
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800658 _test_target = "$target_name.tests." + string_replace(_name, "/", "_")
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700659
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700660 if (current_toolchain == pw_build_PYTHON_TOOLCHAIN) {
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800661 pw_python_action(_test_target) {
662 script = test
663 stamp = true
Wyatt Hepler620b3e02021-01-22 09:14:45 -0800664
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800665 deps = _test_install_deps
666
667 foreach(dep, _python_test_deps) {
668 deps += [ string_replace(dep, "(", ".tests(") ]
669 }
670 }
671 } else {
672 # Create a public version of each test target, so tests can be executed as
673 # //path/to:package.tests.foo.py.
674 group(_test_target) {
Wyatt Heplerf6f74f42021-04-13 19:26:20 -0700675 deps = [ ":$_test_target($pw_build_PYTHON_TOOLCHAIN)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700676 }
677 }
678
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800679 _test_targets += [ ":$_test_target" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700680 }
681
682 group("$target_name.tests") {
683 deps = _test_targets
684 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700685
686 _pw_create_aliases_if_name_matches_directory(target_name) {
687 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700688}
689
690# Declares a group of Python packages or other Python groups. pw_python_groups
691# expose the same set of subtargets as pw_python_package (e.g.
692# "$group_name.lint" and "$group_name.tests"), but these apply to all packages
693# in deps and their dependencies.
694template("pw_python_group") {
695 if (defined(invoker.python_deps)) {
696 _python_deps = invoker.python_deps
697 } else {
698 _python_deps = []
Wyatt Heplera3ca62a2021-05-04 16:21:43 -0700699 not_needed([ "invoker" ]) # Allow empty groups.
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700700 }
701
702 group(target_name) {
703 deps = _python_deps
Wyatt Heplercf184e82021-05-11 18:15:35 -0700704
705 if (defined(invoker.other_deps)) {
706 deps += invoker.other_deps
707 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700708 }
709
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800710 foreach(subtarget, pw_python_package_subtargets) {
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700711 group("$target_name.$subtarget") {
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700712 public_deps = []
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700713 foreach(dep, _python_deps) {
714 # Split out the toolchain to support deps with a toolchain specified.
715 _target = get_label_info(dep, "label_no_toolchain")
716 _toolchain = get_label_info(dep, "toolchain")
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700717 public_deps += [ "$_target.$subtarget($_toolchain)" ]
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700718 }
719 }
720 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700721
722 _pw_create_aliases_if_name_matches_directory(target_name) {
723 }
Wyatt Heplerb16dfd32020-10-12 08:46:38 -0700724}
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700725
726# Declares Python scripts or tests that are not part of a Python package.
727# Similar to pw_python_package, but only supports a subset of its features.
728#
729# pw_python_script accepts the same arguments as pw_python_package, except
730# `setup` cannot be provided.
731#
732# pw_python_script provides the same subtargets as pw_python_package, but
733# $target_name.install and $target_name.wheel only affect the python_deps of
734# this GN target, not the target itself.
Wyatt Hepler473efe02021-05-04 14:15:16 -0700735#
736# pw_python_script allows creating a pw_python_action associated with the
737# script. This is provided by passing an 'action' scope to pw_python_script.
738# This functions like a normal action, with a few additions: the action uses the
739# pw_python_script's python_deps and defaults to using the source file as its
740# 'script' argument, if there is only a single source file.
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700741template("pw_python_script") {
Wyatt Hepler473efe02021-05-04 14:15:16 -0700742 _package_variables = [
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700743 "sources",
744 "tests",
745 "python_deps",
746 "other_deps",
747 "inputs",
Wyatt Hepler8ce90132020-12-03 10:57:20 -0800748 "pylintrc",
Wyatt Heplerc2ce5242021-03-17 08:42:14 -0700749 "mypy_ini",
750 "static_analysis",
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700751 ]
752
753 pw_python_package(target_name) {
754 _pw_standalone = true
Wyatt Hepler473efe02021-05-04 14:15:16 -0700755 forward_variables_from(invoker, _package_variables)
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700756 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700757
758 _pw_create_aliases_if_name_matches_directory(target_name) {
759 }
Wyatt Hepler473efe02021-05-04 14:15:16 -0700760
761 if (defined(invoker.action)) {
762 pw_python_action("$target_name.action") {
763 forward_variables_from(invoker.action, "*", [ "python_deps" ])
764 python_deps = [ ":${invoker.target_name}" ]
765
766 if (!defined(script) && !defined(module) && defined(invoker.sources)) {
767 _sources = invoker.sources
768 assert(_sources != [] && _sources == [ _sources[0] ],
769 "'script' must be specified unless there is only one source " +
770 "in 'sources'")
771 script = _sources[0]
772 }
773 }
774 }
Wyatt Hepler45af57b2020-10-23 08:05:28 -0700775}
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800776
777# Represents a list of Python requirements, as in a requirements.txt.
778#
779# Args:
780# files: One or more requirements.txt files.
781# requirements: A list of requirements.txt-style requirements.
782template("pw_python_requirements") {
783 assert(defined(invoker.files) || defined(invoker.requirements),
784 "pw_python_requirements requires a list of requirements.txt files " +
785 "in the 'files' arg or requirements in 'requirements'")
786
787 _requirements_files = []
788
789 if (defined(invoker.files)) {
790 _requirements_files += invoker.files
791 }
792
793 if (defined(invoker.requirements)) {
794 _requirements_file = "$target_gen_dir/$target_name.requirements.txt"
795 write_file(_requirements_file, invoker.requirements)
796 _requirements_files += [ _requirements_file ]
797 }
798
799 # The default target represents the requirements themselves.
800 pw_input_group(target_name) {
801 inputs = _requirements_files
802 }
803
804 # Use the same subtargets as pw_python_package so these targets can be listed
805 # as python_deps of pw_python_packages.
806 pw_python_action("$target_name.install") {
807 inputs = _requirements_files
808
809 module = "pip"
810 args = [ "install" ]
811
812 foreach(_requirements_file, inputs) {
813 args += [
814 "--requirement",
Michael Spangc8b93902021-05-30 15:53:56 -0400815 rebase_path(_requirements_file, root_build_dir),
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800816 ]
817 }
818
Michael Spang01faaf72021-06-09 22:38:40 -0400819 inputs += pw_build_PIP_CONSTRAINTS
820 foreach(_constraints_file, pw_build_PIP_CONSTRAINTS) {
821 args += [
822 "--constraint",
823 rebase_path(_constraints_file, root_build_dir),
824 ]
825 }
826
Michael Spangda057f72021-06-10 00:50:36 -0400827 pool = "$dir_pw_build/pool:pip($default_toolchain)"
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800828 stamp = true
829 }
830
831 # Create stubs for the unused subtargets so that pw_python_requirements can be
832 # used as python_deps.
Wyatt Heplerdcfcecf2021-03-01 08:36:19 -0800833 foreach(subtarget, pw_python_package_subtargets - [ "install" ]) {
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800834 group("$target_name.$subtarget") {
835 }
836 }
Wyatt Heplerb85cda42021-04-07 13:13:15 -0700837
838 _pw_create_aliases_if_name_matches_directory(target_name) {
839 }
Wyatt Hepler4d1e6aa2021-01-14 08:39:42 -0800840}