blob: 4a05eebe090b415ff5dc9972001d0d1054bad044 [file] [log] [blame]
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001#!/usr/bin/env python
2#
3# Copyright (C) 2019 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""Call cargo -v, parse its output, and generate Android.bp.
17
18Usage: Run this script in a crate workspace root directory.
19The Cargo.toml file should work at least for the host platform.
20
21(1) Without other flags, "cargo2android.py --run"
22 calls cargo clean, calls cargo build -v, and generates Android.bp.
23 The cargo build only generates crates for the host,
24 without test crates.
25
26(2) To build crates for both host and device in Android.bp, use the
27 --device flag, for example:
28 cargo2android.py --run --device
29
30 This is equivalent to using the --cargo flag to add extra builds:
31 cargo2android.py --run
32 --cargo "build"
33 --cargo "build --target x86_64-unknown-linux-gnu"
34
35 On MacOS, use x86_64-apple-darwin as target triple.
36 Here the host target triple is used as a fake cross compilation target.
37 If the crate's Cargo.toml and environment configuration works for an
38 Android target, use that target triple as the cargo build flag.
39
40(3) To build default and test crates, for host and device, use both
41 --device and --tests flags:
42 cargo2android.py --run --device --tests
43
44 This is equivalent to using the --cargo flag to add extra builds:
45 cargo2android.py --run
46 --cargo "build"
47 --cargo "build --tests"
48 --cargo "build --target x86_64-unknown-linux-gnu"
49 --cargo "build --tests --target x86_64-unknown-linux-gnu"
50
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -070051 Note that when there are test modules generated into Android.bp,
52 corresponding test entries will also be added into the TEST_MAPPING file.
53
Chih-Hung Hsieh35ca4bc2020-07-10 16:49:51 -070054If there are rustc warning messages, this script will add
55a warning comment to the owner crate module in Android.bp.
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -080056"""
57
58from __future__ import print_function
59
60import argparse
61import os
62import os.path
63import re
Andrew Walbran80e90be2020-06-09 14:33:18 +010064import sys
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -080065
Chih-Hung Hsiehe02dce12020-07-14 16:05:21 -070066# Some Rust packages include extra unwanted crates.
67# This set contains all such excluded crate names.
68EXCLUDED_CRATES = set(['protobuf_bin_gen_rust_do_not_use'])
69
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -080070RENAME_MAP = {
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -070071 # This map includes all changes to the default rust module names
72 # to resolve name conflicts, avoid confusion, or work as plugin.
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -080073 'libbacktrace': 'libbacktrace_rust',
74 'libgcc': 'libgcc_rust',
75 'liblog': 'liblog_rust',
76 'libsync': 'libsync_rust',
77 'libx86_64': 'libx86_64_rust',
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -070078 'protoc_gen_rust': 'protoc-gen-rust',
79}
80
81RENAME_STEM_MAP = {
82 # This map includes all changes to the default rust module stem names,
83 # which is used for output files when different from the module name.
84 'protoc_gen_rust': 'protoc-gen-rust',
85}
86
87RENAME_DEFAULTS_MAP = {
88 # This map includes all changes to the default prefix of rust_default
89 # module names, to avoid conflict with existing Android modules.
90 'libc': 'rust_libc',
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -080091}
92
93# Header added to all generated Android.bp files.
Andrew Walbran80e90be2020-06-09 14:33:18 +010094ANDROID_BP_HEADER = '// This file is generated by cargo2android.py {args}.\n'
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -080095
96CARGO_OUT = 'cargo.out' # Name of file to keep cargo build -v output.
97
98TARGET_TMP = 'target.tmp' # Name of temporary output directory.
99
100# Message to be displayed when this script is called without the --run flag.
101DRY_RUN_NOTE = (
102 'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' +
103 'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' +
104 'and writes to Android.bp in the current and subdirectories.\n\n' +
105 'To do do all of the above, use the --run flag.\n' +
106 'See --help for other flags, and more usage notes in this script.\n')
107
108# Cargo -v output of a call to rustc.
109RUSTC_PAT = re.compile('^ +Running `rustc (.*)`$')
110
111# Cargo -vv output of a call to rustc could be split into multiple lines.
112# Assume that the first line will contain some CARGO_* env definition.
113RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$')
114# The combined -vv output rustc command line pattern.
115RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$')
116
117# Cargo -vv output of a "cc" or "ar" command; all in one line.
118CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$')
119# Some package, such as ring-0.13.5, has pattern '... running "cc"'.
120
121# Rustc output of file location path pattern for a warning message.
122WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+')
123
124# Rust package name with suffix -d1.d2.d3.
125VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+$')
126
127
128def altered_name(name):
129 return RENAME_MAP[name] if (name in RENAME_MAP) else name
130
131
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700132def altered_stem(name):
133 return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name
134
135
136def altered_defaults(name):
137 return RENAME_DEFAULTS_MAP[name] if (name in RENAME_DEFAULTS_MAP) else name
138
139
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800140def is_build_crate_name(name):
141 # We added special prefix to build script crate names.
142 return name.startswith('build_script_')
143
144
145def is_dependent_file_path(path):
146 # Absolute or dependent '.../' paths are not main files of this crate.
147 return path.startswith('/') or path.startswith('.../')
148
149
150def get_module_name(crate): # to sort crates in a list
151 return crate.module_name
152
153
154def pkg2crate_name(s):
155 return s.replace('-', '_').replace('.', '_')
156
157
158def file_base_name(path):
159 return os.path.splitext(os.path.basename(path))[0]
160
161
162def test_base_name(path):
163 return pkg2crate_name(file_base_name(path))
164
165
166def unquote(s): # remove quotes around str
167 if s and len(s) > 1 and s[0] == '"' and s[-1] == '"':
168 return s[1:-1]
169 return s
170
171
172def remove_version_suffix(s): # remove -d1.d2.d3 suffix
173 if VERSION_SUFFIX_PAT.match(s):
174 return VERSION_SUFFIX_PAT.match(s).group(1)
175 return s
176
177
178def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/*
179 return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s)
180
181
182def escape_quotes(s): # replace '"' with '\\"'
183 return s.replace('"', '\\"')
184
185
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -0700186class TestMapping(object):
187 """Entries for a TEST_MAPPING file."""
188
189 def __init__(self):
190 self.entries = []
191
192 def add_test(self, name, host):
193 self.entries.append((name, host))
194
195 def is_empty(self):
196 return not self.entries
197
198 def dump(self, outf_name):
199 """Append all entries into the output file."""
200 if self.is_empty():
201 return
202 with open(outf_name, 'w') as outf:
203 outf.write('// Generated by cargo2android.py for tests in Android.bp\n')
204 outf.write('{\n "presubmit": [\n')
205 is_first = True
206 for (name, host) in self.entries:
207 if not is_first: # add comma and '\n' after the previous entry
208 outf.write(',\n')
209 is_first = False
210 outf.write(' {\n "name": "' + name + '"')
211 if host:
212 outf.write(',\n "host": true\n }')
213 else:
214 outf.write('\n }')
215 outf.write('\n ]\n}\n')
216
217
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800218class Crate(object):
219 """Information of a Rust crate to collect/emit for an Android.bp module."""
220
221 def __init__(self, runner, outf_name):
222 # Remembered global runner and its members.
223 self.runner = runner
224 self.debug = runner.args.debug
225 self.cargo_dir = '' # directory of my Cargo.toml
226 self.outf_name = outf_name # path to Android.bp
227 self.outf = None # open file handle of outf_name during dump*
228 # Variants/results that could be merged from multiple rustc lines.
229 self.host_supported = False
230 self.device_supported = False
231 self.has_warning = False
232 # Android module properties derived from rustc parameters.
233 self.module_name = '' # unique in Android build system
234 self.module_type = '' # rust_{binary,library,test}[_host] etc.
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700235 self.defaults = '' # rust_defaults used by rust_test* modules
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -0700236 self.default_srcs = False # use 'srcs' defined in self.defaults
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800237 self.root_pkg = '' # parent package name of a sub/test packge, from -L
238 self.srcs = list() # main_src or merged multiple source files
239 self.stem = '' # real base name of output file
240 # Kept parsed status
241 self.errors = '' # all errors found during parsing
242 self.line_num = 1 # runner told input source line number
243 self.line = '' # original rustc command line parameters
244 # Parameters collected from rustc command line.
245 self.crate_name = '' # follows --crate-name
246 self.main_src = '' # follows crate_name parameter, shortened
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700247 self.crate_types = list() # follows --crate-type
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800248 self.cfgs = list() # follows --cfg, without feature= prefix
249 self.features = list() # follows --cfg, name in 'feature="..."'
250 self.codegens = list() # follows -C, some ignored
251 self.externs = list() # follows --extern
252 self.core_externs = list() # first part of self.externs elements
253 self.static_libs = list() # e.g. -l static=host_cpuid
254 self.shared_libs = list() # e.g. -l dylib=wayland-client, -l z
255 self.cap_lints = '' # follows --cap-lints
256 self.emit_list = '' # e.g., --emit=dep-info,metadata,link
257 self.edition = '2015' # rustc default, e.g., --edition=2018
258 self.target = '' # follows --target
259
260 def write(self, s):
261 # convenient way to output one line at a time with EOL.
262 self.outf.write(s + '\n')
263
264 def same_flags(self, other):
265 # host_supported, device_supported, has_warning are not compared but merged
266 # target is not compared, to merge different target/host modules
267 # externs is not compared; only core_externs is compared
268 return (not self.errors and not other.errors and
269 self.edition == other.edition and
270 self.cap_lints == other.cap_lints and
271 self.emit_list == other.emit_list and
272 self.core_externs == other.core_externs and
273 self.codegens == other.codegens and
274 self.features == other.features and
275 self.static_libs == other.static_libs and
276 self.shared_libs == other.shared_libs and self.cfgs == other.cfgs)
277
278 def merge_host_device(self, other):
279 """Returns true if attributes are the same except host/device support."""
280 return (self.crate_name == other.crate_name and
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700281 self.crate_types == other.crate_types and
282 self.main_src == other.main_src and
283 # before merge, each test module has an unique module name and stem
284 (self.stem == other.stem or self.crate_types == ['test']) and
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800285 self.root_pkg == other.root_pkg and not self.skip_crate() and
286 self.same_flags(other))
287
288 def merge_test(self, other):
289 """Returns true if self and other are tests of same root_pkg."""
290 # Before merger, each test has its own crate_name.
291 # A merged test uses its source file base name as output file name,
292 # so a test is mergeable only if its base name equals to its crate name.
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700293 return (self.crate_types == other.crate_types and
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -0700294 self.crate_types == ['test'] and self.root_pkg == other.root_pkg and
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800295 not self.skip_crate() and
296 other.crate_name == test_base_name(other.main_src) and
297 (len(self.srcs) > 1 or
298 (self.crate_name == test_base_name(self.main_src)) and
299 self.host_supported == other.host_supported and
300 self.device_supported == other.device_supported) and
301 self.same_flags(other))
302
303 def merge(self, other, outf_name):
304 """Try to merge crate into self."""
305 should_merge_host_device = self.merge_host_device(other)
306 should_merge_test = False
307 if not should_merge_host_device:
308 should_merge_test = self.merge_test(other)
309 # A for-device test crate can be merged with its for-host version,
310 # or merged with a different test for the same host or device.
311 # Since we run cargo once for each device or host, test crates for the
312 # first device or host will be merged first. Then test crates for a
313 # different device or host should be allowed to be merged into a
314 # previously merged one, maybe for a different device or host.
315 if should_merge_host_device or should_merge_test:
316 self.runner.init_bp_file(outf_name)
317 with open(outf_name, 'a') as outf: # to write debug info
318 self.outf = outf
319 other.outf = outf
320 self.do_merge(other, should_merge_test)
321 return True
322 return False
323
324 def do_merge(self, other, should_merge_test):
325 """Merge attributes of other to self."""
326 if self.debug:
327 self.write('\n// Before merge definition (1):')
328 self.dump_debug_info()
329 self.write('\n// Before merge definition (2):')
330 other.dump_debug_info()
331 # Merge properties of other to self.
332 self.host_supported = self.host_supported or other.host_supported
333 self.device_supported = self.device_supported or other.device_supported
334 self.has_warning = self.has_warning or other.has_warning
335 if not self.target: # okay to keep only the first target triple
336 self.target = other.target
337 # decide_module_type sets up default self.stem,
338 # which can be changed if self is a merged test module.
339 self.decide_module_type()
340 if should_merge_test:
341 self.srcs.append(other.main_src)
342 # use a short unique name as the merged module name.
343 prefix = self.root_pkg + '_tests'
344 self.module_name = self.runner.claim_module_name(prefix, self, 0)
345 self.stem = self.module_name
346 # This normalized root_pkg name although might be the same
347 # as other module's crate_name, it is not actually used for
348 # output file name. A merged test module always have multiple
349 # source files and each source file base name is used as
350 # its output file name.
351 self.crate_name = pkg2crate_name(self.root_pkg)
352 if self.debug:
353 self.write('\n// After merge definition (1):')
354 self.dump_debug_info()
355
356 def find_cargo_dir(self):
357 """Deepest directory with Cargo.toml and contains the main_src."""
358 if not is_dependent_file_path(self.main_src):
359 dir_name = os.path.dirname(self.main_src)
360 while dir_name:
361 if os.path.exists(dir_name + '/Cargo.toml'):
362 self.cargo_dir = dir_name
363 return
364 dir_name = os.path.dirname(dir_name)
365
366 def parse(self, line_num, line):
367 """Find important rustc arguments to convert to Android.bp properties."""
368 self.line_num = line_num
369 self.line = line
370 args = line.split() # Loop through every argument of rustc.
371 i = 0
372 while i < len(args):
373 arg = args[i]
374 if arg == '--crate-name':
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700375 i += 1
376 self.crate_name = args[i]
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800377 elif arg == '--crate-type':
378 i += 1
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700379 # cargo calls rustc with multiple --crate-type flags.
380 # rustc can accept:
381 # --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]
382 self.crate_types.append(args[i])
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800383 elif arg == '--test':
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700384 self.crate_types.append('test')
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800385 elif arg == '--target':
386 i += 1
387 self.target = args[i]
388 elif arg == '--cfg':
389 i += 1
390 if args[i].startswith('\'feature='):
391 self.features.append(unquote(args[i].replace('\'feature=', '')[:-1]))
392 else:
393 self.cfgs.append(args[i])
394 elif arg == '--extern':
395 i += 1
396 extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i])
397 self.externs.append(extern_names)
398 self.core_externs.append(re.sub(' = .*', '', extern_names))
399 elif arg == '-C': # codegen options
400 i += 1
401 # ignore options not used in Android
402 if not (args[i].startswith('debuginfo=') or
403 args[i].startswith('extra-filename=') or
404 args[i].startswith('incremental=') or
405 args[i].startswith('metadata=')):
406 self.codegens.append(args[i])
407 elif arg == '--cap-lints':
408 i += 1
409 self.cap_lints = args[i]
410 elif arg == '-L':
411 i += 1
412 if args[i].startswith('dependency=') and args[i].endswith('/deps'):
413 if '/' + TARGET_TMP + '/' in args[i]:
414 self.root_pkg = re.sub(
415 '^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i]))
416 else:
417 self.root_pkg = re.sub('^.*/', '',
418 re.sub('/[^/]+/[^/]+/deps$', '', args[i]))
419 self.root_pkg = remove_version_suffix(self.root_pkg)
420 elif arg == '-l':
421 i += 1
422 if args[i].startswith('static='):
423 self.static_libs.append(re.sub('static=', '', args[i]))
424 elif args[i].startswith('dylib='):
425 self.shared_libs.append(re.sub('dylib=', '', args[i]))
426 else:
427 self.shared_libs.append(args[i])
428 elif arg == '--out-dir' or arg == '--color': # ignored
429 i += 1
430 elif arg.startswith('--error-format=') or arg.startswith('--json='):
431 _ = arg # ignored
432 elif arg.startswith('--emit='):
433 self.emit_list = arg.replace('--emit=', '')
434 elif arg.startswith('--edition='):
435 self.edition = arg.replace('--edition=', '')
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700436 elif not arg.startswith('-'):
437 # shorten imported crate main source paths like $HOME/.cargo/
438 # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/lib.rs
439 self.main_src = re.sub(r'^/[^ ]*/registry/src/', '.../', args[i])
440 self.main_src = re.sub(r'^\.\.\./github.com-[0-9a-f]*/', '.../',
441 self.main_src)
442 self.find_cargo_dir()
443 if self.cargo_dir and not self.runner.args.onefile:
444 # Write to Android.bp in the subdirectory with Cargo.toml.
445 self.outf_name = self.cargo_dir + '/Android.bp'
446 self.main_src = self.main_src[len(self.cargo_dir) + 1:]
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800447 else:
448 self.errors += 'ERROR: unknown ' + arg + '\n'
449 i += 1
450 if not self.crate_name:
451 self.errors += 'ERROR: missing --crate-name\n'
452 if not self.main_src:
453 self.errors += 'ERROR: missing main source file\n'
454 else:
455 self.srcs.append(self.main_src)
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700456 if not self.crate_types:
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800457 # Treat "--cfg test" as "--test"
458 if 'test' in self.cfgs:
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700459 self.crate_types.append('test')
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800460 else:
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700461 self.errors += 'ERROR: missing --crate-type or --test\n'
462 elif len(self.crate_types) > 1:
463 if 'test' in self.crate_types:
464 self.errors += 'ERROR: cannot handle both --crate-type and --test\n'
465 if 'lib' in self.crate_types and 'rlib' in self.crate_types:
466 self.errors += 'ERROR: cannot generate both lib and rlib crate types\n'
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800467 if not self.root_pkg:
468 self.root_pkg = self.crate_name
469 if self.target:
470 self.device_supported = True
471 self.host_supported = True # assume host supported for all builds
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700472 if self.runner.args.no_host: # unless --no-host was specified
473 self.host_supported = False
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800474 self.cfgs = sorted(set(self.cfgs))
475 self.features = sorted(set(self.features))
476 self.codegens = sorted(set(self.codegens))
477 self.externs = sorted(set(self.externs))
478 self.core_externs = sorted(set(self.core_externs))
479 self.static_libs = sorted(set(self.static_libs))
480 self.shared_libs = sorted(set(self.shared_libs))
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700481 self.crate_types = sorted(set(self.crate_types))
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800482 self.decide_module_type()
483 self.module_name = altered_name(self.stem)
484 return self
485
486 def dump_line(self):
487 self.write('\n// Line ' + str(self.line_num) + ' ' + self.line)
488
489 def feature_list(self):
490 """Return a string of main_src + "feature_list"."""
491 pkg = self.main_src
492 if pkg.startswith('.../'): # keep only the main package name
493 pkg = re.sub('/.*', '', pkg[4:])
494 if not self.features:
495 return pkg
496 return pkg + ' "' + ','.join(self.features) + '"'
497
498 def dump_skip_crate(self, kind):
499 if self.debug:
500 self.write('\n// IGNORED: ' + kind + ' ' + self.main_src)
501 return self
502
503 def skip_crate(self):
504 """Return crate_name or a message if this crate should be skipped."""
Chih-Hung Hsiehe02dce12020-07-14 16:05:21 -0700505 if (is_build_crate_name(self.crate_name) or
506 self.crate_name in EXCLUDED_CRATES):
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800507 return self.crate_name
508 if is_dependent_file_path(self.main_src):
509 return 'dependent crate'
510 return ''
511
512 def dump(self):
513 """Dump all error/debug/module code to the output .bp file."""
514 self.runner.init_bp_file(self.outf_name)
515 with open(self.outf_name, 'a') as outf:
516 self.outf = outf
517 if self.errors:
518 self.dump_line()
519 self.write(self.errors)
520 elif self.skip_crate():
521 self.dump_skip_crate(self.skip_crate())
522 else:
523 if self.debug:
524 self.dump_debug_info()
525 self.dump_android_module()
526
527 def dump_debug_info(self):
528 """Dump parsed data, when cargo2android is called with --debug."""
529
530 def dump(name, value):
531 self.write('//%12s = %s' % (name, value))
532
533 def opt_dump(name, value):
534 if value:
535 dump(name, value)
536
537 def dump_list(fmt, values):
538 for v in values:
539 self.write(fmt % v)
540
541 self.dump_line()
542 dump('module_name', self.module_name)
543 dump('crate_name', self.crate_name)
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700544 dump('crate_types', self.crate_types)
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800545 dump('main_src', self.main_src)
546 dump('has_warning', self.has_warning)
547 dump('for_host', self.host_supported)
548 dump('for_device', self.device_supported)
549 dump('module_type', self.module_type)
550 opt_dump('target', self.target)
551 opt_dump('edition', self.edition)
552 opt_dump('emit_list', self.emit_list)
553 opt_dump('cap_lints', self.cap_lints)
554 dump_list('// cfg = %s', self.cfgs)
555 dump_list('// cfg = \'feature "%s"\'', self.features)
556 # TODO(chh): escape quotes in self.features, but not in other dump_list
557 dump_list('// codegen = %s', self.codegens)
558 dump_list('// externs = %s', self.externs)
559 dump_list('// -l static = %s', self.static_libs)
560 dump_list('// -l (dylib) = %s', self.shared_libs)
561
562 def dump_android_module(self):
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700563 """Dump one or more Android module definition, depending on crate_types."""
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700564 if len(self.crate_types) == 1:
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700565 self.dump_single_type_android_module()
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700566 return
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700567 if 'test' in self.crate_types:
568 self.write('\nERROR: multiple crate types cannot include test type')
569 return
570 # Dump one Android module per crate_type.
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700571 for crate_type in self.crate_types:
572 self.decide_one_module_type(crate_type)
573 self.dump_one_android_module(crate_type)
574
Chih-Hung Hsiehe02dce12020-07-14 16:05:21 -0700575 def build_default_name(self):
576 """Return a short and readable name for the rust_defaults module."""
577 # Choices: (1) root_pkg + '_defaults',
578 # (2) root_pkg + '_defaults_' + crate_name
579 # (3) root_pkg + '_defaults_' + main_src_basename_path
580 # (4) root_pkg + '_defaults_' + a_positive_sequence_number
581 name1 = altered_defaults(self.root_pkg) + '_defaults'
582 if self.runner.try_claim_module_name(name1, self):
583 return name1
584 name2 = name1 + '_' + self.crate_name
585 if self.runner.try_claim_module_name(name2, self):
586 return name2
587 name3 = name1 + '_' + self.main_src_basename_path()
588 if self.runner.try_claim_module_name(name3, self):
589 return name3
590 return self.runner.claim_module_name(name1, self, 0)
591
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700592 def dump_defaults_module(self):
593 """Dump a rust_defaults module to be shared by other modules."""
Chih-Hung Hsiehe02dce12020-07-14 16:05:21 -0700594 name = self.build_default_name()
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700595 self.defaults = name
596 self.write('\nrust_defaults {')
597 self.write(' name: "' + name + '",')
598 self.write(' crate_name: "' + self.crate_name + '",')
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -0700599 if len(self.srcs) == 1: # only one source file; share it in defaults
600 self.default_srcs = True
601 if self.has_warning and not self.cap_lints:
602 self.write(' // has rustc warnings')
603 self.write(' srcs: ["' + self.main_src + '"],')
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700604 if 'test' in self.crate_types:
605 self.write(' test_suites: ["general-tests"],')
606 self.write(' auto_gen_config: true,')
607 self.dump_edition_flags_libs()
608 self.write('}')
609
610 def dump_single_type_android_module(self):
611 """Dump one simple Android module, which has only one crate_type."""
612 crate_type = self.crate_types[0]
613 if crate_type != 'test':
614 # do not change self.stem or self.module_name
615 self.dump_one_android_module(crate_type)
616 return
617 # Dump one test module per source file, and separate host and device tests.
618 # crate_type == 'test'
619 if (self.host_supported and self.device_supported) or len(self.srcs) > 1:
620 self.srcs = sorted(set(self.srcs))
621 self.dump_defaults_module()
622 saved_srcs = self.srcs
623 for src in saved_srcs:
624 self.srcs = [src]
625 saved_device_supported = self.device_supported
626 saved_host_supported = self.host_supported
627 saved_main_src = self.main_src
628 self.main_src = src
629 if saved_host_supported:
630 self.device_supported = False
631 self.host_supported = True
632 self.module_name = self.test_module_name()
633 self.decide_one_module_type(crate_type)
634 self.dump_one_android_module(crate_type)
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -0700635 self.runner.add_test(self.outf_name, self.module_name, True)
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700636 if saved_device_supported:
637 self.device_supported = True
638 self.host_supported = False
639 self.module_name = self.test_module_name()
640 self.decide_one_module_type(crate_type)
641 self.dump_one_android_module(crate_type)
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -0700642 self.runner.add_test(self.outf_name, self.module_name, False)
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700643 self.host_supported = saved_host_supported
644 self.device_supported = saved_device_supported
645 self.main_src = saved_main_src
646 self.srcs = saved_srcs
647
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700648 def dump_one_android_module(self, crate_type):
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800649 """Dump one Android module definition."""
650 if not self.module_type:
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700651 self.write('\nERROR: unknown crate_type ' + crate_type)
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800652 return
653 self.write('\n' + self.module_type + ' {')
654 self.dump_android_core_properties()
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700655 if not self.defaults:
656 self.dump_edition_flags_libs()
657 if self.runner.args.host_first_multilib and self.host_supported and crate_type != 'test':
658 self.write(' compile_multilib: "first",')
659 self.write('}')
660
661 def dump_android_flags(self):
662 """Dump Android module flags property."""
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800663 cfg_fmt = '"--cfg %s"'
664 if self.cap_lints:
665 allowed = '"--cap-lints ' + self.cap_lints + '"'
666 if not self.cfgs:
667 self.write(' flags: [' + allowed + '],')
668 else:
669 self.write(' flags: [\n ' + allowed + ',')
670 self.dump_android_property_list_items(cfg_fmt, self.cfgs)
671 self.write(' ],')
672 else:
673 self.dump_android_property_list('flags', cfg_fmt, self.cfgs)
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700674
675 def dump_edition_flags_libs(self):
676 if self.edition:
677 self.write(' edition: "' + self.edition + '",')
678 self.dump_android_property_list('features', '"%s"', self.features)
679 self.dump_android_flags()
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800680 if self.externs:
681 self.dump_android_externs()
682 self.dump_android_property_list('static_libs', '"lib%s"', self.static_libs)
683 self.dump_android_property_list('shared_libs', '"lib%s"', self.shared_libs)
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800684
Chih-Hung Hsiehe02dce12020-07-14 16:05:21 -0700685 def main_src_basename_path(self):
686 return re.sub('/', '_', re.sub('.rs$', '', self.main_src))
687
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800688 def test_module_name(self):
689 """Return a unique name for a test module."""
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700690 # root_pkg+(_host|_device) + '_test_'+source_file_name
Chih-Hung Hsiehe02dce12020-07-14 16:05:21 -0700691 suffix = self.main_src_basename_path()
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700692 host_device = '_host'
693 if self.device_supported:
694 host_device = '_device'
695 return self.root_pkg + host_device + '_test_' + suffix
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800696
697 def decide_module_type(self):
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700698 # Use the first crate type for the default/first module.
699 crate_type = self.crate_types[0] if self.crate_types else ''
700 self.decide_one_module_type(crate_type)
701
702 def decide_one_module_type(self, crate_type):
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800703 """Decide which Android module type to use."""
704 host = '' if self.device_supported else '_host'
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700705 if crate_type == 'bin': # rust_binary[_host]
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800706 self.module_type = 'rust_binary' + host
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700707 # In rare cases like protobuf-codegen, the output binary name must
708 # be renamed to use as a plugin for protoc.
709 self.stem = altered_stem(self.crate_name)
710 self.module_name = altered_name(self.crate_name)
Chih-Hung Hsieh35ca4bc2020-07-10 16:49:51 -0700711 elif crate_type == 'lib': # rust_library[_host]
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700712 # TODO(chh): should this be rust_library[_host]?
713 # Assuming that Cargo.toml do not use both 'lib' and 'rlib',
714 # because we map them both to rlib.
Chih-Hung Hsieh35ca4bc2020-07-10 16:49:51 -0700715 self.module_type = 'rust_library' + host
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800716 self.stem = 'lib' + self.crate_name
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700717 self.module_name = altered_name(self.stem)
Chih-Hung Hsieh35ca4bc2020-07-10 16:49:51 -0700718 elif crate_type == 'rlib': # rust_library[_host]
719 self.module_type = 'rust_library' + host
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700720 self.stem = 'lib' + self.crate_name
721 self.module_name = altered_name(self.stem)
722 elif crate_type == 'dylib': # rust_library[_host]_dylib
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800723 self.module_type = 'rust_library' + host + '_dylib'
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700724 self.stem = 'lib' + self.crate_name
725 self.module_name = altered_name(self.stem) + '_dylib'
726 elif crate_type == 'cdylib': # rust_library[_host]_shared
727 self.module_type = 'rust_library' + host + '_shared'
728 self.stem = 'lib' + self.crate_name
729 self.module_name = altered_name(self.stem) + '_shared'
730 elif crate_type == 'staticlib': # rust_library[_host]_static
731 self.module_type = 'rust_library' + host + '_static'
732 self.stem = 'lib' + self.crate_name
733 self.module_name = altered_name(self.stem) + '_static'
734 elif crate_type == 'test': # rust_test[_host]
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800735 self.module_type = 'rust_test' + host
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700736 # Before do_merge, stem name is based on the --crate-name parameter.
737 # and test module name is based on stem.
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800738 self.stem = self.test_module_name()
739 # self.stem will be changed after merging with other tests.
740 # self.stem is NOT used for final test binary name.
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700741 # rust_test uses each source file base name as part of output file name.
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700742 # In do_merge, this function is called again, with a module_name.
743 # We make sure that the module name is unique in each package.
744 if self.module_name:
745 # Cargo uses "-C extra-filename=..." and "-C metadata=..." to add
746 # different suffixes and distinguish multiple tests of the same
747 # crate name. We ignore -C and use claim_module_name to get
748 # unique sequential suffix.
749 self.module_name = self.runner.claim_module_name(
750 self.module_name, self, 0)
751 # Now the module name is unique, stem should also match and unique.
752 self.stem = self.module_name
753 elif crate_type == 'proc-macro': # rust_proc_macro
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800754 self.module_type = 'rust_proc_macro'
755 self.stem = 'lib' + self.crate_name
Chih-Hung Hsieh8a1a2302020-04-03 14:33:33 -0700756 self.module_name = altered_name(self.stem)
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800757 else: # unknown module type, rust_prebuilt_dylib? rust_library[_host]?
758 self.module_type = ''
759 self.stem = ''
760
761 def dump_android_property_list_items(self, fmt, values):
762 for v in values:
763 # fmt has quotes, so we need escape_quotes(v)
764 self.write(' ' + (fmt % escape_quotes(v)) + ',')
765
766 def dump_android_property_list(self, name, fmt, values):
767 if values:
768 self.write(' ' + name + ': [')
769 self.dump_android_property_list_items(fmt, values)
770 self.write(' ],')
771
772 def dump_android_core_properties(self):
773 """Dump the module header, name, stem, etc."""
774 self.write(' name: "' + self.module_name + '",')
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700775 # see properties shared by dump_defaults_module
776 if self.defaults:
777 self.write(' defaults: ["' + self.defaults + '"],')
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800778 if self.stem != self.module_name:
779 self.write(' stem: "' + self.stem + '",')
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -0700780 if self.has_warning and not self.cap_lints and not self.default_srcs:
Chih-Hung Hsieh35ca4bc2020-07-10 16:49:51 -0700781 self.write(' // has rustc warnings')
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800782 if self.host_supported and self.device_supported:
783 self.write(' host_supported: true,')
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700784 if not self.defaults:
785 self.write(' crate_name: "' + self.crate_name + '",')
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800786 if len(self.srcs) > 1:
787 self.srcs = sorted(set(self.srcs))
788 self.dump_android_property_list('srcs', '"%s"', self.srcs)
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -0700789 elif not self.default_srcs:
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800790 self.write(' srcs: ["' + self.main_src + '"],')
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700791 if 'test' in self.crate_types and not self.defaults:
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800792 # self.root_pkg can have multiple test modules, with different *_tests[n]
793 # names, but their executables can all be installed under the same _tests
794 # directory. When built from Cargo.toml, all tests should have different
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -0700795 # file or crate names. So we used (root_pkg + '_tests') name as the
796 # relative_install_path.
797 # However, some package like 'slab' can have non-mergeable tests that
798 # must be separated by different module names. So, here we no longer
799 # emit relative_install_path.
800 # self.write(' relative_install_path: "' + self.root_pkg + '_tests",')
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800801 self.write(' test_suites: ["general-tests"],')
802 self.write(' auto_gen_config: true,')
803
804 def dump_android_externs(self):
805 """Dump the dependent rlibs and dylibs property."""
806 so_libs = list()
807 rust_libs = ''
808 deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$')
809 for lib in self.externs:
810 # normal value of lib: "libc = liblibc-*.rlib"
811 # strange case in rand crate: "getrandom_package = libgetrandom-*.rlib"
812 # we should use "libgetrandom", not "lib" + "getrandom_package"
813 groups = deps_libname.match(lib)
814 if groups is not None:
815 lib_name = groups.group(1)
816 else:
817 lib_name = re.sub(' .*$', '', lib)
818 if lib.endswith('.rlib') or lib.endswith('.rmeta'):
819 # On MacOS .rmeta is used when Linux uses .rlib or .rmeta.
820 rust_libs += ' "' + altered_name('lib' + lib_name) + '",\n'
821 elif lib.endswith('.so'):
822 so_libs.append(lib_name)
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -0700823 elif lib != 'proc_macro': # --extern proc_macro is special and ignored
824 rust_libs += ' // ERROR: unknown type of lib ' + lib + '\n'
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800825 if rust_libs:
Chih-Hung Hsieh35ca4bc2020-07-10 16:49:51 -0700826 self.write(' rustlibs: [\n' + rust_libs + ' ],')
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800827 # Are all dependent .so files proc_macros?
828 # TODO(chh): Separate proc_macros and dylib.
829 self.dump_android_property_list('proc_macros', '"lib%s"', so_libs)
830
831
832class ARObject(object):
833 """Information of an "ar" link command."""
834
835 def __init__(self, runner, outf_name):
836 # Remembered global runner and its members.
837 self.runner = runner
838 self.pkg = ''
839 self.outf_name = outf_name # path to Android.bp
840 # "ar" arguments
841 self.line_num = 1
842 self.line = ''
843 self.flags = '' # e.g. "crs"
844 self.lib = '' # e.g. "/.../out/lib*.a"
845 self.objs = list() # e.g. "/.../out/.../*.o"
846
847 def parse(self, pkg, line_num, args_line):
848 """Collect ar obj/lib file names."""
849 self.pkg = pkg
850 self.line_num = line_num
851 self.line = args_line
852 args = args_line.split()
853 num_args = len(args)
854 if num_args < 3:
855 print('ERROR: "ar" command has too few arguments', args_line)
856 else:
857 self.flags = unquote(args[0])
858 self.lib = unquote(args[1])
859 self.objs = sorted(set(map(unquote, args[2:])))
860 return self
861
862 def write(self, s):
863 self.outf.write(s + '\n')
864
865 def dump_debug_info(self):
866 self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line)
867 self.write('// ar_object for %12s' % self.pkg)
868 self.write('// flags = %s' % self.flags)
869 self.write('// lib = %s' % short_out_name(self.pkg, self.lib))
870 for o in self.objs:
871 self.write('// obj = %s' % short_out_name(self.pkg, o))
872
873 def dump_android_lib(self):
874 """Write cc_library_static into Android.bp."""
875 self.write('\ncc_library_static {')
876 self.write(' name: "' + file_base_name(self.lib) + '",')
877 self.write(' host_supported: true,')
878 if self.flags != 'crs':
879 self.write(' // ar flags = %s' % self.flags)
880 if self.pkg not in self.runner.pkg_obj2cc:
881 self.write(' ERROR: cannot find source files.\n}')
882 return
883 self.write(' srcs: [')
884 obj2cc = self.runner.pkg_obj2cc[self.pkg]
885 # Note: wflags are ignored.
886 dflags = list()
887 fflags = list()
888 for obj in self.objs:
889 self.write(' "' + short_out_name(self.pkg, obj2cc[obj].src) + '",')
890 # TODO(chh): union of dflags and flags of all obj
891 # Now, just a temporary hack that uses the last obj's flags
892 dflags = obj2cc[obj].dflags
893 fflags = obj2cc[obj].fflags
894 self.write(' ],')
895 self.write(' cflags: [')
896 self.write(' "-O3",') # TODO(chh): is this default correct?
897 self.write(' "-Wno-error",')
898 for x in fflags:
899 self.write(' "-f' + x + '",')
900 for x in dflags:
901 self.write(' "-D' + x + '",')
902 self.write(' ],')
903 self.write('}')
904
905 def dump(self):
906 """Dump error/debug/module info to the output .bp file."""
907 self.runner.init_bp_file(self.outf_name)
908 with open(self.outf_name, 'a') as outf:
909 self.outf = outf
910 if self.runner.args.debug:
911 self.dump_debug_info()
912 self.dump_android_lib()
913
914
915class CCObject(object):
916 """Information of a "cc" compilation command."""
917
918 def __init__(self, runner, outf_name):
919 # Remembered global runner and its members.
920 self.runner = runner
921 self.pkg = ''
922 self.outf_name = outf_name # path to Android.bp
923 # "cc" arguments
924 self.line_num = 1
925 self.line = ''
926 self.src = ''
927 self.obj = ''
928 self.dflags = list() # -D flags
929 self.fflags = list() # -f flags
930 self.iflags = list() # -I flags
931 self.wflags = list() # -W flags
932 self.other_args = list()
933
934 def parse(self, pkg, line_num, args_line):
935 """Collect cc compilation flags and src/out file names."""
936 self.pkg = pkg
937 self.line_num = line_num
938 self.line = args_line
939 args = args_line.split()
940 i = 0
941 while i < len(args):
942 arg = args[i]
943 if arg == '"-c"':
944 i += 1
945 if args[i].startswith('"-o'):
946 # ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c"
947 self.obj = unquote(args[i])[2:]
948 i += 1
949 self.src = unquote(args[i])
950 else:
951 self.src = unquote(args[i])
952 elif arg == '"-o"':
953 i += 1
954 self.obj = unquote(args[i])
955 elif arg == '"-I"':
956 i += 1
957 self.iflags.append(unquote(args[i]))
958 elif arg.startswith('"-D'):
959 self.dflags.append(unquote(args[i])[2:])
960 elif arg.startswith('"-f'):
961 self.fflags.append(unquote(args[i])[2:])
962 elif arg.startswith('"-W'):
963 self.wflags.append(unquote(args[i])[2:])
964 elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or
965 arg == '"-g3"'):
966 # ignore -O -m64 -g
967 self.other_args.append(unquote(args[i]))
968 i += 1
969 self.dflags = sorted(set(self.dflags))
970 self.fflags = sorted(set(self.fflags))
971 # self.wflags is not sorted because some are order sensitive
972 # and we ignore them anyway.
973 if self.pkg not in self.runner.pkg_obj2cc:
974 self.runner.pkg_obj2cc[self.pkg] = {}
975 self.runner.pkg_obj2cc[self.pkg][self.obj] = self
976 return self
977
978 def write(self, s):
979 self.outf.write(s + '\n')
980
981 def dump_debug_flags(self, name, flags):
982 self.write('// ' + name + ':')
983 for f in flags:
984 self.write('// %s' % f)
985
986 def dump(self):
987 """Dump only error/debug info to the output .bp file."""
988 if not self.runner.args.debug:
989 return
990 self.runner.init_bp_file(self.outf_name)
991 with open(self.outf_name, 'a') as outf:
992 self.outf = outf
993 self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line)
994 self.write('// cc_object for %12s' % self.pkg)
995 self.write('// src = %s' % short_out_name(self.pkg, self.src))
996 self.write('// obj = %s' % short_out_name(self.pkg, self.obj))
997 self.dump_debug_flags('-I flags', self.iflags)
998 self.dump_debug_flags('-D flags', self.dflags)
999 self.dump_debug_flags('-f flags', self.fflags)
1000 self.dump_debug_flags('-W flags', self.wflags)
1001 if self.other_args:
1002 self.dump_debug_flags('other args', self.other_args)
1003
1004
1005class Runner(object):
1006 """Main class to parse cargo -v output and print Android module definitions."""
1007
1008 def __init__(self, args):
1009 self.bp_files = set() # Remember all output Android.bp files.
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -07001010 self.test_mappings = {} # Map from Android.bp file path to TestMapping.
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001011 self.root_pkg = '' # name of package in ./Cargo.toml
1012 # Saved flags, modes, and data.
1013 self.args = args
1014 self.dry_run = not args.run
1015 self.skip_cargo = args.skipcargo
1016 # All cc/ar objects, crates, dependencies, and warning files
1017 self.cc_objects = list()
1018 self.pkg_obj2cc = {}
1019 # pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i]
1020 self.ar_objects = list()
1021 self.crates = list()
1022 self.dependencies = list() # dependent and build script crates
1023 self.warning_files = set()
1024 # Keep a unique mapping from (module name) to crate
1025 self.name_owners = {}
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -07001026 # Save and dump all errors from cargo to Android.bp.
1027 self.errors = ''
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001028 # Default action is cargo clean, followed by build or user given actions.
1029 if args.cargo:
1030 self.cargo = ['clean'] + args.cargo
1031 else:
1032 self.cargo = ['clean', 'build']
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -07001033 if args.no_host: # do not run "cargo build" for host
1034 self.cargo = ['clean']
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001035 default_target = '--target x86_64-unknown-linux-gnu'
1036 if args.device:
1037 self.cargo.append('build ' + default_target)
1038 if args.tests:
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -07001039 if not args.no_host:
1040 self.cargo.append('build --tests')
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001041 self.cargo.append('build --tests ' + default_target)
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -07001042 elif args.tests and not args.no_host:
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001043 self.cargo.append('build --tests')
1044
1045 def init_bp_file(self, name):
1046 if name not in self.bp_files:
1047 self.bp_files.add(name)
1048 with open(name, 'w') as outf:
Andrew Walbran80e90be2020-06-09 14:33:18 +01001049 outf.write(ANDROID_BP_HEADER.format(args=' '.join(sys.argv[1:])))
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001050
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -07001051 def dump_test_mapping_files(self):
1052 if self.dry_run:
1053 print('Dry-run skip dump of TEST_MAPPING')
1054 else:
1055 for bp_file_name in self.test_mappings:
1056 name = os.path.join(os.path.dirname(bp_file_name), 'TEST_MAPPING')
1057 self.test_mappings[bp_file_name].dump(name)
1058 return self
1059
1060 def add_test(self, bp_file_name, test_name, host):
1061 if bp_file_name not in self.test_mappings:
1062 self.test_mappings[bp_file_name] = TestMapping()
1063 mapping = self.test_mappings[bp_file_name]
1064 mapping.add_test(test_name, host)
1065
Chih-Hung Hsiehe02dce12020-07-14 16:05:21 -07001066 def try_claim_module_name(self, name, owner):
1067 """Reserve and return True if it has not been reserved yet."""
1068 if name not in self.name_owners or owner == self.name_owners[name]:
1069 self.name_owners[name] = owner
1070 return True
1071 return False
1072
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001073 def claim_module_name(self, prefix, owner, counter):
1074 """Return prefix if not owned yet, otherwise, prefix+str(counter)."""
1075 while True:
1076 name = prefix
1077 if counter > 0:
Chih-Hung Hsiehe02dce12020-07-14 16:05:21 -07001078 name += '_' + str(counter)
1079 if self.try_claim_module_name(name, owner):
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001080 return name
1081 counter += 1
1082
1083 def find_root_pkg(self):
1084 """Read name of [package] in ./Cargo.toml."""
1085 if not os.path.exists('./Cargo.toml'):
1086 return
1087 with open('./Cargo.toml', 'r') as inf:
1088 pkg_section = re.compile(r'^ *\[package\]')
1089 name = re.compile('^ *name *= * "([^"]*)"')
1090 in_pkg = False
1091 for line in inf:
1092 if in_pkg:
1093 if name.match(line):
1094 self.root_pkg = name.match(line).group(1)
1095 break
1096 else:
1097 in_pkg = pkg_section.match(line) is not None
1098
1099 def run_cargo(self):
1100 """Calls cargo -v and save its output to ./cargo.out."""
1101 if self.skip_cargo:
1102 return self
1103 cargo = './Cargo.toml'
1104 if not os.access(cargo, os.R_OK):
1105 print('ERROR: Cannot find or read', cargo)
1106 return self
1107 if not self.dry_run and os.path.exists('cargo.out'):
1108 os.remove('cargo.out')
1109 cmd_tail = ' --target-dir ' + TARGET_TMP + ' >> cargo.out 2>&1'
1110 for c in self.cargo:
1111 features = ''
Chih-Hung Hsieh6c8d52f2020-03-30 18:28:52 -07001112 if c != 'clean':
1113 if self.args.features is not None:
1114 features = ' --no-default-features'
1115 if self.args.features:
1116 features += ' --features ' + self.args.features
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001117 cmd = 'cargo -vv ' if self.args.vv else 'cargo -v '
1118 cmd += c + features + cmd_tail
1119 if self.args.rustflags and c != 'clean':
1120 cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd
1121 if self.dry_run:
1122 print('Dry-run skip:', cmd)
1123 else:
1124 if self.args.verbose:
1125 print('Running:', cmd)
1126 with open('cargo.out', 'a') as cargo_out:
1127 cargo_out.write('### Running: ' + cmd + '\n')
1128 os.system(cmd)
1129 return self
1130
1131 def dump_dependencies(self):
1132 """Append dependencies and their features to Android.bp."""
1133 if not self.dependencies:
1134 return
1135 dependent_list = list()
1136 for c in self.dependencies:
1137 dependent_list.append(c.feature_list())
1138 sorted_dependencies = sorted(set(dependent_list))
1139 self.init_bp_file('Android.bp')
1140 with open('Android.bp', 'a') as outf:
1141 outf.write('\n// dependent_library ["feature_list"]\n')
1142 for s in sorted_dependencies:
1143 outf.write('// ' + s + '\n')
1144
1145 def dump_pkg_obj2cc(self):
1146 """Dump debug info of the pkg_obj2cc map."""
1147 if not self.args.debug:
1148 return
1149 self.init_bp_file('Android.bp')
1150 with open('Android.bp', 'a') as outf:
1151 sorted_pkgs = sorted(self.pkg_obj2cc.keys())
1152 for pkg in sorted_pkgs:
1153 if not self.pkg_obj2cc[pkg]:
1154 continue
1155 outf.write('\n// obj => src for %s\n' % pkg)
1156 obj2cc = self.pkg_obj2cc[pkg]
1157 for obj in sorted(obj2cc.keys()):
1158 outf.write('// ' + short_out_name(pkg, obj) + ' => ' +
1159 short_out_name(pkg, obj2cc[obj].src) + '\n')
1160
1161 def gen_bp(self):
1162 """Parse cargo.out and generate Android.bp files."""
1163 if self.dry_run:
1164 print('Dry-run skip: read', CARGO_OUT, 'write Android.bp')
1165 elif os.path.exists(CARGO_OUT):
1166 self.find_root_pkg()
1167 with open(CARGO_OUT, 'r') as cargo_out:
1168 self.parse(cargo_out, 'Android.bp')
1169 self.crates.sort(key=get_module_name)
1170 for obj in self.cc_objects:
1171 obj.dump()
1172 self.dump_pkg_obj2cc()
1173 for crate in self.crates:
1174 crate.dump()
1175 dumped_libs = set()
1176 for lib in self.ar_objects:
1177 if lib.pkg == self.root_pkg:
1178 lib_name = file_base_name(lib.lib)
1179 if lib_name not in dumped_libs:
1180 dumped_libs.add(lib_name)
1181 lib.dump()
1182 if self.args.dependencies and self.dependencies:
1183 self.dump_dependencies()
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -07001184 if self.errors:
1185 self.append_to_bp('\nErrors in ' + CARGO_OUT + ':\n' + self.errors)
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001186 return self
1187
1188 def add_ar_object(self, obj):
1189 self.ar_objects.append(obj)
1190
1191 def add_cc_object(self, obj):
1192 self.cc_objects.append(obj)
1193
1194 def add_crate(self, crate):
1195 """Merge crate with someone in crates, or append to it. Return crates."""
1196 if crate.skip_crate():
1197 if self.args.debug: # include debug info of all crates
1198 self.crates.append(crate)
1199 if self.args.dependencies: # include only dependent crates
1200 if (is_dependent_file_path(crate.main_src) and
1201 not is_build_crate_name(crate.crate_name)):
1202 self.dependencies.append(crate)
1203 else:
1204 for c in self.crates:
1205 if c.merge(crate, 'Android.bp'):
1206 return
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -07001207 # If not merged, decide module type and name now.
1208 crate.decide_module_type()
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001209 self.crates.append(crate)
1210
1211 def find_warning_owners(self):
1212 """For each warning file, find its owner crate."""
1213 missing_owner = False
1214 for f in self.warning_files:
1215 cargo_dir = '' # find lowest crate, with longest path
1216 owner = None # owner crate of this warning
1217 for c in self.crates:
1218 if (f.startswith(c.cargo_dir + '/') and
1219 len(cargo_dir) < len(c.cargo_dir)):
1220 cargo_dir = c.cargo_dir
1221 owner = c
1222 if owner:
1223 owner.has_warning = True
1224 else:
1225 missing_owner = True
1226 if missing_owner and os.path.exists('Cargo.toml'):
1227 # owner is the root cargo, with empty cargo_dir
1228 for c in self.crates:
1229 if not c.cargo_dir:
1230 c.has_warning = True
1231
1232 def rustc_command(self, n, rustc_line, line, outf_name):
1233 """Process a rustc command line from cargo -vv output."""
1234 # cargo build -vv output can have multiple lines for a rustc command
1235 # due to '\n' in strings for environment variables.
1236 # strip removes leading spaces and '\n' at the end
1237 new_rustc = (rustc_line.strip() + line) if rustc_line else line
1238 # Use an heuristic to detect the completions of a multi-line command.
1239 # This might fail for some very rare case, but easy to fix manually.
1240 if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0:
1241 return new_rustc
1242 if RUSTC_VV_CMD_ARGS.match(new_rustc):
1243 args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(1)
1244 self.add_crate(Crate(self, outf_name).parse(n, args))
1245 else:
1246 self.assert_empty_vv_line(new_rustc)
1247 return ''
1248
1249 def cc_ar_command(self, n, groups, outf_name):
1250 pkg = groups.group(1)
1251 line = groups.group(3)
1252 if groups.group(2) == 'cc':
1253 self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line))
1254 else:
1255 self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line))
1256
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -07001257 def append_to_bp(self, line):
1258 self.init_bp_file('Android.bp')
1259 with open('Android.bp', 'a') as outf:
1260 outf.write(line)
1261
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001262 def assert_empty_vv_line(self, line):
1263 if line: # report error if line is not empty
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -07001264 self.append_to_bp('ERROR -vv line: ' + line)
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001265 return ''
1266
1267 def parse(self, inf, outf_name):
1268 """Parse rustc and warning messages in inf, return a list of Crates."""
1269 n = 0 # line number
1270 prev_warning = False # true if the previous line was warning: ...
1271 rustc_line = '' # previous line(s) matching RUSTC_VV_PAT
1272 for line in inf:
1273 n += 1
1274 if line.startswith('warning: '):
1275 prev_warning = True
1276 rustc_line = self.assert_empty_vv_line(rustc_line)
1277 continue
1278 new_rustc = ''
1279 if RUSTC_PAT.match(line):
1280 args_line = RUSTC_PAT.match(line).group(1)
1281 self.add_crate(Crate(self, outf_name).parse(n, args_line))
1282 self.assert_empty_vv_line(rustc_line)
1283 elif rustc_line or RUSTC_VV_PAT.match(line):
1284 new_rustc = self.rustc_command(n, rustc_line, line, outf_name)
1285 elif CC_AR_VV_PAT.match(line):
1286 self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name)
1287 elif prev_warning and WARNING_FILE_PAT.match(line):
1288 self.assert_empty_vv_line(rustc_line)
1289 fpath = WARNING_FILE_PAT.match(line).group(1)
1290 if fpath[0] != '/': # ignore absolute path
1291 self.warning_files.add(fpath)
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -07001292 elif line.startswith('error: ') or line.startswith('error[E'):
1293 self.errors += line
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001294 prev_warning = False
1295 rustc_line = new_rustc
1296 self.find_warning_owners()
1297
1298
1299def parse_args():
1300 """Parse main arguments."""
1301 parser = argparse.ArgumentParser('cargo2android')
1302 parser.add_argument(
1303 '--cargo',
1304 action='append',
1305 metavar='args_string',
1306 help=('extra cargo build -v args in a string, ' +
1307 'each --cargo flag calls cargo build -v once'))
1308 parser.add_argument(
1309 '--debug',
1310 action='store_true',
1311 default=False,
1312 help='dump debug info into Android.bp')
1313 parser.add_argument(
1314 '--dependencies',
1315 action='store_true',
1316 default=False,
1317 help='dump debug info of dependent crates')
1318 parser.add_argument(
1319 '--device',
1320 action='store_true',
1321 default=False,
1322 help='run cargo also for a default device target')
1323 parser.add_argument(
Chih-Hung Hsieh3725e082020-07-12 00:51:20 -07001324 '--no-host',
1325 action='store_true',
1326 default=False,
1327 help='do not run cargo for the host; only for the device target')
1328 parser.add_argument(
1329 '--host-first-multilib',
1330 action='store_true',
1331 default=False,
1332 help=('add a compile_multilib:"first" property ' +
1333 'to Android.bp host modules.'))
1334 parser.add_argument(
Chih-Hung Hsieh185052a2020-05-07 14:48:57 -07001335 '--features',
1336 type=str,
Chih-Hung Hsieh6c8d52f2020-03-30 18:28:52 -07001337 help=('pass features to cargo build, ' +
1338 'empty string means no default features'))
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001339 parser.add_argument(
1340 '--onefile',
1341 action='store_true',
1342 default=False,
1343 help=('output all into one ./Android.bp, default will generate ' +
1344 'one Android.bp per Cargo.toml in subdirectories'))
1345 parser.add_argument(
1346 '--run',
1347 action='store_true',
1348 default=False,
1349 help='run it, default is dry-run')
1350 parser.add_argument('--rustflags', type=str, help='passing flags to rustc')
1351 parser.add_argument(
1352 '--skipcargo',
1353 action='store_true',
1354 default=False,
1355 help='skip cargo command, parse cargo.out, and generate Android.bp')
1356 parser.add_argument(
1357 '--tests',
1358 action='store_true',
1359 default=False,
1360 help='run cargo build --tests after normal build')
1361 parser.add_argument(
1362 '--verbose',
1363 action='store_true',
1364 default=False,
1365 help='echo executed commands')
1366 parser.add_argument(
1367 '--vv',
1368 action='store_true',
1369 default=False,
1370 help='run cargo with -vv instead of default -v')
1371 return parser.parse_args()
1372
1373
1374def main():
1375 args = parse_args()
1376 if not args.run: # default is dry-run
1377 print(DRY_RUN_NOTE)
Chih-Hung Hsiehf7eff152020-07-16 15:36:22 -07001378 Runner(args).run_cargo().gen_bp().dump_test_mapping_files()
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001379
1380
1381if __name__ == '__main__':
1382 main()