blob: 9f3dd575a74724c0272ac32422945040cd4ac186 [file] [log] [blame]
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001#!/usr/bin/env python
2#
3# Copyright (C) 2019 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""Call cargo -v, parse its output, and generate Android.bp.
17
18Usage: Run this script in a crate workspace root directory.
19The Cargo.toml file should work at least for the host platform.
20
21(1) Without other flags, "cargo2android.py --run"
22 calls cargo clean, calls cargo build -v, and generates Android.bp.
23 The cargo build only generates crates for the host,
24 without test crates.
25
26(2) To build crates for both host and device in Android.bp, use the
27 --device flag, for example:
28 cargo2android.py --run --device
29
30 This is equivalent to using the --cargo flag to add extra builds:
31 cargo2android.py --run
32 --cargo "build"
33 --cargo "build --target x86_64-unknown-linux-gnu"
34
35 On MacOS, use x86_64-apple-darwin as target triple.
36 Here the host target triple is used as a fake cross compilation target.
37 If the crate's Cargo.toml and environment configuration works for an
38 Android target, use that target triple as the cargo build flag.
39
40(3) To build default and test crates, for host and device, use both
41 --device and --tests flags:
42 cargo2android.py --run --device --tests
43
44 This is equivalent to using the --cargo flag to add extra builds:
45 cargo2android.py --run
46 --cargo "build"
47 --cargo "build --tests"
48 --cargo "build --target x86_64-unknown-linux-gnu"
49 --cargo "build --tests --target x86_64-unknown-linux-gnu"
50
51Since Android rust builds by default treat all warnings as errors,
52if there are rustc warning messages, this script will add
53deny_warnings:false to the owner crate module in Android.bp.
54"""
55
56from __future__ import print_function
57
58import argparse
59import os
60import os.path
61import re
62
63RENAME_MAP = {
64 # This map includes all changes to the default rust library module
65 # names to resolve name conflicts or avoid confusion.
66 'libbacktrace': 'libbacktrace_rust',
67 'libgcc': 'libgcc_rust',
68 'liblog': 'liblog_rust',
69 'libsync': 'libsync_rust',
70 'libx86_64': 'libx86_64_rust',
71}
72
73# Header added to all generated Android.bp files.
74ANDROID_BP_HEADER = '// This file is generated by cargo2android.py.\n'
75
76CARGO_OUT = 'cargo.out' # Name of file to keep cargo build -v output.
77
78TARGET_TMP = 'target.tmp' # Name of temporary output directory.
79
80# Message to be displayed when this script is called without the --run flag.
81DRY_RUN_NOTE = (
82 'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' +
83 'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' +
84 'and writes to Android.bp in the current and subdirectories.\n\n' +
85 'To do do all of the above, use the --run flag.\n' +
86 'See --help for other flags, and more usage notes in this script.\n')
87
88# Cargo -v output of a call to rustc.
89RUSTC_PAT = re.compile('^ +Running `rustc (.*)`$')
90
91# Cargo -vv output of a call to rustc could be split into multiple lines.
92# Assume that the first line will contain some CARGO_* env definition.
93RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$')
94# The combined -vv output rustc command line pattern.
95RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$')
96
97# Cargo -vv output of a "cc" or "ar" command; all in one line.
98CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$')
99# Some package, such as ring-0.13.5, has pattern '... running "cc"'.
100
101# Rustc output of file location path pattern for a warning message.
102WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+')
103
104# Rust package name with suffix -d1.d2.d3.
105VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+$')
106
107
108def altered_name(name):
109 return RENAME_MAP[name] if (name in RENAME_MAP) else name
110
111
112def is_build_crate_name(name):
113 # We added special prefix to build script crate names.
114 return name.startswith('build_script_')
115
116
117def is_dependent_file_path(path):
118 # Absolute or dependent '.../' paths are not main files of this crate.
119 return path.startswith('/') or path.startswith('.../')
120
121
122def get_module_name(crate): # to sort crates in a list
123 return crate.module_name
124
125
126def pkg2crate_name(s):
127 return s.replace('-', '_').replace('.', '_')
128
129
130def file_base_name(path):
131 return os.path.splitext(os.path.basename(path))[0]
132
133
134def test_base_name(path):
135 return pkg2crate_name(file_base_name(path))
136
137
138def unquote(s): # remove quotes around str
139 if s and len(s) > 1 and s[0] == '"' and s[-1] == '"':
140 return s[1:-1]
141 return s
142
143
144def remove_version_suffix(s): # remove -d1.d2.d3 suffix
145 if VERSION_SUFFIX_PAT.match(s):
146 return VERSION_SUFFIX_PAT.match(s).group(1)
147 return s
148
149
150def short_out_name(pkg, s): # replace /.../pkg-*/out/* with .../out/*
151 return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s)
152
153
154def escape_quotes(s): # replace '"' with '\\"'
155 return s.replace('"', '\\"')
156
157
158class Crate(object):
159 """Information of a Rust crate to collect/emit for an Android.bp module."""
160
161 def __init__(self, runner, outf_name):
162 # Remembered global runner and its members.
163 self.runner = runner
164 self.debug = runner.args.debug
165 self.cargo_dir = '' # directory of my Cargo.toml
166 self.outf_name = outf_name # path to Android.bp
167 self.outf = None # open file handle of outf_name during dump*
168 # Variants/results that could be merged from multiple rustc lines.
169 self.host_supported = False
170 self.device_supported = False
171 self.has_warning = False
172 # Android module properties derived from rustc parameters.
173 self.module_name = '' # unique in Android build system
174 self.module_type = '' # rust_{binary,library,test}[_host] etc.
175 self.root_pkg = '' # parent package name of a sub/test packge, from -L
176 self.srcs = list() # main_src or merged multiple source files
177 self.stem = '' # real base name of output file
178 # Kept parsed status
179 self.errors = '' # all errors found during parsing
180 self.line_num = 1 # runner told input source line number
181 self.line = '' # original rustc command line parameters
182 # Parameters collected from rustc command line.
183 self.crate_name = '' # follows --crate-name
184 self.main_src = '' # follows crate_name parameter, shortened
185 self.crate_type = '' # bin|lib|test (see --test flag)
186 self.cfgs = list() # follows --cfg, without feature= prefix
187 self.features = list() # follows --cfg, name in 'feature="..."'
188 self.codegens = list() # follows -C, some ignored
189 self.externs = list() # follows --extern
190 self.core_externs = list() # first part of self.externs elements
191 self.static_libs = list() # e.g. -l static=host_cpuid
192 self.shared_libs = list() # e.g. -l dylib=wayland-client, -l z
193 self.cap_lints = '' # follows --cap-lints
194 self.emit_list = '' # e.g., --emit=dep-info,metadata,link
195 self.edition = '2015' # rustc default, e.g., --edition=2018
196 self.target = '' # follows --target
197
198 def write(self, s):
199 # convenient way to output one line at a time with EOL.
200 self.outf.write(s + '\n')
201
202 def same_flags(self, other):
203 # host_supported, device_supported, has_warning are not compared but merged
204 # target is not compared, to merge different target/host modules
205 # externs is not compared; only core_externs is compared
206 return (not self.errors and not other.errors and
207 self.edition == other.edition and
208 self.cap_lints == other.cap_lints and
209 self.emit_list == other.emit_list and
210 self.core_externs == other.core_externs and
211 self.codegens == other.codegens and
212 self.features == other.features and
213 self.static_libs == other.static_libs and
214 self.shared_libs == other.shared_libs and self.cfgs == other.cfgs)
215
216 def merge_host_device(self, other):
217 """Returns true if attributes are the same except host/device support."""
218 return (self.crate_name == other.crate_name and
219 self.crate_type == other.crate_type and
220 self.main_src == other.main_src and self.stem == other.stem and
221 self.root_pkg == other.root_pkg and not self.skip_crate() and
222 self.same_flags(other))
223
224 def merge_test(self, other):
225 """Returns true if self and other are tests of same root_pkg."""
226 # Before merger, each test has its own crate_name.
227 # A merged test uses its source file base name as output file name,
228 # so a test is mergeable only if its base name equals to its crate name.
229 return (self.crate_type == other.crate_type and
230 self.crate_type == 'test' and self.root_pkg == other.root_pkg and
231 not self.skip_crate() and
232 other.crate_name == test_base_name(other.main_src) and
233 (len(self.srcs) > 1 or
234 (self.crate_name == test_base_name(self.main_src)) and
235 self.host_supported == other.host_supported and
236 self.device_supported == other.device_supported) and
237 self.same_flags(other))
238
239 def merge(self, other, outf_name):
240 """Try to merge crate into self."""
241 should_merge_host_device = self.merge_host_device(other)
242 should_merge_test = False
243 if not should_merge_host_device:
244 should_merge_test = self.merge_test(other)
245 # A for-device test crate can be merged with its for-host version,
246 # or merged with a different test for the same host or device.
247 # Since we run cargo once for each device or host, test crates for the
248 # first device or host will be merged first. Then test crates for a
249 # different device or host should be allowed to be merged into a
250 # previously merged one, maybe for a different device or host.
251 if should_merge_host_device or should_merge_test:
252 self.runner.init_bp_file(outf_name)
253 with open(outf_name, 'a') as outf: # to write debug info
254 self.outf = outf
255 other.outf = outf
256 self.do_merge(other, should_merge_test)
257 return True
258 return False
259
260 def do_merge(self, other, should_merge_test):
261 """Merge attributes of other to self."""
262 if self.debug:
263 self.write('\n// Before merge definition (1):')
264 self.dump_debug_info()
265 self.write('\n// Before merge definition (2):')
266 other.dump_debug_info()
267 # Merge properties of other to self.
268 self.host_supported = self.host_supported or other.host_supported
269 self.device_supported = self.device_supported or other.device_supported
270 self.has_warning = self.has_warning or other.has_warning
271 if not self.target: # okay to keep only the first target triple
272 self.target = other.target
273 # decide_module_type sets up default self.stem,
274 # which can be changed if self is a merged test module.
275 self.decide_module_type()
276 if should_merge_test:
277 self.srcs.append(other.main_src)
278 # use a short unique name as the merged module name.
279 prefix = self.root_pkg + '_tests'
280 self.module_name = self.runner.claim_module_name(prefix, self, 0)
281 self.stem = self.module_name
282 # This normalized root_pkg name although might be the same
283 # as other module's crate_name, it is not actually used for
284 # output file name. A merged test module always have multiple
285 # source files and each source file base name is used as
286 # its output file name.
287 self.crate_name = pkg2crate_name(self.root_pkg)
288 if self.debug:
289 self.write('\n// After merge definition (1):')
290 self.dump_debug_info()
291
292 def find_cargo_dir(self):
293 """Deepest directory with Cargo.toml and contains the main_src."""
294 if not is_dependent_file_path(self.main_src):
295 dir_name = os.path.dirname(self.main_src)
296 while dir_name:
297 if os.path.exists(dir_name + '/Cargo.toml'):
298 self.cargo_dir = dir_name
299 return
300 dir_name = os.path.dirname(dir_name)
301
302 def parse(self, line_num, line):
303 """Find important rustc arguments to convert to Android.bp properties."""
304 self.line_num = line_num
305 self.line = line
306 args = line.split() # Loop through every argument of rustc.
307 i = 0
308 while i < len(args):
309 arg = args[i]
310 if arg == '--crate-name':
311 self.crate_name = args[i + 1]
312 i += 2
313 # shorten imported crate main source path
314 self.main_src = re.sub('^/[^ ]*/registry/src/', '.../', args[i])
315 self.main_src = re.sub('^.../github.com-[0-9a-f]*/', '.../',
316 self.main_src)
317 self.find_cargo_dir()
318 if self.cargo_dir and not self.runner.args.onefile:
319 # Write to Android.bp in the subdirectory with Cargo.toml.
320 self.outf_name = self.cargo_dir + '/Android.bp'
321 self.main_src = self.main_src[len(self.cargo_dir) + 1:]
322 elif arg == '--crate-type':
323 i += 1
324 if self.crate_type:
325 self.errors += ' ERROR: multiple --crate-type '
326 self.errors += self.crate_type + ' ' + args[i] + '\n'
327 # TODO(chh): handle multiple types, e.g. lexical-core-0.4.6 has
328 # crate-type = ["lib", "staticlib", "cdylib"]
329 # output: debug/liblexical_core.{a,so,rlib}
330 # cargo calls rustc with multiple --crate-type flags.
331 # rustc can accept:
332 # --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]
333 # Comma separated list of types of crates for the compiler to emit
334 self.crate_type = args[i]
335 elif arg == '--test':
336 # only --test or --crate-type should appear once
337 if self.crate_type:
338 self.errors += (' ERROR: found both --test and --crate-type ' +
339 self.crate_type + '\n')
340 else:
341 self.crate_type = 'test'
342 elif arg == '--target':
343 i += 1
344 self.target = args[i]
345 elif arg == '--cfg':
346 i += 1
347 if args[i].startswith('\'feature='):
348 self.features.append(unquote(args[i].replace('\'feature=', '')[:-1]))
349 else:
350 self.cfgs.append(args[i])
351 elif arg == '--extern':
352 i += 1
353 extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i])
354 self.externs.append(extern_names)
355 self.core_externs.append(re.sub(' = .*', '', extern_names))
356 elif arg == '-C': # codegen options
357 i += 1
358 # ignore options not used in Android
359 if not (args[i].startswith('debuginfo=') or
360 args[i].startswith('extra-filename=') or
361 args[i].startswith('incremental=') or
362 args[i].startswith('metadata=')):
363 self.codegens.append(args[i])
364 elif arg == '--cap-lints':
365 i += 1
366 self.cap_lints = args[i]
367 elif arg == '-L':
368 i += 1
369 if args[i].startswith('dependency=') and args[i].endswith('/deps'):
370 if '/' + TARGET_TMP + '/' in args[i]:
371 self.root_pkg = re.sub(
372 '^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i]))
373 else:
374 self.root_pkg = re.sub('^.*/', '',
375 re.sub('/[^/]+/[^/]+/deps$', '', args[i]))
376 self.root_pkg = remove_version_suffix(self.root_pkg)
377 elif arg == '-l':
378 i += 1
379 if args[i].startswith('static='):
380 self.static_libs.append(re.sub('static=', '', args[i]))
381 elif args[i].startswith('dylib='):
382 self.shared_libs.append(re.sub('dylib=', '', args[i]))
383 else:
384 self.shared_libs.append(args[i])
385 elif arg == '--out-dir' or arg == '--color': # ignored
386 i += 1
387 elif arg.startswith('--error-format=') or arg.startswith('--json='):
388 _ = arg # ignored
389 elif arg.startswith('--emit='):
390 self.emit_list = arg.replace('--emit=', '')
391 elif arg.startswith('--edition='):
392 self.edition = arg.replace('--edition=', '')
393 else:
394 self.errors += 'ERROR: unknown ' + arg + '\n'
395 i += 1
396 if not self.crate_name:
397 self.errors += 'ERROR: missing --crate-name\n'
398 if not self.main_src:
399 self.errors += 'ERROR: missing main source file\n'
400 else:
401 self.srcs.append(self.main_src)
402 if not self.crate_type:
403 # Treat "--cfg test" as "--test"
404 if 'test' in self.cfgs:
405 self.crate_type = 'test'
406 else:
407 self.errors += 'ERROR: missing --crate-type\n'
408 if not self.root_pkg:
409 self.root_pkg = self.crate_name
410 if self.target:
411 self.device_supported = True
412 self.host_supported = True # assume host supported for all builds
413 self.cfgs = sorted(set(self.cfgs))
414 self.features = sorted(set(self.features))
415 self.codegens = sorted(set(self.codegens))
416 self.externs = sorted(set(self.externs))
417 self.core_externs = sorted(set(self.core_externs))
418 self.static_libs = sorted(set(self.static_libs))
419 self.shared_libs = sorted(set(self.shared_libs))
420 self.decide_module_type()
421 self.module_name = altered_name(self.stem)
422 return self
423
424 def dump_line(self):
425 self.write('\n// Line ' + str(self.line_num) + ' ' + self.line)
426
427 def feature_list(self):
428 """Return a string of main_src + "feature_list"."""
429 pkg = self.main_src
430 if pkg.startswith('.../'): # keep only the main package name
431 pkg = re.sub('/.*', '', pkg[4:])
432 if not self.features:
433 return pkg
434 return pkg + ' "' + ','.join(self.features) + '"'
435
436 def dump_skip_crate(self, kind):
437 if self.debug:
438 self.write('\n// IGNORED: ' + kind + ' ' + self.main_src)
439 return self
440
441 def skip_crate(self):
442 """Return crate_name or a message if this crate should be skipped."""
443 if is_build_crate_name(self.crate_name):
444 return self.crate_name
445 if is_dependent_file_path(self.main_src):
446 return 'dependent crate'
447 return ''
448
449 def dump(self):
450 """Dump all error/debug/module code to the output .bp file."""
451 self.runner.init_bp_file(self.outf_name)
452 with open(self.outf_name, 'a') as outf:
453 self.outf = outf
454 if self.errors:
455 self.dump_line()
456 self.write(self.errors)
457 elif self.skip_crate():
458 self.dump_skip_crate(self.skip_crate())
459 else:
460 if self.debug:
461 self.dump_debug_info()
462 self.dump_android_module()
463
464 def dump_debug_info(self):
465 """Dump parsed data, when cargo2android is called with --debug."""
466
467 def dump(name, value):
468 self.write('//%12s = %s' % (name, value))
469
470 def opt_dump(name, value):
471 if value:
472 dump(name, value)
473
474 def dump_list(fmt, values):
475 for v in values:
476 self.write(fmt % v)
477
478 self.dump_line()
479 dump('module_name', self.module_name)
480 dump('crate_name', self.crate_name)
481 dump('crate_type', self.crate_type)
482 dump('main_src', self.main_src)
483 dump('has_warning', self.has_warning)
484 dump('for_host', self.host_supported)
485 dump('for_device', self.device_supported)
486 dump('module_type', self.module_type)
487 opt_dump('target', self.target)
488 opt_dump('edition', self.edition)
489 opt_dump('emit_list', self.emit_list)
490 opt_dump('cap_lints', self.cap_lints)
491 dump_list('// cfg = %s', self.cfgs)
492 dump_list('// cfg = \'feature "%s"\'', self.features)
493 # TODO(chh): escape quotes in self.features, but not in other dump_list
494 dump_list('// codegen = %s', self.codegens)
495 dump_list('// externs = %s', self.externs)
496 dump_list('// -l static = %s', self.static_libs)
497 dump_list('// -l (dylib) = %s', self.shared_libs)
498
499 def dump_android_module(self):
500 """Dump one Android module definition."""
501 if not self.module_type:
502 self.write('\nERROR: unknown crate_type ' + self.crate_type)
503 return
504 self.write('\n' + self.module_type + ' {')
505 self.dump_android_core_properties()
506 if self.edition:
507 self.write(' edition: "' + self.edition + '",')
508 self.dump_android_property_list('features', '"%s"', self.features)
509 cfg_fmt = '"--cfg %s"'
510 if self.cap_lints:
511 allowed = '"--cap-lints ' + self.cap_lints + '"'
512 if not self.cfgs:
513 self.write(' flags: [' + allowed + '],')
514 else:
515 self.write(' flags: [\n ' + allowed + ',')
516 self.dump_android_property_list_items(cfg_fmt, self.cfgs)
517 self.write(' ],')
518 else:
519 self.dump_android_property_list('flags', cfg_fmt, self.cfgs)
520 if self.externs:
521 self.dump_android_externs()
522 self.dump_android_property_list('static_libs', '"lib%s"', self.static_libs)
523 self.dump_android_property_list('shared_libs', '"lib%s"', self.shared_libs)
524 self.write('}')
525
526 def test_module_name(self):
527 """Return a unique name for a test module."""
528 # root_pkg+'_tests_'+(crate_name|source_file_path)
529 suffix = self.crate_name
530 if not suffix:
531 suffix = re.sub('/', '_', re.sub('.rs$', '', self.main_src))
532 return self.root_pkg + '_tests_' + suffix
533
534 def decide_module_type(self):
535 """Decide which Android module type to use."""
536 host = '' if self.device_supported else '_host'
537 if self.crate_type == 'bin': # rust_binary[_host]
538 self.module_type = 'rust_binary' + host
539 self.stem = self.crate_name
540 elif self.crate_type == 'lib': # rust_library[_host]_rlib
541 self.module_type = 'rust_library' + host + '_rlib'
542 self.stem = 'lib' + self.crate_name
543 elif self.crate_type == 'cdylib': # rust_library[_host]_dylib
544 # TODO(chh): complete and test cdylib module type
545 self.module_type = 'rust_library' + host + '_dylib'
546 self.stem = 'lib' + self.crate_name + '.so'
547 elif self.crate_type == 'test': # rust_test[_host]
548 self.module_type = 'rust_test' + host
549 self.stem = self.test_module_name()
550 # self.stem will be changed after merging with other tests.
551 # self.stem is NOT used for final test binary name.
552 # rust_test uses each source file base name as its output file name,
553 # unless crate_name is specified by user in Cargo.toml.
554 elif self.crate_type == 'proc-macro': # rust_proc_macro
555 self.module_type = 'rust_proc_macro'
556 self.stem = 'lib' + self.crate_name
557 else: # unknown module type, rust_prebuilt_dylib? rust_library[_host]?
558 self.module_type = ''
559 self.stem = ''
560
561 def dump_android_property_list_items(self, fmt, values):
562 for v in values:
563 # fmt has quotes, so we need escape_quotes(v)
564 self.write(' ' + (fmt % escape_quotes(v)) + ',')
565
566 def dump_android_property_list(self, name, fmt, values):
567 if values:
568 self.write(' ' + name + ': [')
569 self.dump_android_property_list_items(fmt, values)
570 self.write(' ],')
571
572 def dump_android_core_properties(self):
573 """Dump the module header, name, stem, etc."""
574 self.write(' name: "' + self.module_name + '",')
575 if self.stem != self.module_name:
576 self.write(' stem: "' + self.stem + '",')
577 if self.has_warning and not self.cap_lints:
578 self.write(' deny_warnings: false,')
579 if self.host_supported and self.device_supported:
580 self.write(' host_supported: true,')
581 self.write(' crate_name: "' + self.crate_name + '",')
582 if len(self.srcs) > 1:
583 self.srcs = sorted(set(self.srcs))
584 self.dump_android_property_list('srcs', '"%s"', self.srcs)
585 else:
586 self.write(' srcs: ["' + self.main_src + '"],')
587 if self.crate_type == 'test':
588 # self.root_pkg can have multiple test modules, with different *_tests[n]
589 # names, but their executables can all be installed under the same _tests
590 # directory. When built from Cargo.toml, all tests should have different
591 # file or crate names.
592 self.write(' relative_install_path: "' + self.root_pkg + '_tests",')
593 self.write(' test_suites: ["general-tests"],')
594 self.write(' auto_gen_config: true,')
595
596 def dump_android_externs(self):
597 """Dump the dependent rlibs and dylibs property."""
598 so_libs = list()
599 rust_libs = ''
600 deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$')
601 for lib in self.externs:
602 # normal value of lib: "libc = liblibc-*.rlib"
603 # strange case in rand crate: "getrandom_package = libgetrandom-*.rlib"
604 # we should use "libgetrandom", not "lib" + "getrandom_package"
605 groups = deps_libname.match(lib)
606 if groups is not None:
607 lib_name = groups.group(1)
608 else:
609 lib_name = re.sub(' .*$', '', lib)
610 if lib.endswith('.rlib') or lib.endswith('.rmeta'):
611 # On MacOS .rmeta is used when Linux uses .rlib or .rmeta.
612 rust_libs += ' "' + altered_name('lib' + lib_name) + '",\n'
613 elif lib.endswith('.so'):
614 so_libs.append(lib_name)
615 else:
616 rust_libs += ' // ERROR: unknown type of lib ' + lib_name + '\n'
617 if rust_libs:
618 self.write(' rlibs: [\n' + rust_libs + ' ],')
619 # Are all dependent .so files proc_macros?
620 # TODO(chh): Separate proc_macros and dylib.
621 self.dump_android_property_list('proc_macros', '"lib%s"', so_libs)
622
623
624class ARObject(object):
625 """Information of an "ar" link command."""
626
627 def __init__(self, runner, outf_name):
628 # Remembered global runner and its members.
629 self.runner = runner
630 self.pkg = ''
631 self.outf_name = outf_name # path to Android.bp
632 # "ar" arguments
633 self.line_num = 1
634 self.line = ''
635 self.flags = '' # e.g. "crs"
636 self.lib = '' # e.g. "/.../out/lib*.a"
637 self.objs = list() # e.g. "/.../out/.../*.o"
638
639 def parse(self, pkg, line_num, args_line):
640 """Collect ar obj/lib file names."""
641 self.pkg = pkg
642 self.line_num = line_num
643 self.line = args_line
644 args = args_line.split()
645 num_args = len(args)
646 if num_args < 3:
647 print('ERROR: "ar" command has too few arguments', args_line)
648 else:
649 self.flags = unquote(args[0])
650 self.lib = unquote(args[1])
651 self.objs = sorted(set(map(unquote, args[2:])))
652 return self
653
654 def write(self, s):
655 self.outf.write(s + '\n')
656
657 def dump_debug_info(self):
658 self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line)
659 self.write('// ar_object for %12s' % self.pkg)
660 self.write('// flags = %s' % self.flags)
661 self.write('// lib = %s' % short_out_name(self.pkg, self.lib))
662 for o in self.objs:
663 self.write('// obj = %s' % short_out_name(self.pkg, o))
664
665 def dump_android_lib(self):
666 """Write cc_library_static into Android.bp."""
667 self.write('\ncc_library_static {')
668 self.write(' name: "' + file_base_name(self.lib) + '",')
669 self.write(' host_supported: true,')
670 if self.flags != 'crs':
671 self.write(' // ar flags = %s' % self.flags)
672 if self.pkg not in self.runner.pkg_obj2cc:
673 self.write(' ERROR: cannot find source files.\n}')
674 return
675 self.write(' srcs: [')
676 obj2cc = self.runner.pkg_obj2cc[self.pkg]
677 # Note: wflags are ignored.
678 dflags = list()
679 fflags = list()
680 for obj in self.objs:
681 self.write(' "' + short_out_name(self.pkg, obj2cc[obj].src) + '",')
682 # TODO(chh): union of dflags and flags of all obj
683 # Now, just a temporary hack that uses the last obj's flags
684 dflags = obj2cc[obj].dflags
685 fflags = obj2cc[obj].fflags
686 self.write(' ],')
687 self.write(' cflags: [')
688 self.write(' "-O3",') # TODO(chh): is this default correct?
689 self.write(' "-Wno-error",')
690 for x in fflags:
691 self.write(' "-f' + x + '",')
692 for x in dflags:
693 self.write(' "-D' + x + '",')
694 self.write(' ],')
695 self.write('}')
696
697 def dump(self):
698 """Dump error/debug/module info to the output .bp file."""
699 self.runner.init_bp_file(self.outf_name)
700 with open(self.outf_name, 'a') as outf:
701 self.outf = outf
702 if self.runner.args.debug:
703 self.dump_debug_info()
704 self.dump_android_lib()
705
706
707class CCObject(object):
708 """Information of a "cc" compilation command."""
709
710 def __init__(self, runner, outf_name):
711 # Remembered global runner and its members.
712 self.runner = runner
713 self.pkg = ''
714 self.outf_name = outf_name # path to Android.bp
715 # "cc" arguments
716 self.line_num = 1
717 self.line = ''
718 self.src = ''
719 self.obj = ''
720 self.dflags = list() # -D flags
721 self.fflags = list() # -f flags
722 self.iflags = list() # -I flags
723 self.wflags = list() # -W flags
724 self.other_args = list()
725
726 def parse(self, pkg, line_num, args_line):
727 """Collect cc compilation flags and src/out file names."""
728 self.pkg = pkg
729 self.line_num = line_num
730 self.line = args_line
731 args = args_line.split()
732 i = 0
733 while i < len(args):
734 arg = args[i]
735 if arg == '"-c"':
736 i += 1
737 if args[i].startswith('"-o'):
738 # ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c"
739 self.obj = unquote(args[i])[2:]
740 i += 1
741 self.src = unquote(args[i])
742 else:
743 self.src = unquote(args[i])
744 elif arg == '"-o"':
745 i += 1
746 self.obj = unquote(args[i])
747 elif arg == '"-I"':
748 i += 1
749 self.iflags.append(unquote(args[i]))
750 elif arg.startswith('"-D'):
751 self.dflags.append(unquote(args[i])[2:])
752 elif arg.startswith('"-f'):
753 self.fflags.append(unquote(args[i])[2:])
754 elif arg.startswith('"-W'):
755 self.wflags.append(unquote(args[i])[2:])
756 elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or
757 arg == '"-g3"'):
758 # ignore -O -m64 -g
759 self.other_args.append(unquote(args[i]))
760 i += 1
761 self.dflags = sorted(set(self.dflags))
762 self.fflags = sorted(set(self.fflags))
763 # self.wflags is not sorted because some are order sensitive
764 # and we ignore them anyway.
765 if self.pkg not in self.runner.pkg_obj2cc:
766 self.runner.pkg_obj2cc[self.pkg] = {}
767 self.runner.pkg_obj2cc[self.pkg][self.obj] = self
768 return self
769
770 def write(self, s):
771 self.outf.write(s + '\n')
772
773 def dump_debug_flags(self, name, flags):
774 self.write('// ' + name + ':')
775 for f in flags:
776 self.write('// %s' % f)
777
778 def dump(self):
779 """Dump only error/debug info to the output .bp file."""
780 if not self.runner.args.debug:
781 return
782 self.runner.init_bp_file(self.outf_name)
783 with open(self.outf_name, 'a') as outf:
784 self.outf = outf
785 self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line)
786 self.write('// cc_object for %12s' % self.pkg)
787 self.write('// src = %s' % short_out_name(self.pkg, self.src))
788 self.write('// obj = %s' % short_out_name(self.pkg, self.obj))
789 self.dump_debug_flags('-I flags', self.iflags)
790 self.dump_debug_flags('-D flags', self.dflags)
791 self.dump_debug_flags('-f flags', self.fflags)
792 self.dump_debug_flags('-W flags', self.wflags)
793 if self.other_args:
794 self.dump_debug_flags('other args', self.other_args)
795
796
797class Runner(object):
798 """Main class to parse cargo -v output and print Android module definitions."""
799
800 def __init__(self, args):
801 self.bp_files = set() # Remember all output Android.bp files.
802 self.root_pkg = '' # name of package in ./Cargo.toml
803 # Saved flags, modes, and data.
804 self.args = args
805 self.dry_run = not args.run
806 self.skip_cargo = args.skipcargo
807 # All cc/ar objects, crates, dependencies, and warning files
808 self.cc_objects = list()
809 self.pkg_obj2cc = {}
810 # pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i]
811 self.ar_objects = list()
812 self.crates = list()
813 self.dependencies = list() # dependent and build script crates
814 self.warning_files = set()
815 # Keep a unique mapping from (module name) to crate
816 self.name_owners = {}
817 # Default action is cargo clean, followed by build or user given actions.
818 if args.cargo:
819 self.cargo = ['clean'] + args.cargo
820 else:
821 self.cargo = ['clean', 'build']
822 default_target = '--target x86_64-unknown-linux-gnu'
823 if args.device:
824 self.cargo.append('build ' + default_target)
825 if args.tests:
826 self.cargo.append('build --tests')
827 self.cargo.append('build --tests ' + default_target)
828 elif args.tests:
829 self.cargo.append('build --tests')
830
831 def init_bp_file(self, name):
832 if name not in self.bp_files:
833 self.bp_files.add(name)
834 with open(name, 'w') as outf:
835 outf.write(ANDROID_BP_HEADER)
836
837 def claim_module_name(self, prefix, owner, counter):
838 """Return prefix if not owned yet, otherwise, prefix+str(counter)."""
839 while True:
840 name = prefix
841 if counter > 0:
842 name += str(counter)
843 if name not in self.name_owners:
844 self.name_owners[name] = owner
845 return name
846 if owner == self.name_owners[name]:
847 return name
848 counter += 1
849
850 def find_root_pkg(self):
851 """Read name of [package] in ./Cargo.toml."""
852 if not os.path.exists('./Cargo.toml'):
853 return
854 with open('./Cargo.toml', 'r') as inf:
855 pkg_section = re.compile(r'^ *\[package\]')
856 name = re.compile('^ *name *= * "([^"]*)"')
857 in_pkg = False
858 for line in inf:
859 if in_pkg:
860 if name.match(line):
861 self.root_pkg = name.match(line).group(1)
862 break
863 else:
864 in_pkg = pkg_section.match(line) is not None
865
866 def run_cargo(self):
867 """Calls cargo -v and save its output to ./cargo.out."""
868 if self.skip_cargo:
869 return self
870 cargo = './Cargo.toml'
871 if not os.access(cargo, os.R_OK):
872 print('ERROR: Cannot find or read', cargo)
873 return self
874 if not self.dry_run and os.path.exists('cargo.out'):
875 os.remove('cargo.out')
876 cmd_tail = ' --target-dir ' + TARGET_TMP + ' >> cargo.out 2>&1'
877 for c in self.cargo:
878 features = ''
Chih-Hung Hsieh6c8d52f2020-03-30 18:28:52 -0700879 if c != 'clean':
880 if self.args.features is not None:
881 features = ' --no-default-features'
882 if self.args.features:
883 features += ' --features ' + self.args.features
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -0800884 cmd = 'cargo -vv ' if self.args.vv else 'cargo -v '
885 cmd += c + features + cmd_tail
886 if self.args.rustflags and c != 'clean':
887 cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd
888 if self.dry_run:
889 print('Dry-run skip:', cmd)
890 else:
891 if self.args.verbose:
892 print('Running:', cmd)
893 with open('cargo.out', 'a') as cargo_out:
894 cargo_out.write('### Running: ' + cmd + '\n')
895 os.system(cmd)
896 return self
897
898 def dump_dependencies(self):
899 """Append dependencies and their features to Android.bp."""
900 if not self.dependencies:
901 return
902 dependent_list = list()
903 for c in self.dependencies:
904 dependent_list.append(c.feature_list())
905 sorted_dependencies = sorted(set(dependent_list))
906 self.init_bp_file('Android.bp')
907 with open('Android.bp', 'a') as outf:
908 outf.write('\n// dependent_library ["feature_list"]\n')
909 for s in sorted_dependencies:
910 outf.write('// ' + s + '\n')
911
912 def dump_pkg_obj2cc(self):
913 """Dump debug info of the pkg_obj2cc map."""
914 if not self.args.debug:
915 return
916 self.init_bp_file('Android.bp')
917 with open('Android.bp', 'a') as outf:
918 sorted_pkgs = sorted(self.pkg_obj2cc.keys())
919 for pkg in sorted_pkgs:
920 if not self.pkg_obj2cc[pkg]:
921 continue
922 outf.write('\n// obj => src for %s\n' % pkg)
923 obj2cc = self.pkg_obj2cc[pkg]
924 for obj in sorted(obj2cc.keys()):
925 outf.write('// ' + short_out_name(pkg, obj) + ' => ' +
926 short_out_name(pkg, obj2cc[obj].src) + '\n')
927
928 def gen_bp(self):
929 """Parse cargo.out and generate Android.bp files."""
930 if self.dry_run:
931 print('Dry-run skip: read', CARGO_OUT, 'write Android.bp')
932 elif os.path.exists(CARGO_OUT):
933 self.find_root_pkg()
934 with open(CARGO_OUT, 'r') as cargo_out:
935 self.parse(cargo_out, 'Android.bp')
936 self.crates.sort(key=get_module_name)
937 for obj in self.cc_objects:
938 obj.dump()
939 self.dump_pkg_obj2cc()
940 for crate in self.crates:
941 crate.dump()
942 dumped_libs = set()
943 for lib in self.ar_objects:
944 if lib.pkg == self.root_pkg:
945 lib_name = file_base_name(lib.lib)
946 if lib_name not in dumped_libs:
947 dumped_libs.add(lib_name)
948 lib.dump()
949 if self.args.dependencies and self.dependencies:
950 self.dump_dependencies()
951 return self
952
953 def add_ar_object(self, obj):
954 self.ar_objects.append(obj)
955
956 def add_cc_object(self, obj):
957 self.cc_objects.append(obj)
958
959 def add_crate(self, crate):
960 """Merge crate with someone in crates, or append to it. Return crates."""
961 if crate.skip_crate():
962 if self.args.debug: # include debug info of all crates
963 self.crates.append(crate)
964 if self.args.dependencies: # include only dependent crates
965 if (is_dependent_file_path(crate.main_src) and
966 not is_build_crate_name(crate.crate_name)):
967 self.dependencies.append(crate)
968 else:
969 for c in self.crates:
970 if c.merge(crate, 'Android.bp'):
971 return
972 self.crates.append(crate)
973
974 def find_warning_owners(self):
975 """For each warning file, find its owner crate."""
976 missing_owner = False
977 for f in self.warning_files:
978 cargo_dir = '' # find lowest crate, with longest path
979 owner = None # owner crate of this warning
980 for c in self.crates:
981 if (f.startswith(c.cargo_dir + '/') and
982 len(cargo_dir) < len(c.cargo_dir)):
983 cargo_dir = c.cargo_dir
984 owner = c
985 if owner:
986 owner.has_warning = True
987 else:
988 missing_owner = True
989 if missing_owner and os.path.exists('Cargo.toml'):
990 # owner is the root cargo, with empty cargo_dir
991 for c in self.crates:
992 if not c.cargo_dir:
993 c.has_warning = True
994
995 def rustc_command(self, n, rustc_line, line, outf_name):
996 """Process a rustc command line from cargo -vv output."""
997 # cargo build -vv output can have multiple lines for a rustc command
998 # due to '\n' in strings for environment variables.
999 # strip removes leading spaces and '\n' at the end
1000 new_rustc = (rustc_line.strip() + line) if rustc_line else line
1001 # Use an heuristic to detect the completions of a multi-line command.
1002 # This might fail for some very rare case, but easy to fix manually.
1003 if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0:
1004 return new_rustc
1005 if RUSTC_VV_CMD_ARGS.match(new_rustc):
1006 args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(1)
1007 self.add_crate(Crate(self, outf_name).parse(n, args))
1008 else:
1009 self.assert_empty_vv_line(new_rustc)
1010 return ''
1011
1012 def cc_ar_command(self, n, groups, outf_name):
1013 pkg = groups.group(1)
1014 line = groups.group(3)
1015 if groups.group(2) == 'cc':
1016 self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line))
1017 else:
1018 self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line))
1019
1020 def assert_empty_vv_line(self, line):
1021 if line: # report error if line is not empty
1022 self.init_bp_file('Android.bp')
1023 with open('Android.bp', 'a') as outf:
1024 outf.write('ERROR -vv line: ', line)
1025 return ''
1026
1027 def parse(self, inf, outf_name):
1028 """Parse rustc and warning messages in inf, return a list of Crates."""
1029 n = 0 # line number
1030 prev_warning = False # true if the previous line was warning: ...
1031 rustc_line = '' # previous line(s) matching RUSTC_VV_PAT
1032 for line in inf:
1033 n += 1
1034 if line.startswith('warning: '):
1035 prev_warning = True
1036 rustc_line = self.assert_empty_vv_line(rustc_line)
1037 continue
1038 new_rustc = ''
1039 if RUSTC_PAT.match(line):
1040 args_line = RUSTC_PAT.match(line).group(1)
1041 self.add_crate(Crate(self, outf_name).parse(n, args_line))
1042 self.assert_empty_vv_line(rustc_line)
1043 elif rustc_line or RUSTC_VV_PAT.match(line):
1044 new_rustc = self.rustc_command(n, rustc_line, line, outf_name)
1045 elif CC_AR_VV_PAT.match(line):
1046 self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name)
1047 elif prev_warning and WARNING_FILE_PAT.match(line):
1048 self.assert_empty_vv_line(rustc_line)
1049 fpath = WARNING_FILE_PAT.match(line).group(1)
1050 if fpath[0] != '/': # ignore absolute path
1051 self.warning_files.add(fpath)
1052 prev_warning = False
1053 rustc_line = new_rustc
1054 self.find_warning_owners()
1055
1056
1057def parse_args():
1058 """Parse main arguments."""
1059 parser = argparse.ArgumentParser('cargo2android')
1060 parser.add_argument(
1061 '--cargo',
1062 action='append',
1063 metavar='args_string',
1064 help=('extra cargo build -v args in a string, ' +
1065 'each --cargo flag calls cargo build -v once'))
1066 parser.add_argument(
1067 '--debug',
1068 action='store_true',
1069 default=False,
1070 help='dump debug info into Android.bp')
1071 parser.add_argument(
1072 '--dependencies',
1073 action='store_true',
1074 default=False,
1075 help='dump debug info of dependent crates')
1076 parser.add_argument(
1077 '--device',
1078 action='store_true',
1079 default=False,
1080 help='run cargo also for a default device target')
1081 parser.add_argument(
Chih-Hung Hsieh6c8d52f2020-03-30 18:28:52 -07001082 '--features', type=str,
1083 help=('pass features to cargo build, ' +
1084 'empty string means no default features'))
Chih-Hung Hsiehe8887372019-11-05 10:34:17 -08001085 parser.add_argument(
1086 '--onefile',
1087 action='store_true',
1088 default=False,
1089 help=('output all into one ./Android.bp, default will generate ' +
1090 'one Android.bp per Cargo.toml in subdirectories'))
1091 parser.add_argument(
1092 '--run',
1093 action='store_true',
1094 default=False,
1095 help='run it, default is dry-run')
1096 parser.add_argument('--rustflags', type=str, help='passing flags to rustc')
1097 parser.add_argument(
1098 '--skipcargo',
1099 action='store_true',
1100 default=False,
1101 help='skip cargo command, parse cargo.out, and generate Android.bp')
1102 parser.add_argument(
1103 '--tests',
1104 action='store_true',
1105 default=False,
1106 help='run cargo build --tests after normal build')
1107 parser.add_argument(
1108 '--verbose',
1109 action='store_true',
1110 default=False,
1111 help='echo executed commands')
1112 parser.add_argument(
1113 '--vv',
1114 action='store_true',
1115 default=False,
1116 help='run cargo with -vv instead of default -v')
1117 return parser.parse_args()
1118
1119
1120def main():
1121 args = parse_args()
1122 if not args.run: # default is dry-run
1123 print(DRY_RUN_NOTE)
1124 Runner(args).run_cargo().gen_bp()
1125
1126
1127if __name__ == '__main__':
1128 main()