scripts/cargo2android.py

#!/usr/bin/env python
#
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Call cargo -v, parse its output, and generate Android.bp.

Usage: Run this script in a crate workspace root directory.
The Cargo.toml file should work at least for the host platform.

(1) Without other flags, "cargo2android.py --run"
    calls cargo clean, calls cargo build -v, and generates Android.bp.
    The cargo build only generates crates for the host,
    without test crates.

(2) To build crates for both host and device in Android.bp, use the
    --device flag, for example:
    cargo2android.py --run --device

    This is equivalent to using the --cargo flag to add extra builds:
    cargo2android.py --run
      --cargo "build"
      --cargo "build --target x86_64-unknown-linux-gnu"

    On MacOS, use x86_64-apple-darwin as target triple.
    Here the host target triple is used as a fake cross compilation target.
    If the crate's Cargo.toml and environment configuration works for an
    Android target, use that target triple as the cargo build flag.

(3) To build default and test crates, for host and device, use both
    --device and --tests flags:
    cargo2android.py --run --device --tests

    This is equivalent to using the --cargo flag to add extra builds:
    cargo2android.py --run
      --cargo "build"
      --cargo "build --tests"
      --cargo "build --target x86_64-unknown-linux-gnu"
      --cargo "build --tests --target x86_64-unknown-linux-gnu"

Since Android rust builds by default treat all warnings as errors,
if there are rustc warning messages, this script will add
deny_warnings:false to the owner crate module in Android.bp.
"""

from __future__ import print_function

import argparse
import os
import os.path
import re
import sys

RENAME_MAP = {
    # This map includes all changes to the default rust library module
    # names to resolve name conflicts or avoid confusion.
    'libbacktrace': 'libbacktrace_rust',
    'libgcc': 'libgcc_rust',
    'liblog': 'liblog_rust',
    'libsync': 'libsync_rust',
    'libx86_64': 'libx86_64_rust',
}

# Header added to all generated Android.bp files.
ANDROID_BP_HEADER = '// This file is generated by cargo2android.py {args}.\n'

CARGO_OUT = 'cargo.out'  # Name of file to keep cargo build -v output.

TARGET_TMP = 'target.tmp'  # Name of temporary output directory.

# Message to be displayed when this script is called without the --run flag.
DRY_RUN_NOTE = (
    'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' +
    'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' +
    'and writes to Android.bp in the current and subdirectories.\n\n' +
    'To do do all of the above, use the --run flag.\n' +
    'See --help for other flags, and more usage notes in this script.\n')

# Cargo -v output of a call to rustc.
RUSTC_PAT = re.compile('^ +Running `rustc (.*)`$')

# Cargo -vv output of a call to rustc could be split into multiple lines.
# Assume that the first line will contain some CARGO_* env definition.
RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$')
# The combined -vv output rustc command line pattern.
RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$')

# Cargo -vv output of a "cc" or "ar" command; all in one line.
CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$')
# Some package, such as ring-0.13.5, has pattern '... running "cc"'.

# Rustc output of file location path pattern for a warning message.
WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+')

# Rust package name with suffix -d1.d2.d3.
VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+$')


def altered_name(name):
  return RENAME_MAP[name] if (name in RENAME_MAP) else name


def is_build_crate_name(name):
  # We added special prefix to build script crate names.
  return name.startswith('build_script_')


def is_dependent_file_path(path):
  # Absolute or dependent '.../' paths are not main files of this crate.
  return path.startswith('/') or path.startswith('.../')


def get_module_name(crate):  # to sort crates in a list
  return crate.module_name


def pkg2crate_name(s):
  return s.replace('-', '_').replace('.', '_')


def file_base_name(path):
  return os.path.splitext(os.path.basename(path))[0]


def test_base_name(path):
  return pkg2crate_name(file_base_name(path))


def unquote(s):  # remove quotes around str
  if s and len(s) > 1 and s[0] == '"' and s[-1] == '"':
    return s[1:-1]
  return s


def remove_version_suffix(s):  # remove -d1.d2.d3 suffix
  if VERSION_SUFFIX_PAT.match(s):
    return VERSION_SUFFIX_PAT.match(s).group(1)
  return s


def short_out_name(pkg, s):  # replace /.../pkg-*/out/* with .../out/*
  return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s)


def escape_quotes(s):  # replace '"' with '\\"'
  return s.replace('"', '\\"')


class Crate(object):
  """Information of a Rust crate to collect/emit for an Android.bp module."""

  def __init__(self, runner, outf_name):
    # Remembered global runner and its members.
    self.runner = runner
    self.debug = runner.args.debug
    self.cargo_dir = ''  # directory of my Cargo.toml
    self.outf_name = outf_name  # path to Android.bp
    self.outf = None  # open file handle of outf_name during dump*
    # Variants/results that could be merged from multiple rustc lines.
    self.host_supported = False
    self.device_supported = False
    self.has_warning = False
    # Android module properties derived from rustc parameters.
    self.module_name = ''  # unique in Android build system
    self.module_type = ''  # rust_{binary,library,test}[_host] etc.
    self.root_pkg = ''  # parent package name of a sub/test packge, from -L
    self.srcs = list()  # main_src or merged multiple source files
    self.stem = ''  # real base name of output file
    # Kept parsed status
    self.errors = ''  # all errors found during parsing
    self.line_num = 1  # runner told input source line number
    self.line = ''  # original rustc command line parameters
    # Parameters collected from rustc command line.
    self.crate_name = ''  # follows --crate-name
    self.main_src = ''  # follows crate_name parameter, shortened
    self.crate_types = list()  # follows --crate-type
    self.cfgs = list()  # follows --cfg, without feature= prefix
    self.features = list()  # follows --cfg, name in 'feature="..."'
    self.codegens = list()  # follows -C, some ignored
    self.externs = list()  # follows --extern
    self.core_externs = list()  # first part of self.externs elements
    self.static_libs = list()  # e.g.  -l static=host_cpuid
    self.shared_libs = list()  # e.g.  -l dylib=wayland-client, -l z
    self.cap_lints = ''  # follows --cap-lints
    self.emit_list = ''  # e.g., --emit=dep-info,metadata,link
    self.edition = '2015'  # rustc default, e.g., --edition=2018
    self.target = ''  # follows --target

  def write(self, s):
    # convenient way to output one line at a time with EOL.
    self.outf.write(s + '\n')

  def same_flags(self, other):
    # host_supported, device_supported, has_warning are not compared but merged
    # target is not compared, to merge different target/host modules
    # externs is not compared; only core_externs is compared
    return (not self.errors and not other.errors and
            self.edition == other.edition and
            self.cap_lints == other.cap_lints and
            self.emit_list == other.emit_list and
            self.core_externs == other.core_externs and
            self.codegens == other.codegens and
            self.features == other.features and
            self.static_libs == other.static_libs and
            self.shared_libs == other.shared_libs and self.cfgs == other.cfgs)

  def merge_host_device(self, other):
    """Returns true if attributes are the same except host/device support."""
    return (self.crate_name == other.crate_name and
            self.crate_types == other.crate_types and
            self.main_src == other.main_src and
            # before merge, each test module has an unique module name and stem
            (self.stem == other.stem or self.crate_types == ['test']) and
            self.root_pkg == other.root_pkg and not self.skip_crate() and
            self.same_flags(other))

  def merge_test(self, other):
    """Returns true if self and other are tests of same root_pkg."""
    # Before merger, each test has its own crate_name.
    # A merged test uses its source file base name as output file name,
    # so a test is mergeable only if its base name equals to its crate name.
    return (self.crate_types == other.crate_types and
            self.crate_types == ['test'] and self.root_pkg == other.root_pkg and
            not self.skip_crate() and
            other.crate_name == test_base_name(other.main_src) and
            (len(self.srcs) > 1 or
             (self.crate_name == test_base_name(self.main_src)) and
             self.host_supported == other.host_supported and
             self.device_supported == other.device_supported) and
            self.same_flags(other))

  def merge(self, other, outf_name):
    """Try to merge crate into self."""
    should_merge_host_device = self.merge_host_device(other)
    should_merge_test = False
    if not should_merge_host_device:
      should_merge_test = self.merge_test(other)
    # A for-device test crate can be merged with its for-host version,
    # or merged with a different test for the same host or device.
    # Since we run cargo once for each device or host, test crates for the
    # first device or host will be merged first. Then test crates for a
    # different device or host should be allowed to be merged into a
    # previously merged one, maybe for a different device or host.
    if should_merge_host_device or should_merge_test:
      self.runner.init_bp_file(outf_name)
      with open(outf_name, 'a') as outf:  # to write debug info
        self.outf = outf
        other.outf = outf
        self.do_merge(other, should_merge_test)
      return True
    return False

  def do_merge(self, other, should_merge_test):
    """Merge attributes of other to self."""
    if self.debug:
      self.write('\n// Before merge definition (1):')
      self.dump_debug_info()
      self.write('\n// Before merge definition (2):')
      other.dump_debug_info()
    # Merge properties of other to self.
    self.host_supported = self.host_supported or other.host_supported
    self.device_supported = self.device_supported or other.device_supported
    self.has_warning = self.has_warning or other.has_warning
    if not self.target:  # okay to keep only the first target triple
      self.target = other.target
    # decide_module_type sets up default self.stem,
    # which can be changed if self is a merged test module.
    self.decide_module_type()
    if should_merge_test:
      self.srcs.append(other.main_src)
      # use a short unique name as the merged module name.
      prefix = self.root_pkg + '_tests'
      self.module_name = self.runner.claim_module_name(prefix, self, 0)
      self.stem = self.module_name
      # This normalized root_pkg name although might be the same
      # as other module's crate_name, it is not actually used for
      # output file name. A merged test module always have multiple
      # source files and each source file base name is used as
      # its output file name.
      self.crate_name = pkg2crate_name(self.root_pkg)
    if self.debug:
      self.write('\n// After merge definition (1):')
      self.dump_debug_info()

  def find_cargo_dir(self):
    """Deepest directory with Cargo.toml and contains the main_src."""
    if not is_dependent_file_path(self.main_src):
      dir_name = os.path.dirname(self.main_src)
      while dir_name:
        if os.path.exists(dir_name + '/Cargo.toml'):
          self.cargo_dir = dir_name
          return
        dir_name = os.path.dirname(dir_name)

  def parse(self, line_num, line):
    """Find important rustc arguments to convert to Android.bp properties."""
    self.line_num = line_num
    self.line = line
    args = line.split()  # Loop through every argument of rustc.
    i = 0
    while i < len(args):
      arg = args[i]
      if arg == '--crate-name':
        i += 1
        self.crate_name = args[i]
      elif arg == '--crate-type':
        i += 1
        # cargo calls rustc with multiple --crate-type flags.
        # rustc can accept:
        #   --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]
        self.crate_types.append(args[i])
      elif arg == '--test':
        self.crate_types.append('test')
      elif arg == '--target':
        i += 1
        self.target = args[i]
      elif arg == '--cfg':
        i += 1
        if args[i].startswith('\'feature='):
          self.features.append(unquote(args[i].replace('\'feature=', '')[:-1]))
        else:
          self.cfgs.append(args[i])
      elif arg == '--extern':
        i += 1
        extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i])
        self.externs.append(extern_names)
        self.core_externs.append(re.sub(' = .*', '', extern_names))
      elif arg == '-C':  # codegen options
        i += 1
        # ignore options not used in Android
        if not (args[i].startswith('debuginfo=') or
                args[i].startswith('extra-filename=') or
                args[i].startswith('incremental=') or
                args[i].startswith('metadata=')):
          self.codegens.append(args[i])
      elif arg == '--cap-lints':
        i += 1
        self.cap_lints = args[i]
      elif arg == '-L':
        i += 1
        if args[i].startswith('dependency=') and args[i].endswith('/deps'):
          if '/' + TARGET_TMP + '/' in args[i]:
            self.root_pkg = re.sub(
                '^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i]))
          else:
            self.root_pkg = re.sub('^.*/', '',
                                   re.sub('/[^/]+/[^/]+/deps$', '', args[i]))
          self.root_pkg = remove_version_suffix(self.root_pkg)
      elif arg == '-l':
        i += 1
        if args[i].startswith('static='):
          self.static_libs.append(re.sub('static=', '', args[i]))
        elif args[i].startswith('dylib='):
          self.shared_libs.append(re.sub('dylib=', '', args[i]))
        else:
          self.shared_libs.append(args[i])
      elif arg == '--out-dir' or arg == '--color':  # ignored
        i += 1
      elif arg.startswith('--error-format=') or arg.startswith('--json='):
        _ = arg  # ignored
      elif arg.startswith('--emit='):
        self.emit_list = arg.replace('--emit=', '')
      elif arg.startswith('--edition='):
        self.edition = arg.replace('--edition=', '')
      elif not arg.startswith('-'):
        # shorten imported crate main source paths like $HOME/.cargo/
        # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/lib.rs
        self.main_src = re.sub(r'^/[^ ]*/registry/src/', '.../', args[i])
        self.main_src = re.sub(r'^\.\.\./github.com-[0-9a-f]*/', '.../',
                               self.main_src)
        self.find_cargo_dir()
        if self.cargo_dir and not self.runner.args.onefile:
          # Write to Android.bp in the subdirectory with Cargo.toml.
          self.outf_name = self.cargo_dir + '/Android.bp'
          self.main_src = self.main_src[len(self.cargo_dir) + 1:]
      else:
        self.errors += 'ERROR: unknown ' + arg + '\n'
      i += 1
    if not self.crate_name:
      self.errors += 'ERROR: missing --crate-name\n'
    if not self.main_src:
      self.errors += 'ERROR: missing main source file\n'
    else:
      self.srcs.append(self.main_src)
    if not self.crate_types:
      # Treat "--cfg test" as "--test"
      if 'test' in self.cfgs:
        self.crate_types.append('test')
      else:
        self.errors += 'ERROR: missing --crate-type or --test\n'
    elif len(self.crate_types) > 1:
      if 'test' in self.crate_types:
        self.errors += 'ERROR: cannot handle both --crate-type and --test\n'
      if 'lib' in self.crate_types and 'rlib' in self.crate_types:
        self.errors += 'ERROR: cannot generate both lib and rlib crate types\n'
    if not self.root_pkg:
      self.root_pkg = self.crate_name
    if self.target:
      self.device_supported = True
    self.host_supported = True  # assume host supported for all builds
    self.cfgs = sorted(set(self.cfgs))
    self.features = sorted(set(self.features))
    self.codegens = sorted(set(self.codegens))
    self.externs = sorted(set(self.externs))
    self.core_externs = sorted(set(self.core_externs))
    self.static_libs = sorted(set(self.static_libs))
    self.shared_libs = sorted(set(self.shared_libs))
    self.crate_types = sorted(set(self.crate_types))
    self.decide_module_type()
    self.module_name = altered_name(self.stem)
    return self

  def dump_line(self):
    self.write('\n// Line ' + str(self.line_num) + ' ' + self.line)

  def feature_list(self):
    """Return a string of main_src + "feature_list"."""
    pkg = self.main_src
    if pkg.startswith('.../'):  # keep only the main package name
      pkg = re.sub('/.*', '', pkg[4:])
    if not self.features:
      return pkg
    return pkg + ' "' + ','.join(self.features) + '"'

  def dump_skip_crate(self, kind):
    if self.debug:
      self.write('\n// IGNORED: ' + kind + ' ' + self.main_src)
    return self

  def skip_crate(self):
    """Return crate_name or a message if this crate should be skipped."""
    if is_build_crate_name(self.crate_name):
      return self.crate_name
    if is_dependent_file_path(self.main_src):
      return 'dependent crate'
    return ''

  def dump(self):
    """Dump all error/debug/module code to the output .bp file."""
    self.runner.init_bp_file(self.outf_name)
    with open(self.outf_name, 'a') as outf:
      self.outf = outf
      if self.errors:
        self.dump_line()
        self.write(self.errors)
      elif self.skip_crate():
        self.dump_skip_crate(self.skip_crate())
      else:
        if self.debug:
          self.dump_debug_info()
        self.dump_android_module()

  def dump_debug_info(self):
    """Dump parsed data, when cargo2android is called with --debug."""

    def dump(name, value):
      self.write('//%12s = %s' % (name, value))

    def opt_dump(name, value):
      if value:
        dump(name, value)

    def dump_list(fmt, values):
      for v in values:
        self.write(fmt % v)

    self.dump_line()
    dump('module_name', self.module_name)
    dump('crate_name', self.crate_name)
    dump('crate_types', self.crate_types)
    dump('main_src', self.main_src)
    dump('has_warning', self.has_warning)
    dump('for_host', self.host_supported)
    dump('for_device', self.device_supported)
    dump('module_type', self.module_type)
    opt_dump('target', self.target)
    opt_dump('edition', self.edition)
    opt_dump('emit_list', self.emit_list)
    opt_dump('cap_lints', self.cap_lints)
    dump_list('//         cfg = %s', self.cfgs)
    dump_list('//         cfg = \'feature "%s"\'', self.features)
    # TODO(chh): escape quotes in self.features, but not in other dump_list
    dump_list('//     codegen = %s', self.codegens)
    dump_list('//     externs = %s', self.externs)
    dump_list('//   -l static = %s', self.static_libs)
    dump_list('//  -l (dylib) = %s', self.shared_libs)

  def dump_android_module(self):
    # Dump one Android module per crate_type.
    if len(self.crate_types) == 1:
      # do not change self.stem or self.module_name
      self.dump_one_android_module(self.crate_types[0])
      return
    for crate_type in self.crate_types:
      self.decide_one_module_type(crate_type)
      self.dump_one_android_module(crate_type)

  def dump_one_android_module(self, crate_type):
    """Dump one Android module definition."""
    if not self.module_type:
      self.write('\nERROR: unknown crate_type ' + crate_type)
      return
    self.write('\n' + self.module_type + ' {')
    self.dump_android_core_properties()
    if self.edition:
      self.write('    edition: "' + self.edition + '",')
    self.dump_android_property_list('features', '"%s"', self.features)
    cfg_fmt = '"--cfg %s"'
    if self.cap_lints:
      allowed = '"--cap-lints ' + self.cap_lints + '"'
      if not self.cfgs:
        self.write('    flags: [' + allowed + '],')
      else:
        self.write('    flags: [\n       ' + allowed + ',')
        self.dump_android_property_list_items(cfg_fmt, self.cfgs)
        self.write('    ],')
    else:
      self.dump_android_property_list('flags', cfg_fmt, self.cfgs)
    if self.externs:
      self.dump_android_externs()
    self.dump_android_property_list('static_libs', '"lib%s"', self.static_libs)
    self.dump_android_property_list('shared_libs', '"lib%s"', self.shared_libs)
    self.write('}')

  def test_module_name(self):
    """Return a unique name for a test module."""
    # root_pkg+'_tests_'+(crate_name|source_file_path)
    suffix = self.crate_name
    if not suffix:
      suffix = re.sub('/', '_', re.sub('.rs$', '', self.main_src))
    return self.root_pkg + '_tests_' + suffix

  def decide_module_type(self):
    # Use the first crate type for the default/first module.
    crate_type = self.crate_types[0] if self.crate_types else ''
    self.decide_one_module_type(crate_type)

  def decide_one_module_type(self, crate_type):
    """Decide which Android module type to use."""
    host = '' if self.device_supported else '_host'
    if crate_type == 'bin':  # rust_binary[_host]
      self.module_type = 'rust_binary' + host
      self.stem = self.crate_name
      self.module_name = altered_name(self.stem)
    elif crate_type == 'lib':  # rust_library[_host]_rlib
      # TODO(chh): should this be rust_library[_host]?
      # Assuming that Cargo.toml do not use both 'lib' and 'rlib',
      # because we map them both to rlib.
      self.module_type = 'rust_library' + host + '_rlib'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem)
    elif crate_type == 'rlib':  # rust_library[_host]_rlib
      self.module_type = 'rust_library' + host + '_rlib'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem)
    elif crate_type == 'dylib':  # rust_library[_host]_dylib
      self.module_type = 'rust_library' + host + '_dylib'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem) + '_dylib'
    elif crate_type == 'cdylib':  # rust_library[_host]_shared
      self.module_type = 'rust_library' + host + '_shared'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem) + '_shared'
    elif crate_type == 'staticlib':  # rust_library[_host]_static
      self.module_type = 'rust_library' + host + '_static'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem) + '_static'
    elif crate_type == 'test':  # rust_test[_host]
      self.module_type = 'rust_test' + host
      # Before do_merge, stem name is based on the --crate-name parameter.
      # and test module name is based on stem.
      self.stem = self.test_module_name()
      # self.stem will be changed after merging with other tests.
      # self.stem is NOT used for final test binary name.
      # rust_test uses each source file base name as its output file name,
      # unless crate_name is specified by user in Cargo.toml.
      # In do_merge, this function is called again, with a module_name.
      # We make sure that the module name is unique in each package.
      if self.module_name:
        # Cargo uses "-C extra-filename=..." and "-C metadata=..." to add
        # different suffixes and distinguish multiple tests of the same
        # crate name. We ignore -C and use claim_module_name to get
        # unique sequential suffix.
        self.module_name = self.runner.claim_module_name(
            self.module_name, self, 0)
        # Now the module name is unique, stem should also match and unique.
        self.stem = self.module_name
    elif crate_type == 'proc-macro':  # rust_proc_macro
      self.module_type = 'rust_proc_macro'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem)
    else:  # unknown module type, rust_prebuilt_dylib? rust_library[_host]?
      self.module_type = ''
      self.stem = ''

  def dump_android_property_list_items(self, fmt, values):
    for v in values:
      # fmt has quotes, so we need escape_quotes(v)
      self.write('        ' + (fmt % escape_quotes(v)) + ',')

  def dump_android_property_list(self, name, fmt, values):
    if values:
      self.write('    ' + name + ': [')
      self.dump_android_property_list_items(fmt, values)
      self.write('    ],')

  def dump_android_core_properties(self):
    """Dump the module header, name, stem, etc."""
    self.write('    name: "' + self.module_name + '",')
    if self.stem != self.module_name:
      self.write('    stem: "' + self.stem + '",')
    if self.has_warning and not self.cap_lints:
      self.write('    deny_warnings: false,')
    if self.host_supported and self.device_supported:
      self.write('    host_supported: true,')
    self.write('    crate_name: "' + self.crate_name + '",')
    if len(self.srcs) > 1:
      self.srcs = sorted(set(self.srcs))
      self.dump_android_property_list('srcs', '"%s"', self.srcs)
    else:
      self.write('    srcs: ["' + self.main_src + '"],')
    if 'test' in self.crate_types:
      # self.root_pkg can have multiple test modules, with different *_tests[n]
      # names, but their executables can all be installed under the same _tests
      # directory. When built from Cargo.toml, all tests should have different
      # file or crate names. So we used (root_pkg + '_tests') name as the
      # relative_install_path.
      # However, some package like 'slab' can have non-mergeable tests that
      # must be separated by different module names. So, here we no longer
      # emit relative_install_path.
      # self.write('    relative_install_path: "' + self.root_pkg + '_tests",')
      self.write('    test_suites: ["general-tests"],')
      self.write('    auto_gen_config: true,')

  def dump_android_externs(self):
    """Dump the dependent rlibs and dylibs property."""
    so_libs = list()
    rust_libs = ''
    deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$')
    for lib in self.externs:
      # normal value of lib: "libc = liblibc-*.rlib"
      # strange case in rand crate:  "getrandom_package = libgetrandom-*.rlib"
      # we should use "libgetrandom", not "lib" + "getrandom_package"
      groups = deps_libname.match(lib)
      if groups is not None:
        lib_name = groups.group(1)
      else:
        lib_name = re.sub(' .*$', '', lib)
      if lib.endswith('.rlib') or lib.endswith('.rmeta'):
        # On MacOS .rmeta is used when Linux uses .rlib or .rmeta.
        rust_libs += '        "' + altered_name('lib' + lib_name) + '",\n'
      elif lib.endswith('.so'):
        so_libs.append(lib_name)
      else:
        rust_libs += '        // ERROR: unknown type of lib ' + lib_name + '\n'
    if rust_libs:
      self.write('    rlibs: [\n' + rust_libs + '    ],')
    # Are all dependent .so files proc_macros?
    # TODO(chh): Separate proc_macros and dylib.
    self.dump_android_property_list('proc_macros', '"lib%s"', so_libs)


class ARObject(object):
  """Information of an "ar" link command."""

  def __init__(self, runner, outf_name):
    # Remembered global runner and its members.
    self.runner = runner
    self.pkg = ''
    self.outf_name = outf_name  # path to Android.bp
    # "ar" arguments
    self.line_num = 1
    self.line = ''
    self.flags = ''  # e.g. "crs"
    self.lib = ''  # e.g. "/.../out/lib*.a"
    self.objs = list()  # e.g. "/.../out/.../*.o"

  def parse(self, pkg, line_num, args_line):
    """Collect ar obj/lib file names."""
    self.pkg = pkg
    self.line_num = line_num
    self.line = args_line
    args = args_line.split()
    num_args = len(args)
    if num_args < 3:
      print('ERROR: "ar" command has too few arguments', args_line)
    else:
      self.flags = unquote(args[0])
      self.lib = unquote(args[1])
      self.objs = sorted(set(map(unquote, args[2:])))
    return self

  def write(self, s):
    self.outf.write(s + '\n')

  def dump_debug_info(self):
    self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line)
    self.write('// ar_object for %12s' % self.pkg)
    self.write('//   flags = %s' % self.flags)
    self.write('//     lib = %s' % short_out_name(self.pkg, self.lib))
    for o in self.objs:
      self.write('//     obj = %s' % short_out_name(self.pkg, o))

  def dump_android_lib(self):
    """Write cc_library_static into Android.bp."""
    self.write('\ncc_library_static {')
    self.write('    name: "' + file_base_name(self.lib) + '",')
    self.write('    host_supported: true,')
    if self.flags != 'crs':
      self.write('    // ar flags = %s' % self.flags)
    if self.pkg not in self.runner.pkg_obj2cc:
      self.write('    ERROR: cannot find source files.\n}')
      return
    self.write('    srcs: [')
    obj2cc = self.runner.pkg_obj2cc[self.pkg]
    # Note: wflags are ignored.
    dflags = list()
    fflags = list()
    for obj in self.objs:
      self.write('        "' + short_out_name(self.pkg, obj2cc[obj].src) + '",')
      # TODO(chh): union of dflags and flags of all obj
      # Now, just a temporary hack that uses the last obj's flags
      dflags = obj2cc[obj].dflags
      fflags = obj2cc[obj].fflags
    self.write('    ],')
    self.write('    cflags: [')
    self.write('        "-O3",')  # TODO(chh): is this default correct?
    self.write('        "-Wno-error",')
    for x in fflags:
      self.write('        "-f' + x + '",')
    for x in dflags:
      self.write('        "-D' + x + '",')
    self.write('    ],')
    self.write('}')

  def dump(self):
    """Dump error/debug/module info to the output .bp file."""
    self.runner.init_bp_file(self.outf_name)
    with open(self.outf_name, 'a') as outf:
      self.outf = outf
      if self.runner.args.debug:
        self.dump_debug_info()
      self.dump_android_lib()


class CCObject(object):
  """Information of a "cc" compilation command."""

  def __init__(self, runner, outf_name):
    # Remembered global runner and its members.
    self.runner = runner
    self.pkg = ''
    self.outf_name = outf_name  # path to Android.bp
    # "cc" arguments
    self.line_num = 1
    self.line = ''
    self.src = ''
    self.obj = ''
    self.dflags = list()  # -D flags
    self.fflags = list()  # -f flags
    self.iflags = list()  # -I flags
    self.wflags = list()  # -W flags
    self.other_args = list()

  def parse(self, pkg, line_num, args_line):
    """Collect cc compilation flags and src/out file names."""
    self.pkg = pkg
    self.line_num = line_num
    self.line = args_line
    args = args_line.split()
    i = 0
    while i < len(args):
      arg = args[i]
      if arg == '"-c"':
        i += 1
        if args[i].startswith('"-o'):
          # ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c"
          self.obj = unquote(args[i])[2:]
          i += 1
          self.src = unquote(args[i])
        else:
          self.src = unquote(args[i])
      elif arg == '"-o"':
        i += 1
        self.obj = unquote(args[i])
      elif arg == '"-I"':
        i += 1
        self.iflags.append(unquote(args[i]))
      elif arg.startswith('"-D'):
        self.dflags.append(unquote(args[i])[2:])
      elif arg.startswith('"-f'):
        self.fflags.append(unquote(args[i])[2:])
      elif arg.startswith('"-W'):
        self.wflags.append(unquote(args[i])[2:])
      elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or
                arg == '"-g3"'):
        # ignore -O -m64 -g
        self.other_args.append(unquote(args[i]))
      i += 1
    self.dflags = sorted(set(self.dflags))
    self.fflags = sorted(set(self.fflags))
    # self.wflags is not sorted because some are order sensitive
    # and we ignore them anyway.
    if self.pkg not in self.runner.pkg_obj2cc:
      self.runner.pkg_obj2cc[self.pkg] = {}
    self.runner.pkg_obj2cc[self.pkg][self.obj] = self
    return self

  def write(self, s):
    self.outf.write(s + '\n')

  def dump_debug_flags(self, name, flags):
    self.write('//  ' + name + ':')
    for f in flags:
      self.write('//    %s' % f)

  def dump(self):
    """Dump only error/debug info to the output .bp file."""
    if not self.runner.args.debug:
      return
    self.runner.init_bp_file(self.outf_name)
    with open(self.outf_name, 'a') as outf:
      self.outf = outf
      self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line)
      self.write('// cc_object for %12s' % self.pkg)
      self.write('//    src = %s' % short_out_name(self.pkg, self.src))
      self.write('//    obj = %s' % short_out_name(self.pkg, self.obj))
      self.dump_debug_flags('-I flags', self.iflags)
      self.dump_debug_flags('-D flags', self.dflags)
      self.dump_debug_flags('-f flags', self.fflags)
      self.dump_debug_flags('-W flags', self.wflags)
      if self.other_args:
        self.dump_debug_flags('other args', self.other_args)


class Runner(object):
  """Main class to parse cargo -v output and print Android module definitions."""

  def __init__(self, args):
    self.bp_files = set()  # Remember all output Android.bp files.
    self.root_pkg = ''  # name of package in ./Cargo.toml
    # Saved flags, modes, and data.
    self.args = args
    self.dry_run = not args.run
    self.skip_cargo = args.skipcargo
    # All cc/ar objects, crates, dependencies, and warning files
    self.cc_objects = list()
    self.pkg_obj2cc = {}
    # pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i]
    self.ar_objects = list()
    self.crates = list()
    self.dependencies = list()  # dependent and build script crates
    self.warning_files = set()
    # Keep a unique mapping from (module name) to crate
    self.name_owners = {}
    # Save and dump all errors from cargo to Android.bp.
    self.errors = ''
    # Default action is cargo clean, followed by build or user given actions.
    if args.cargo:
      self.cargo = ['clean'] + args.cargo
    else:
      self.cargo = ['clean', 'build']
      default_target = '--target x86_64-unknown-linux-gnu'
      if args.device:
        self.cargo.append('build ' + default_target)
        if args.tests:
          self.cargo.append('build --tests')
          self.cargo.append('build --tests ' + default_target)
      elif args.tests:
        self.cargo.append('build --tests')

  def init_bp_file(self, name):
    if name not in self.bp_files:
      self.bp_files.add(name)
      with open(name, 'w') as outf:
        outf.write(ANDROID_BP_HEADER.format(args=' '.join(sys.argv[1:])))

  def claim_module_name(self, prefix, owner, counter):
    """Return prefix if not owned yet, otherwise, prefix+str(counter)."""
    while True:
      name = prefix
      if counter > 0:
        name += str(counter)
      if name not in self.name_owners:
        self.name_owners[name] = owner
        return name
      if owner == self.name_owners[name]:
        return name
      counter += 1

  def find_root_pkg(self):
    """Read name of [package] in ./Cargo.toml."""
    if not os.path.exists('./Cargo.toml'):
      return
    with open('./Cargo.toml', 'r') as inf:
      pkg_section = re.compile(r'^ *\[package\]')
      name = re.compile('^ *name *= * "([^"]*)"')
      in_pkg = False
      for line in inf:
        if in_pkg:
          if name.match(line):
            self.root_pkg = name.match(line).group(1)
            break
        else:
          in_pkg = pkg_section.match(line) is not None

  def run_cargo(self):
    """Calls cargo -v and save its output to ./cargo.out."""
    if self.skip_cargo:
      return self
    cargo = './Cargo.toml'
    if not os.access(cargo, os.R_OK):
      print('ERROR: Cannot find or read', cargo)
      return self
    if not self.dry_run and os.path.exists('cargo.out'):
      os.remove('cargo.out')
    cmd_tail = ' --target-dir ' + TARGET_TMP + ' >> cargo.out 2>&1'
    for c in self.cargo:
      features = ''
      if c != 'clean':
        if self.args.features is not None:
          features = ' --no-default-features'
        if self.args.features:
          features += ' --features ' + self.args.features
      cmd = 'cargo -vv ' if self.args.vv else 'cargo -v '
      cmd += c + features + cmd_tail
      if self.args.rustflags and c != 'clean':
        cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd
      if self.dry_run:
        print('Dry-run skip:', cmd)
      else:
        if self.args.verbose:
          print('Running:', cmd)
        with open('cargo.out', 'a') as cargo_out:
          cargo_out.write('### Running: ' + cmd + '\n')
        os.system(cmd)
    return self

  def dump_dependencies(self):
    """Append dependencies and their features to Android.bp."""
    if not self.dependencies:
      return
    dependent_list = list()
    for c in self.dependencies:
      dependent_list.append(c.feature_list())
    sorted_dependencies = sorted(set(dependent_list))
    self.init_bp_file('Android.bp')
    with open('Android.bp', 'a') as outf:
      outf.write('\n// dependent_library ["feature_list"]\n')
      for s in sorted_dependencies:
        outf.write('//   ' + s + '\n')

  def dump_pkg_obj2cc(self):
    """Dump debug info of the pkg_obj2cc map."""
    if not self.args.debug:
      return
    self.init_bp_file('Android.bp')
    with open('Android.bp', 'a') as outf:
      sorted_pkgs = sorted(self.pkg_obj2cc.keys())
      for pkg in sorted_pkgs:
        if not self.pkg_obj2cc[pkg]:
          continue
        outf.write('\n// obj => src for %s\n' % pkg)
        obj2cc = self.pkg_obj2cc[pkg]
        for obj in sorted(obj2cc.keys()):
          outf.write('//  ' + short_out_name(pkg, obj) + ' => ' +
                     short_out_name(pkg, obj2cc[obj].src) + '\n')

  def gen_bp(self):
    """Parse cargo.out and generate Android.bp files."""
    if self.dry_run:
      print('Dry-run skip: read', CARGO_OUT, 'write Android.bp')
    elif os.path.exists(CARGO_OUT):
      self.find_root_pkg()
      with open(CARGO_OUT, 'r') as cargo_out:
        self.parse(cargo_out, 'Android.bp')
        self.crates.sort(key=get_module_name)
        for obj in self.cc_objects:
          obj.dump()
        self.dump_pkg_obj2cc()
        for crate in self.crates:
          crate.dump()
        dumped_libs = set()
        for lib in self.ar_objects:
          if lib.pkg == self.root_pkg:
            lib_name = file_base_name(lib.lib)
            if lib_name not in dumped_libs:
              dumped_libs.add(lib_name)
              lib.dump()
        if self.args.dependencies and self.dependencies:
          self.dump_dependencies()
        if self.errors:
          self.append_to_bp('\nErrors in ' + CARGO_OUT + ':\n' + self.errors)
    return self

  def add_ar_object(self, obj):
    self.ar_objects.append(obj)

  def add_cc_object(self, obj):
    self.cc_objects.append(obj)

  def add_crate(self, crate):
    """Merge crate with someone in crates, or append to it. Return crates."""
    if crate.skip_crate():
      if self.args.debug:  # include debug info of all crates
        self.crates.append(crate)
      if self.args.dependencies:  # include only dependent crates
        if (is_dependent_file_path(crate.main_src) and
            not is_build_crate_name(crate.crate_name)):
          self.dependencies.append(crate)
    else:
      for c in self.crates:
        if c.merge(crate, 'Android.bp'):
          return
      # If not merged, decide module type and name now.
      crate.decide_module_type()
      self.crates.append(crate)

  def find_warning_owners(self):
    """For each warning file, find its owner crate."""
    missing_owner = False
    for f in self.warning_files:
      cargo_dir = ''  # find lowest crate, with longest path
      owner = None  # owner crate of this warning
      for c in self.crates:
        if (f.startswith(c.cargo_dir + '/') and
            len(cargo_dir) < len(c.cargo_dir)):
          cargo_dir = c.cargo_dir
          owner = c
      if owner:
        owner.has_warning = True
      else:
        missing_owner = True
    if missing_owner and os.path.exists('Cargo.toml'):
      # owner is the root cargo, with empty cargo_dir
      for c in self.crates:
        if not c.cargo_dir:
          c.has_warning = True

  def rustc_command(self, n, rustc_line, line, outf_name):
    """Process a rustc command line from cargo -vv output."""
    # cargo build -vv output can have multiple lines for a rustc command
    # due to '\n' in strings for environment variables.
    # strip removes leading spaces and '\n' at the end
    new_rustc = (rustc_line.strip() + line) if rustc_line else line
    # Use an heuristic to detect the completions of a multi-line command.
    # This might fail for some very rare case, but easy to fix manually.
    if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0:
      return new_rustc
    if RUSTC_VV_CMD_ARGS.match(new_rustc):
      args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(1)
      self.add_crate(Crate(self, outf_name).parse(n, args))
    else:
      self.assert_empty_vv_line(new_rustc)
    return ''

  def cc_ar_command(self, n, groups, outf_name):
    pkg = groups.group(1)
    line = groups.group(3)
    if groups.group(2) == 'cc':
      self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line))
    else:
      self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line))

  def append_to_bp(self, line):
    self.init_bp_file('Android.bp')
    with open('Android.bp', 'a') as outf:
      outf.write(line)

  def assert_empty_vv_line(self, line):
    if line:  # report error if line is not empty
      self.append_to_bp('ERROR -vv line: ' + line)
    return ''

  def parse(self, inf, outf_name):
    """Parse rustc and warning messages in inf, return a list of Crates."""
    n = 0  # line number
    prev_warning = False  # true if the previous line was warning: ...
    rustc_line = ''  # previous line(s) matching RUSTC_VV_PAT
    for line in inf:
      n += 1
      if line.startswith('warning: '):
        prev_warning = True
        rustc_line = self.assert_empty_vv_line(rustc_line)
        continue
      new_rustc = ''
      if RUSTC_PAT.match(line):
        args_line = RUSTC_PAT.match(line).group(1)
        self.add_crate(Crate(self, outf_name).parse(n, args_line))
        self.assert_empty_vv_line(rustc_line)
      elif rustc_line or RUSTC_VV_PAT.match(line):
        new_rustc = self.rustc_command(n, rustc_line, line, outf_name)
      elif CC_AR_VV_PAT.match(line):
        self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name)
      elif prev_warning and WARNING_FILE_PAT.match(line):
        self.assert_empty_vv_line(rustc_line)
        fpath = WARNING_FILE_PAT.match(line).group(1)
        if fpath[0] != '/':  # ignore absolute path
          self.warning_files.add(fpath)
      elif line.startswith('error: ') or line.startswith('error[E'):
        self.errors += line
      prev_warning = False
      rustc_line = new_rustc
    self.find_warning_owners()


def parse_args():
  """Parse main arguments."""
  parser = argparse.ArgumentParser('cargo2android')
  parser.add_argument(
      '--cargo',
      action='append',
      metavar='args_string',
      help=('extra cargo build -v args in a string, ' +
            'each --cargo flag calls cargo build -v once'))
  parser.add_argument(
      '--debug',
      action='store_true',
      default=False,
      help='dump debug info into Android.bp')
  parser.add_argument(
      '--dependencies',
      action='store_true',
      default=False,
      help='dump debug info of dependent crates')
  parser.add_argument(
      '--device',
      action='store_true',
      default=False,
      help='run cargo also for a default device target')
  parser.add_argument(
      '--features',
      type=str,
      help=('pass features to cargo build, ' +
            'empty string means no default features'))
  parser.add_argument(
      '--onefile',
      action='store_true',
      default=False,
      help=('output all into one ./Android.bp, default will generate ' +
            'one Android.bp per Cargo.toml in subdirectories'))
  parser.add_argument(
      '--run',
      action='store_true',
      default=False,
      help='run it, default is dry-run')
  parser.add_argument('--rustflags', type=str, help='passing flags to rustc')
  parser.add_argument(
      '--skipcargo',
      action='store_true',
      default=False,
      help='skip cargo command, parse cargo.out, and generate Android.bp')
  parser.add_argument(
      '--tests',
      action='store_true',
      default=False,
      help='run cargo build --tests after normal build')
  parser.add_argument(
      '--verbose',
      action='store_true',
      default=False,
      help='echo executed commands')
  parser.add_argument(
      '--vv',
      action='store_true',
      default=False,
      help='run cargo with -vv instead of default -v')
  return parser.parse_args()


def main():
  args = parse_args()
  if not args.run:  # default is dry-run
    print(DRY_RUN_NOTE)
  Runner(args).run_cargo().gen_bp()


if __name__ == '__main__':
  main()