scripts/cargo2android.py

#!/usr/bin/env python3
#
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Call cargo -v, parse its output, and generate Android.bp.

Usage: Run this script in a crate workspace root directory.
The Cargo.toml file should work at least for the host platform.

(1) Without other flags, "cargo2android.py --run"
    calls cargo clean, calls cargo build -v, and generates Android.bp.
    The cargo build only generates crates for the host,
    without test crates.

(2) To build crates for both host and device in Android.bp, use the
    --device flag, for example:
    cargo2android.py --run --device

    Note that cargo build is only called once with the default target
    x86_64-unknown-linux-gnu.

(3) To build default and test crates, for host and device, use both
    --device and --tests flags:
    cargo2android.py --run --device --tests

    This is equivalent to using the --cargo flag to add extra builds:
    cargo2android.py --run
      --cargo "build --target x86_64-unknown-linux-gnu"
      --cargo "build --tests --target x86_64-unknown-linux-gnu"

If there are rustc warning messages, this script will add
a warning comment to the owner crate module in Android.bp.
"""

from __future__ import print_function

import argparse
import glob
import json
import os
import os.path
import platform
import re
import shutil
import subprocess
import sys

# Some Rust packages include extra unwanted crates.
# This set contains all such excluded crate names.
EXCLUDED_CRATES = set(['protobuf_bin_gen_rust_do_not_use'])

RENAME_MAP = {
    # This map includes all changes to the default rust module names
    # to resolve name conflicts, avoid confusion, or work as plugin.
    'libash': 'libash_rust',
    'libbacktrace': 'libbacktrace_rust',
    'libbase': 'libbase_rust',
    'libbase64': 'libbase64_rust',
    'libfuse': 'libfuse_rust',
    'libgcc': 'libgcc_rust',
    'liblog': 'liblog_rust',
    'libminijail': 'libminijail_rust',
    'libsync': 'libsync_rust',
    'libx86_64': 'libx86_64_rust',
    'libxml': 'libxml_rust',
    'protoc_gen_rust': 'protoc-gen-rust',
}

RENAME_STEM_MAP = {
    # This map includes all changes to the default rust module stem names,
    # which is used for output files when different from the module name.
    'protoc_gen_rust': 'protoc-gen-rust',
}

RENAME_DEFAULTS_MAP = {
    # This map includes all changes to the default prefix of rust_default
    # module names, to avoid conflict with existing Android modules.
    'libc': 'rust_libc',
}

# Header added to all generated Android.bp files.
ANDROID_BP_HEADER = (
    '// This file is generated by cargo2android.py {args}.\n' +
    '// Do not modify this file as changes will be overridden on upgrade.\n')

CARGO_OUT = 'cargo.out'  # Name of file to keep cargo build -v output.

# This should be kept in sync with tools/external_updater/crates_updater.py.
ERRORS_LINE = 'Errors in ' + CARGO_OUT + ':'

TARGET_TMP = 'target.tmp'  # Name of temporary output directory.

# Message to be displayed when this script is called without the --run flag.
DRY_RUN_NOTE = (
    'Dry-run: This script uses ./' + TARGET_TMP + ' for output directory,\n' +
    'runs cargo clean, runs cargo build -v, saves output to ./cargo.out,\n' +
    'and writes to Android.bp in the current and subdirectories.\n\n' +
    'To do do all of the above, use the --run flag.\n' +
    'See --help for other flags, and more usage notes in this script.\n')

# Cargo -v output of a call to rustc.
RUSTC_PAT = re.compile('^ +Running `rustc (.*)`$')

# Cargo -vv output of a call to rustc could be split into multiple lines.
# Assume that the first line will contain some CARGO_* env definition.
RUSTC_VV_PAT = re.compile('^ +Running `.*CARGO_.*=.*$')
# The combined -vv output rustc command line pattern.
RUSTC_VV_CMD_ARGS = re.compile('^ *Running `.*CARGO_.*=.* rustc (.*)`$')

# Cargo -vv output of a "cc" or "ar" command; all in one line.
CC_AR_VV_PAT = re.compile(r'^\[([^ ]*)[^\]]*\] running:? "(cc|ar)" (.*)$')
# Some package, such as ring-0.13.5, has pattern '... running "cc"'.

# Rustc output of file location path pattern for a warning message.
WARNING_FILE_PAT = re.compile('^ *--> ([^:]*):[0-9]+')

# cargo test --list output of the start of running a binary.
CARGO_TEST_LIST_START_PAT = re.compile('^\s*Running (.*) \(.*\)$')

# cargo test --list output of the end of running a binary.
CARGO_TEST_LIST_END_PAT = re.compile('^(\d+) tests, (\d+) benchmarks$')

CARGO2ANDROID_RUNNING_PAT = re.compile('^### Running: .*$')

# Rust package name with suffix -d1.d2.d3(+.*)?.
VERSION_SUFFIX_PAT = re.compile(r'^(.*)-[0-9]+\.[0-9]+\.[0-9]+(?:\+.*)?$')

# Crate types corresponding to a C ABI library
C_LIBRARY_CRATE_TYPES = ['staticlib', 'cdylib']
# Crate types corresponding to a Rust ABI library
RUST_LIBRARY_CRATE_TYPES = ['lib', 'rlib', 'dylib']
# Crate types corresponding to a library
LIBRARY_CRATE_TYPES = C_LIBRARY_CRATE_TYPES + RUST_LIBRARY_CRATE_TYPES

def altered_name(name):
  return RENAME_MAP[name] if (name in RENAME_MAP) else name


def altered_stem(name):
  return RENAME_STEM_MAP[name] if (name in RENAME_STEM_MAP) else name


def altered_defaults(name):
  return RENAME_DEFAULTS_MAP[name] if (name in RENAME_DEFAULTS_MAP) else name


def is_build_crate_name(name):
  # We added special prefix to build script crate names.
  return name.startswith('build_script_')


def is_dependent_file_path(path):
  # Absolute or dependent '.../' paths are not main files of this crate.
  return path.startswith('/') or path.startswith('.../')


def get_module_name(crate):  # to sort crates in a list
  return crate.module_name


def pkg2crate_name(s):
  return s.replace('-', '_').replace('.', '_')


def file_base_name(path):
  return os.path.splitext(os.path.basename(path))[0]


def test_base_name(path):
  return pkg2crate_name(file_base_name(path))


def unquote(s):  # remove quotes around str
  if s and len(s) > 1 and s[0] == '"' and s[-1] == '"':
    return s[1:-1]
  return s


def remove_version_suffix(s):  # remove -d1.d2.d3 suffix
  if VERSION_SUFFIX_PAT.match(s):
    return VERSION_SUFFIX_PAT.match(s).group(1)
  return s


def short_out_name(pkg, s):  # replace /.../pkg-*/out/* with .../out/*
  return re.sub('^/.*/' + pkg + '-[0-9a-f]*/out/', '.../out/', s)


def escape_quotes(s):  # replace '"' with '\\"'
  return s.replace('"', '\\"')


class Crate(object):
  """Information of a Rust crate to collect/emit for an Android.bp module."""

  def __init__(self, runner, outf_name):
    # Remembered global runner and its members.
    self.runner = runner
    self.debug = runner.args.debug
    self.cargo_dir = ''  # directory of my Cargo.toml
    self.outf_name = outf_name  # path to Android.bp
    self.outf = None  # open file handle of outf_name during dump*
    # Variants/results that could be merged from multiple rustc lines.
    self.host_supported = False
    self.device_supported = False
    self.has_warning = False
    # Android module properties derived from rustc parameters.
    self.module_name = ''  # unique in Android build system
    self.module_type = ''  # rust_{binary,library,test}[_host] etc.
    self.defaults = ''  # rust_defaults used by rust_test* modules
    self.default_srcs = False  # use 'srcs' defined in self.defaults
    self.root_pkg = ''  # parent package name of a sub/test packge, from -L
    self.srcs = list()  # main_src or merged multiple source files
    self.stem = ''  # real base name of output file
    # Kept parsed status
    self.errors = ''  # all errors found during parsing
    self.line_num = 1  # runner told input source line number
    self.line = ''  # original rustc command line parameters
    # Parameters collected from rustc command line.
    self.crate_name = ''  # follows --crate-name
    self.main_src = ''  # follows crate_name parameter, shortened
    self.crate_types = list()  # follows --crate-type
    self.cfgs = list()  # follows --cfg, without feature= prefix
    self.features = list()  # follows --cfg, name in 'feature="..."'
    self.codegens = list()  # follows -C, some ignored
    self.externs = list()  # follows --extern
    self.core_externs = list()  # first part of self.externs elements
    self.static_libs = list()  # e.g.  -l static=host_cpuid
    self.shared_libs = list()  # e.g.  -l dylib=wayland-client, -l z
    self.cap_lints = ''  # follows --cap-lints
    self.emit_list = ''  # e.g., --emit=dep-info,metadata,link
    self.edition = '2015'  # rustc default, e.g., --edition=2018
    self.target = ''  # follows --target
    self.cargo_env_compat = True
    self.cargo_pkg_version = ''  # value extracted from Cargo.toml version field

  def write(self, s):
    # convenient way to output one line at a time with EOL.
    self.outf.write(s + '\n')

  def same_flags(self, other):
    # host_supported, device_supported, has_warning are not compared but merged
    # target is not compared, to merge different target/host modules
    # externs is not compared; only core_externs is compared
    return (not self.errors and not other.errors and
            self.edition == other.edition and
            self.cap_lints == other.cap_lints and
            self.emit_list == other.emit_list and
            self.core_externs == other.core_externs and
            self.codegens == other.codegens and
            self.features == other.features and
            self.static_libs == other.static_libs and
            self.shared_libs == other.shared_libs and self.cfgs == other.cfgs)

  def merge_host_device(self, other):
    """Returns true if attributes are the same except host/device support."""
    return (self.crate_name == other.crate_name and
            self.crate_types == other.crate_types and
            self.main_src == other.main_src and
            # before merge, each test module has an unique module name and stem
            (self.stem == other.stem or self.crate_types == ['test']) and
            self.root_pkg == other.root_pkg and not self.skip_crate() and
            self.same_flags(other))

  def merge_test(self, other):
    """Returns true if self and other are tests of same root_pkg."""
    # Before merger, each test has its own crate_name.
    # A merged test uses its source file base name as output file name,
    # so a test is mergeable only if its base name equals to its crate name.
    return (self.crate_types == other.crate_types and
            self.crate_types == ['test'] and self.root_pkg == other.root_pkg and
            not self.skip_crate() and
            other.crate_name == test_base_name(other.main_src) and
            (len(self.srcs) > 1 or
             (self.crate_name == test_base_name(self.main_src)) and
             self.host_supported == other.host_supported and
             self.device_supported == other.device_supported) and
            self.same_flags(other))

  def merge(self, other, outf_name):
    """Try to merge crate into self."""
    # Cargo build --tests could recompile a library for tests.
    # We need to merge such duplicated calls to rustc, with
    # the algorithm in merge_host_device.
    should_merge_host_device = self.merge_host_device(other)
    should_merge_test = False
    if not should_merge_host_device:
      should_merge_test = self.merge_test(other)
    if should_merge_host_device or should_merge_test:
      self.runner.init_bp_file(outf_name)
      with open(outf_name, 'a') as outf:  # to write debug info
        self.outf = outf
        other.outf = outf
        self.do_merge(other, should_merge_test)
      return True
    return False

  def do_merge(self, other, should_merge_test):
    """Merge attributes of other to self."""
    if self.debug:
      self.write('\n// Before merge definition (1):')
      self.dump_debug_info()
      self.write('\n// Before merge definition (2):')
      other.dump_debug_info()
    # Merge properties of other to self.
    self.has_warning = self.has_warning or other.has_warning
    if not self.target:  # okay to keep only the first target triple
      self.target = other.target
    # decide_module_type sets up default self.stem,
    # which can be changed if self is a merged test module.
    self.decide_module_type()
    if should_merge_test:
      if (self.runner.should_ignore_test(self.main_src)
          and not self.runner.should_ignore_test(other.main_src)):
        self.main_src = other.main_src
      self.srcs.append(other.main_src)
      # use a short unique name as the merged module name.
      prefix = self.root_pkg + '_tests'
      self.module_name = self.runner.claim_module_name(prefix, self, 0)
      self.stem = self.module_name
      # This normalized root_pkg name although might be the same
      # as other module's crate_name, it is not actually used for
      # output file name. A merged test module always have multiple
      # source files and each source file base name is used as
      # its output file name.
      self.crate_name = pkg2crate_name(self.root_pkg)
    if self.debug:
      self.write('\n// After merge definition (1):')
      self.dump_debug_info()

  def find_cargo_dir(self):
    """Deepest directory with Cargo.toml and contains the main_src."""
    if not is_dependent_file_path(self.main_src):
      dir_name = os.path.dirname(self.main_src)
      while dir_name:
        if os.path.exists(dir_name + '/Cargo.toml'):
          self.cargo_dir = dir_name
          return
        dir_name = os.path.dirname(dir_name)

  def add_codegens_flag(self, flag):
    """Ignore options not used in Android."""
    # 'prefer-dynamic' does not work with common flag -C lto
    # 'embed-bitcode' is ignored; we might control LTO with other .bp flag
    # 'codegen-units' is set in Android global config or by default
    if not (flag.startswith('codegen-units=') or
            flag.startswith('debuginfo=') or
            flag.startswith('embed-bitcode=') or
            flag.startswith('extra-filename=') or
            flag.startswith('incremental=') or
            flag.startswith('metadata=') or
            flag == 'prefer-dynamic'):
      self.codegens.append(flag)

  def parse(self, line_num, line):
    """Find important rustc arguments to convert to Android.bp properties."""
    self.line_num = line_num
    self.line = line
    args = line.split()  # Loop through every argument of rustc.
    i = 0
    while i < len(args):
      arg = args[i]
      if arg == '--crate-name':
        i += 1
        self.crate_name = args[i]
      elif arg == '--crate-type':
        i += 1
        # cargo calls rustc with multiple --crate-type flags.
        # rustc can accept:
        #   --crate-type [bin|lib|rlib|dylib|cdylib|staticlib|proc-macro]
        self.crate_types.append(args[i])
      elif arg == '--test':
        self.crate_types.append('test')
      elif arg == '--target':
        i += 1
        self.target = args[i]
      elif arg == '--cfg':
        i += 1
        if args[i].startswith('\'feature='):
          self.features.append(unquote(args[i].replace('\'feature=', '')[:-1]))
        else:
          self.cfgs.append(args[i])
      elif arg == '--extern':
        i += 1
        extern_names = re.sub('=/[^ ]*/deps/', ' = ', args[i])
        self.externs.append(extern_names)
        self.core_externs.append(re.sub(' = .*', '', extern_names))
      elif arg == '-C':  # codegen options
        i += 1
        self.add_codegens_flag(args[i])
      elif arg.startswith('-C'):
        # cargo has been passing "-C <xyz>" flag to rustc,
        # but newer cargo could pass '-Cembed-bitcode=no' to rustc.
        self.add_codegens_flag(arg[2:])
      elif arg == '--cap-lints':
        i += 1
        self.cap_lints = args[i]
      elif arg == '-L':
        i += 1
        if args[i].startswith('dependency=') and args[i].endswith('/deps'):
          if '/' + TARGET_TMP + '/' in args[i]:
            self.root_pkg = re.sub(
                '^.*/', '', re.sub('/' + TARGET_TMP + '/.*/deps$', '', args[i]))
          else:
            self.root_pkg = re.sub('^.*/', '',
                                   re.sub('/[^/]+/[^/]+/deps$', '', args[i]))
          self.root_pkg = remove_version_suffix(self.root_pkg)
      elif arg == '-l':
        i += 1
        if args[i].startswith('static='):
          self.static_libs.append(re.sub('static=', '', args[i]))
        elif args[i].startswith('dylib='):
          self.shared_libs.append(re.sub('dylib=', '', args[i]))
        else:
          self.shared_libs.append(args[i])
      elif arg == '--out-dir' or arg == '--color':  # ignored
        i += 1
      elif arg.startswith('--error-format=') or arg.startswith('--json='):
        _ = arg  # ignored
      elif arg.startswith('--emit='):
        self.emit_list = arg.replace('--emit=', '')
      elif arg.startswith('--edition='):
        self.edition = arg.replace('--edition=', '')
      elif not arg.startswith('-'):
        # shorten imported crate main source paths like $HOME/.cargo/
        # registry/src/github.com-1ecc6299db9ec823/memchr-2.3.3/src/lib.rs
        self.main_src = re.sub(r'^/[^ ]*/registry/src/', '.../', args[i])
        self.main_src = re.sub(r'^\.\.\./github.com-[0-9a-f]*/', '.../',
                               self.main_src)
        self.find_cargo_dir()
        if self.cargo_dir:  # for a subdirectory
          if self.runner.args.no_subdir:  # all .bp content to /dev/null
            self.outf_name = '/dev/null'
          elif not self.runner.args.onefile:
            # Write to Android.bp in the subdirectory with Cargo.toml.
            self.outf_name = self.cargo_dir + '/Android.bp'
            self.main_src = self.main_src[len(self.cargo_dir) + 1:]

      else:
        self.errors += 'ERROR: unknown ' + arg + '\n'
      i += 1
    if not self.crate_name:
      self.errors += 'ERROR: missing --crate-name\n'
    if not self.main_src:
      self.errors += 'ERROR: missing main source file\n'
    else:
      self.srcs.append(self.main_src)
    if not self.crate_types:
      # Treat "--cfg test" as "--test"
      if 'test' in self.cfgs:
        self.crate_types.append('test')
      else:
        self.errors += 'ERROR: missing --crate-type or --test\n'
    elif len(self.crate_types) > 1:
      if 'test' in self.crate_types:
        self.errors += 'ERROR: cannot handle both --crate-type and --test\n'
      if 'lib' in self.crate_types and 'rlib' in self.crate_types:
        self.errors += 'ERROR: cannot generate both lib and rlib crate types\n'
    if not self.root_pkg:
      self.root_pkg = self.crate_name

    # get the package version from running cargo metadata
    if not self.runner.args.no_pkg_vers and not self.skip_crate():
        self.get_pkg_version()

    self.device_supported = self.runner.args.device
    self.host_supported = not self.runner.args.no_host
    self.cfgs = sorted(set(self.cfgs))
    self.features = sorted(set(self.features))
    self.codegens = sorted(set(self.codegens))
    self.externs = sorted(set(self.externs))
    self.core_externs = sorted(set(self.core_externs))
    self.static_libs = sorted(set(self.static_libs))
    self.shared_libs = sorted(set(self.shared_libs))
    self.crate_types = sorted(set(self.crate_types))
    self.decide_module_type()
    self.module_name = altered_name(self.stem)
    return self

  def get_pkg_version(self):
    """Attempt to retrieve the package version from the Cargo.toml

    If there is only one package, use its version. Otherwise, try to
    match the emitted `--crate_name` arg against the package name.

    This may fail in cases where multiple packages are defined (workspaces)
    and where the package name does not match the emitted crate_name
    (e.g. [lib.name] is set).
    """
    cargo_metadata = subprocess.run([self.runner.cargo_path, 'metadata', '--no-deps',
                                     '--format-version', '1'],
                                    cwd=os.path.abspath(self.cargo_dir),
                                    stdout=subprocess.PIPE)
    if cargo_metadata.returncode:
        self.errors += ('ERROR: unable to get cargo metadata for package version; ' +
                'return code ' + cargo_metadata.returncode + '\n')
    else:
        metadata_json = json.loads(cargo_metadata.stdout)
        if len(metadata_json['packages']) > 1:
            for package in metadata_json['packages']:
                # package names may contain '-', but is changed to '_' in the crate_name
                if package['name'].replace('-','_') == self.crate_name:
                    self.cargo_pkg_version = package['version']
                    break
        else:
            self.cargo_pkg_version = metadata_json['packages'][0]['version']

        if not self.cargo_pkg_version:
            self.errors += ('ERROR: Unable to retrieve package version; ' +
                'to disable, run with arg "--no-pkg-vers"\n')

  def dump_line(self):
    self.write('\n// Line ' + str(self.line_num) + ' ' + self.line)

  def feature_list(self):
    """Return a string of main_src + "feature_list"."""
    pkg = self.main_src
    if pkg.startswith('.../'):  # keep only the main package name
      pkg = re.sub('/.*', '', pkg[4:])
    elif pkg.startswith('/'):  # use relative path for a local package
      pkg = os.path.relpath(pkg)
    if not self.features:
      return pkg
    return pkg + ' "' + ','.join(self.features) + '"'

  def dump_skip_crate(self, kind):
    if self.debug:
      self.write('\n// IGNORED: ' + kind + ' ' + self.main_src)
    return self

  def skip_crate(self):
    """Return crate_name or a message if this crate should be skipped."""
    if (is_build_crate_name(self.crate_name) or
        self.crate_name in EXCLUDED_CRATES):
      return self.crate_name
    if is_dependent_file_path(self.main_src):
      return 'dependent crate'
    return ''

  def dump(self):
    """Dump all error/debug/module code to the output .bp file."""
    self.runner.init_bp_file(self.outf_name)
    with open(self.outf_name, 'a') as outf:
      self.outf = outf
      if self.errors:
        self.dump_line()
        self.write(self.errors)
      elif self.skip_crate():
        self.dump_skip_crate(self.skip_crate())
      else:
        if self.debug:
          self.dump_debug_info()
        self.dump_android_module()

  def dump_debug_info(self):
    """Dump parsed data, when cargo2android is called with --debug."""

    def dump(name, value):
      self.write('//%12s = %s' % (name, value))

    def opt_dump(name, value):
      if value:
        dump(name, value)

    def dump_list(fmt, values):
      for v in values:
        self.write(fmt % v)

    self.dump_line()
    dump('module_name', self.module_name)
    dump('crate_name', self.crate_name)
    dump('crate_types', self.crate_types)
    dump('main_src', self.main_src)
    dump('has_warning', self.has_warning)
    dump('for_host', self.host_supported)
    dump('for_device', self.device_supported)
    dump('module_type', self.module_type)
    opt_dump('target', self.target)
    opt_dump('edition', self.edition)
    opt_dump('emit_list', self.emit_list)
    opt_dump('cap_lints', self.cap_lints)
    dump_list('//         cfg = %s', self.cfgs)
    dump_list('//         cfg = \'feature "%s"\'', self.features)
    # TODO(chh): escape quotes in self.features, but not in other dump_list
    dump_list('//     codegen = %s', self.codegens)
    dump_list('//     externs = %s', self.externs)
    dump_list('//   -l static = %s', self.static_libs)
    dump_list('//  -l (dylib) = %s', self.shared_libs)

  def dump_android_module(self):
    """Dump one or more Android module definition, depending on crate_types."""
    if len(self.crate_types) == 1:
      self.dump_single_type_android_module()
      return
    if 'test' in self.crate_types:
      self.write('\nERROR: multiple crate types cannot include test type')
      return
    # Dump one Android module per crate_type.
    for crate_type in self.crate_types:
      self.decide_one_module_type(crate_type)
      self.dump_one_android_module(crate_type)

  def build_default_name(self):
    """Return a short and readable name for the rust_defaults module."""
    # Choices: (1) root_pkg + '_test'? + '_defaults',
    # (2) root_pkg + '_test'? + '_defaults_' + crate_name
    # (3) root_pkg + '_test'? + '_defaults_' + main_src_basename_path
    # (4) root_pkg + '_test'? + '_defaults_' + a_positive_sequence_number
    test = "_test" if self.crate_types == ['test'] else ""
    name1 = altered_defaults(self.root_pkg) + test + '_defaults'
    if self.runner.try_claim_module_name(name1, self):
      return name1
    name2 = name1 + '_' + self.crate_name
    if self.runner.try_claim_module_name(name2, self):
      return name2
    name3 = name1 + '_' + self.main_src_basename_path()
    if self.runner.try_claim_module_name(name3, self):
      return name3
    return self.runner.claim_module_name(name1, self, 0)

  def dump_srcs_list(self):
    """Dump the srcs list, for defaults or regular modules."""
    if len(self.srcs) > 1:
      srcs = sorted(set(self.srcs))  # make a copy and dedup
    else:
      srcs = [self.main_src]
    copy_out = self.runner.copy_out_module_name()
    if copy_out:
      srcs.append(':' + copy_out)
    self.dump_android_property_list('srcs', '"%s"', srcs)

  def dump_defaults_module(self):
    """Dump a rust_defaults module to be shared by other modules."""
    name = self.build_default_name()
    self.defaults = name
    self.write('\nrust_defaults {')
    self.write('    name: "' + name + '",')
    if self.runner.args.global_defaults:
      self.write('    defaults: ["' + self.runner.args.global_defaults + '"],')
    self.write('    crate_name: "' + self.crate_name + '",')
    if len(self.srcs) == 1:  # only one source file; share it in defaults
      self.default_srcs = True
      if self.has_warning and not self.cap_lints:
        self.write('    // has rustc warnings')
      self.dump_srcs_list()
    if self.cargo_env_compat:
      self.write('    cargo_env_compat: true,')
      if not self.runner.args.no_pkg_vers:
        self.write('    cargo_pkg_version: "' + self.cargo_pkg_version + '",')
    if 'test' in self.crate_types:
      self.write('    test_suites: ["general-tests"],')
      self.write('    auto_gen_config: true,')
    self.dump_edition_flags_libs()
    if 'test' in self.crate_types and len(self.srcs) == 1:
      self.dump_test_data()
    self.write('}')

  def dump_single_type_android_module(self):
    """Dump one simple Android module, which has only one crate_type."""
    crate_type = self.crate_types[0]
    if crate_type != 'test':
      # do not change self.stem or self.module_name
      self.dump_one_android_module(crate_type)
      return
    # Dump one test module per source file, and separate host and device tests.
    # crate_type == 'test'
    self.srcs = [src for src in self.srcs if not self.runner.should_ignore_test(src)]
    if ((self.host_supported and self.device_supported and len(self.srcs) > 0) or
        len(self.srcs) > 1):
      self.srcs = sorted(set(self.srcs))
      self.dump_defaults_module()
    saved_srcs = self.srcs
    for src in saved_srcs:
      self.srcs = [src]
      saved_device_supported = self.device_supported
      saved_host_supported = self.host_supported
      saved_main_src = self.main_src
      self.main_src = src
      if saved_host_supported:
        self.device_supported = False
        self.host_supported = True
        self.module_name = self.test_module_name()
        self.decide_one_module_type(crate_type)
        self.dump_one_android_module(crate_type)
      if saved_device_supported:
        self.device_supported = True
        self.host_supported = False
        self.module_name = self.test_module_name()
        self.decide_one_module_type(crate_type)
        self.dump_one_android_module(crate_type)
      self.host_supported = saved_host_supported
      self.device_supported = saved_device_supported
      self.main_src = saved_main_src
    self.srcs = saved_srcs

  def dump_one_android_module(self, crate_type):
    """Dump one Android module definition."""
    if not self.module_type:
      self.write('\nERROR: unknown crate_type ' + crate_type)
      return
    self.write('\n' + self.module_type + ' {')
    self.dump_android_core_properties()
    if not self.defaults:
      self.dump_edition_flags_libs()
    if self.runner.args.host_first_multilib and self.host_supported and crate_type != 'test':
      self.write('    compile_multilib: "first",')
    if self.runner.args.exported_c_header_dir and crate_type in C_LIBRARY_CRATE_TYPES:
      self.write('    include_dirs: [')
      for header_dir in self.runner.args.exported_c_header_dir:
        self.write('        "%s",' % header_dir)
      self.write('    ],')
    if self.runner.args.apex_available and crate_type in LIBRARY_CRATE_TYPES:
      self.write('    apex_available: [')
      for apex in self.runner.args.apex_available:
        self.write('        "%s",' % apex)
      self.write('    ],')
    if crate_type != 'test':
      if self.runner.args.native_bridge_supported:
        self.write('    native_bridge_supported: true,')
      if self.runner.args.product_available:
        self.write('    product_available: true,')
      if self.runner.args.recovery_available:
        self.write('    recovery_available: true,')
      if self.runner.args.vendor_available:
        self.write('    vendor_available: true,')
      if self.runner.args.vendor_ramdisk_available:
        self.write('    vendor_ramdisk_available: true,')
      if self.runner.args.ramdisk_available:
        self.write('    ramdisk_available: true,')
    if self.runner.args.min_sdk_version and crate_type in LIBRARY_CRATE_TYPES:
      self.write('    min_sdk_version: "%s",' % self.runner.args.min_sdk_version)
    if crate_type == 'test' and not self.default_srcs:
      self.dump_test_data()
    if self.runner.args.add_module_block:
      with open(self.runner.args.add_module_block, 'r') as f:
        self.write('    %s,' % f.read().replace('\n', '\n    '))
    self.write('}')

  def dump_android_flags(self):
    """Dump Android module flags property."""
    if not self.codegens and not self.cap_lints:
      return
    self.write('    flags: [')
    if self.cap_lints:
      self.write('        "--cap-lints ' + self.cap_lints + '",')
    codegens_fmt = '"-C %s"'
    self.dump_android_property_list_items(codegens_fmt, self.codegens)
    self.write('    ],')

  def dump_edition_flags_libs(self):
    if self.edition:
      self.write('    edition: "' + self.edition + '",')
    self.dump_android_property_list('features', '"%s"', self.features)
    cfgs = [cfg for cfg in self.cfgs if not cfg in self.runner.args.cfg_blocklist]
    self.dump_android_property_list('cfgs', '"%s"', cfgs)
    self.dump_android_flags()
    if self.externs:
      self.dump_android_externs()
    all_static_libs = [lib for lib in self.static_libs if not lib in self.runner.args.lib_blocklist]
    static_libs = [lib for lib in all_static_libs if not lib in self.runner.args.whole_static_libs]
    self.dump_android_property_list('static_libs', '"lib%s"', static_libs)
    whole_static_libs = [lib for lib in all_static_libs if lib in self.runner.args.whole_static_libs]
    self.dump_android_property_list('whole_static_libs', '"lib%s"', whole_static_libs)
    shared_libs = [lib for lib in self.shared_libs if not lib in self.runner.args.lib_blocklist]
    self.dump_android_property_list('shared_libs', '"lib%s"', shared_libs)

  def dump_test_data(self):
    data = [data for (name, data) in map(lambda kv: kv.split('=', 1), self.runner.args.test_data)
            if self.srcs == [name]]
    if data:
      self.dump_android_property_list('data', '"%s"', data)

  def main_src_basename_path(self):
    return re.sub('/', '_', re.sub('.rs$', '', self.main_src))

  def test_module_name(self):
    """Return a unique name for a test module."""
    # root_pkg+(_host|_device) + '_test_'+source_file_name
    suffix = self.main_src_basename_path()
    host_device = '_host'
    if self.device_supported:
      host_device = '_device'
    return self.root_pkg + host_device + '_test_' + suffix

  def decide_module_type(self):
    # Use the first crate type for the default/first module.
    crate_type = self.crate_types[0] if self.crate_types else ''
    self.decide_one_module_type(crate_type)

  def decide_one_module_type(self, crate_type):
    """Decide which Android module type to use."""
    host = '' if self.device_supported else '_host'
    rlib = '_rlib' if self.runner.args.force_rlib else ''
    if crate_type == 'bin':  # rust_binary[_host]
      self.module_type = 'rust_binary' + host
      # In rare cases like protobuf-codegen, the output binary name must
      # be renamed to use as a plugin for protoc.
      self.stem = altered_stem(self.crate_name)
      self.module_name = altered_name(self.crate_name)
    elif crate_type == 'lib':  # rust_library[_host]
      # TODO(chh): should this be rust_library[_host]?
      # Assuming that Cargo.toml do not use both 'lib' and 'rlib',
      # because we map them both to rlib.
      self.module_type = 'rust_library' + rlib + host
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem)
    elif crate_type == 'rlib':  # rust_library[_host]
      self.module_type = 'rust_library' + rlib + host
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem)
    elif crate_type == 'dylib':  # rust_library[_host]_dylib
      self.module_type = 'rust_library' + host + '_dylib'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem) + '_dylib'
    elif crate_type == 'cdylib':  # rust_library[_host]_shared
      self.module_type = 'rust_ffi' + host + '_shared'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem) + '_shared'
    elif crate_type == 'staticlib':  # rust_library[_host]_static
      self.module_type = 'rust_ffi' + host + '_static'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem) + '_static'
    elif crate_type == 'test':  # rust_test[_host]
      self.module_type = 'rust_test' + host
      # Before do_merge, stem name is based on the --crate-name parameter.
      # and test module name is based on stem.
      self.stem = self.test_module_name()
      # self.stem will be changed after merging with other tests.
      # self.stem is NOT used for final test binary name.
      # rust_test uses each source file base name as part of output file name.
      # In do_merge, this function is called again, with a module_name.
      # We make sure that the module name is unique in each package.
      if self.module_name:
        # Cargo uses "-C extra-filename=..." and "-C metadata=..." to add
        # different suffixes and distinguish multiple tests of the same
        # crate name. We ignore -C and use claim_module_name to get
        # unique sequential suffix.
        self.module_name = self.runner.claim_module_name(
            self.module_name, self, 0)
        # Now the module name is unique, stem should also match and unique.
        self.stem = self.module_name
    elif crate_type == 'proc-macro':  # rust_proc_macro
      self.module_type = 'rust_proc_macro'
      self.stem = 'lib' + self.crate_name
      self.module_name = altered_name(self.stem)
    else:  # unknown module type, rust_prebuilt_dylib? rust_library[_host]?
      self.module_type = ''
      self.stem = ''

  def dump_android_property_list_items(self, fmt, values):
    for v in values:
      # fmt has quotes, so we need escape_quotes(v)
      self.write('        ' + (fmt % escape_quotes(v)) + ',')

  def dump_android_property_list(self, name, fmt, values):
    if not values:
      return
    if len(values) > 1:
      self.write('    ' + name + ': [')
      self.dump_android_property_list_items(fmt, values)
      self.write('    ],')
    else:
      self.write('    ' + name + ': [' +
                 (fmt % escape_quotes(values[0])) + '],')

  def dump_android_core_properties(self):
    """Dump the module header, name, stem, etc."""
    self.write('    name: "' + self.module_name + '",')
    # see properties shared by dump_defaults_module
    if self.defaults:
      self.write('    defaults: ["' + self.defaults + '"],')
    elif self.runner.args.global_defaults:
      self.write('    defaults: ["' + self.runner.args.global_defaults + '"],')
    if self.stem != self.module_name:
      self.write('    stem: "' + self.stem + '",')
    if self.has_warning and not self.cap_lints and not self.default_srcs:
      self.write('    // has rustc warnings')
    if self.host_supported and self.device_supported and self.module_type != 'rust_proc_macro':
      self.write('    host_supported: true,')
    if not self.defaults:
      self.write('    crate_name: "' + self.crate_name + '",')
    if not self.defaults and self.cargo_env_compat:
      self.write('    cargo_env_compat: true,')
      if not self.runner.args.no_pkg_vers:
        self.write('    cargo_pkg_version: "' + self.cargo_pkg_version + '",')
    if not self.default_srcs:
      self.dump_srcs_list()
    if 'test' in self.crate_types and not self.defaults:
      # self.root_pkg can have multiple test modules, with different *_tests[n]
      # names, but their executables can all be installed under the same _tests
      # directory. When built from Cargo.toml, all tests should have different
      # file or crate names. So we used (root_pkg + '_tests') name as the
      # relative_install_path.
      # However, some package like 'slab' can have non-mergeable tests that
      # must be separated by different module names. So, here we no longer
      # emit relative_install_path.
      # self.write('    relative_install_path: "' + self.root_pkg + '_tests",')
      self.write('    test_suites: ["general-tests"],')
      self.write('    auto_gen_config: true,')
    if 'test' in self.crate_types and self.host_supported:
      self.write('    test_options: {')
      self.write('        unit_test: true,')
      self.write('    },')

  def dump_android_externs(self):
    """Dump the dependent rlibs and dylibs property."""
    so_libs = list()
    rust_libs = ''
    deps_libname = re.compile('^.* = lib(.*)-[0-9a-f]*.(rlib|so|rmeta)$')
    for lib in self.externs:
      # normal value of lib: "libc = liblibc-*.rlib"
      # strange case in rand crate:  "getrandom_package = libgetrandom-*.rlib"
      # we should use "libgetrandom", not "lib" + "getrandom_package"
      groups = deps_libname.match(lib)
      if groups is not None:
        lib_name = groups.group(1)
      else:
        lib_name = re.sub(' .*$', '', lib)
      if lib_name in self.runner.args.dependency_blocklist:
        continue
      if lib.endswith('.rlib') or lib.endswith('.rmeta'):
        # On MacOS .rmeta is used when Linux uses .rlib or .rmeta.
        rust_libs += '        "' + altered_name('lib' + lib_name) + '",\n'
      elif lib.endswith('.so'):
        so_libs.append(lib_name)
      elif lib != 'proc_macro':  # --extern proc_macro is special and ignored
        rust_libs += '        // ERROR: unknown type of lib ' + lib + '\n'
    if rust_libs:
      self.write('    rustlibs: [\n' + rust_libs + '    ],')
    # Are all dependent .so files proc_macros?
    # TODO(chh): Separate proc_macros and dylib.
    self.dump_android_property_list('proc_macros', '"lib%s"', so_libs)


class ARObject(object):
  """Information of an "ar" link command."""

  def __init__(self, runner, outf_name):
    # Remembered global runner and its members.
    self.runner = runner
    self.pkg = ''
    self.outf_name = outf_name  # path to Android.bp
    # "ar" arguments
    self.line_num = 1
    self.line = ''
    self.flags = ''  # e.g. "crs"
    self.lib = ''  # e.g. "/.../out/lib*.a"
    self.objs = list()  # e.g. "/.../out/.../*.o"

  def parse(self, pkg, line_num, args_line):
    """Collect ar obj/lib file names."""
    self.pkg = pkg
    self.line_num = line_num
    self.line = args_line
    args = args_line.split()
    num_args = len(args)
    if num_args < 3:
      print('ERROR: "ar" command has too few arguments', args_line)
    else:
      self.flags = unquote(args[0])
      self.lib = unquote(args[1])
      self.objs = sorted(set(map(unquote, args[2:])))
    return self

  def write(self, s):
    self.outf.write(s + '\n')

  def dump_debug_info(self):
    self.write('\n// Line ' + str(self.line_num) + ' "ar" ' + self.line)
    self.write('// ar_object for %12s' % self.pkg)
    self.write('//   flags = %s' % self.flags)
    self.write('//     lib = %s' % short_out_name(self.pkg, self.lib))
    for o in self.objs:
      self.write('//     obj = %s' % short_out_name(self.pkg, o))

  def dump_android_lib(self):
    """Write cc_library_static into Android.bp."""
    self.write('\ncc_library_static {')
    self.write('    name: "' + file_base_name(self.lib) + '",')
    self.write('    host_supported: true,')
    if self.flags != 'crs':
      self.write('    // ar flags = %s' % self.flags)
    if self.pkg not in self.runner.pkg_obj2cc:
      self.write('    ERROR: cannot find source files.\n}')
      return
    self.write('    srcs: [')
    obj2cc = self.runner.pkg_obj2cc[self.pkg]
    # Note: wflags are ignored.
    dflags = list()
    fflags = list()
    for obj in self.objs:
      self.write('        "' + short_out_name(self.pkg, obj2cc[obj].src) + '",')
      # TODO(chh): union of dflags and flags of all obj
      # Now, just a temporary hack that uses the last obj's flags
      dflags = obj2cc[obj].dflags
      fflags = obj2cc[obj].fflags
    self.write('    ],')
    self.write('    cflags: [')
    self.write('        "-O3",')  # TODO(chh): is this default correct?
    self.write('        "-Wno-error",')
    for x in fflags:
      self.write('        "-f' + x + '",')
    for x in dflags:
      self.write('        "-D' + x + '",')
    self.write('    ],')
    self.write('}')

  def dump(self):
    """Dump error/debug/module info to the output .bp file."""
    self.runner.init_bp_file(self.outf_name)
    with open(self.outf_name, 'a') as outf:
      self.outf = outf
      if self.runner.args.debug:
        self.dump_debug_info()
      self.dump_android_lib()


class CCObject(object):
  """Information of a "cc" compilation command."""

  def __init__(self, runner, outf_name):
    # Remembered global runner and its members.
    self.runner = runner
    self.pkg = ''
    self.outf_name = outf_name  # path to Android.bp
    # "cc" arguments
    self.line_num = 1
    self.line = ''
    self.src = ''
    self.obj = ''
    self.dflags = list()  # -D flags
    self.fflags = list()  # -f flags
    self.iflags = list()  # -I flags
    self.wflags = list()  # -W flags
    self.other_args = list()

  def parse(self, pkg, line_num, args_line):
    """Collect cc compilation flags and src/out file names."""
    self.pkg = pkg
    self.line_num = line_num
    self.line = args_line
    args = args_line.split()
    i = 0
    while i < len(args):
      arg = args[i]
      if arg == '"-c"':
        i += 1
        if args[i].startswith('"-o'):
          # ring-0.13.5 dumps: ... "-c" "-o/.../*.o" ".../*.c"
          self.obj = unquote(args[i])[2:]
          i += 1
          self.src = unquote(args[i])
        else:
          self.src = unquote(args[i])
      elif arg == '"-o"':
        i += 1
        self.obj = unquote(args[i])
      elif arg == '"-I"':
        i += 1
        self.iflags.append(unquote(args[i]))
      elif arg.startswith('"-D'):
        self.dflags.append(unquote(args[i])[2:])
      elif arg.startswith('"-f'):
        self.fflags.append(unquote(args[i])[2:])
      elif arg.startswith('"-W'):
        self.wflags.append(unquote(args[i])[2:])
      elif not (arg.startswith('"-O') or arg == '"-m64"' or arg == '"-g"' or
                arg == '"-g3"'):
        # ignore -O -m64 -g
        self.other_args.append(unquote(args[i]))
      i += 1
    self.dflags = sorted(set(self.dflags))
    self.fflags = sorted(set(self.fflags))
    # self.wflags is not sorted because some are order sensitive
    # and we ignore them anyway.
    if self.pkg not in self.runner.pkg_obj2cc:
      self.runner.pkg_obj2cc[self.pkg] = {}
    self.runner.pkg_obj2cc[self.pkg][self.obj] = self
    return self

  def write(self, s):
    self.outf.write(s + '\n')

  def dump_debug_flags(self, name, flags):
    self.write('//  ' + name + ':')
    for f in flags:
      self.write('//    %s' % f)

  def dump(self):
    """Dump only error/debug info to the output .bp file."""
    if not self.runner.args.debug:
      return
    self.runner.init_bp_file(self.outf_name)
    with open(self.outf_name, 'a') as outf:
      self.outf = outf
      self.write('\n// Line ' + str(self.line_num) + ' "cc" ' + self.line)
      self.write('// cc_object for %12s' % self.pkg)
      self.write('//    src = %s' % short_out_name(self.pkg, self.src))
      self.write('//    obj = %s' % short_out_name(self.pkg, self.obj))
      self.dump_debug_flags('-I flags', self.iflags)
      self.dump_debug_flags('-D flags', self.dflags)
      self.dump_debug_flags('-f flags', self.fflags)
      self.dump_debug_flags('-W flags', self.wflags)
      if self.other_args:
        self.dump_debug_flags('other args', self.other_args)


class Runner(object):
  """Main class to parse cargo -v output and print Android module definitions."""

  def __init__(self, args):
    self.bp_files = set()  # Remember all output Android.bp files.
    self.root_pkg = ''  # name of package in ./Cargo.toml
    # Saved flags, modes, and data.
    self.args = args
    self.dry_run = not args.run
    self.skip_cargo = args.skipcargo
    self.cargo_path = './cargo'  # path to cargo, will be set later
    self.checked_out_files = False  # to check only once
    self.build_out_files = []  # output files generated by build.rs
    # All cc/ar objects, crates, dependencies, and warning files
    self.cc_objects = list()
    self.pkg_obj2cc = {}
    # pkg_obj2cc[cc_object[i].pkg][cc_objects[i].obj] = cc_objects[i]
    self.ar_objects = list()
    self.crates = list()
    self.warning_files = set()
    # Keep a unique mapping from (module name) to crate
    self.name_owners = {}
    # Save and dump all errors from cargo to Android.bp.
    self.errors = ''
    self.test_errors = ''
    self.setup_cargo_path()
    # Default action is cargo clean, followed by build or user given actions.
    if args.cargo:
      self.cargo = ['clean'] + args.cargo
    else:
      default_target = '--target x86_64-unknown-linux-gnu'
      # Use the same target for both host and default device builds.
      # Same target is used as default in host x86_64 Android compilation.
      # Note: b/169872957, prebuilt cargo failed to build vsock
      # on x86_64-unknown-linux-musl systems.
      self.cargo = ['clean', 'build ' + default_target]
      if args.tests:
        self.cargo.append('build --tests ' + default_target)
    self.empty_tests = set()
    self.empty_unittests = False

  def setup_cargo_path(self):
    """Find cargo in the --cargo_bin or prebuilt rust bin directory."""
    if self.args.cargo_bin:
      self.cargo_path = os.path.join(self.args.cargo_bin, 'cargo')
      if not os.path.isfile(self.cargo_path):
        sys.exit('ERROR: cannot find cargo in ' + self.args.cargo_bin)
      print('WARNING: using cargo in ' + self.args.cargo_bin)
      return
    # We have only tested this on Linux.
    if platform.system() != 'Linux':
      sys.exit('ERROR: this script has only been tested on Linux with cargo.')
    # Assuming that this script is in development/scripts.
    my_dir = os.path.dirname(os.path.abspath(__file__))
    linux_dir = os.path.join(my_dir, '..', '..',
                             'prebuilts', 'rust', 'linux-x86')
    if not os.path.isdir(linux_dir):
      sys.exit('ERROR: cannot find directory ' + linux_dir)
    rust_version = self.find_rust_version(my_dir, linux_dir)
    cargo_bin = os.path.join(linux_dir, rust_version, 'bin')
    self.cargo_path = os.path.join(cargo_bin, 'cargo')
    if not os.path.isfile(self.cargo_path):
      sys.exit('ERROR: cannot find cargo in ' + cargo_bin
               + '; please try --cargo_bin= flag.')
    return

  def find_rust_version(self, my_dir, linux_dir):
    """Use my script directory, find prebuilt rust version."""
    # First look up build/soong/rust/config/global.go.
    path2global = os.path.join(my_dir, '..', '..',
                               'build', 'soong', 'rust', 'config', 'global.go')
    if os.path.isfile(path2global):
      # try to find: RustDefaultVersion = "1.44.0"
      version_pat = re.compile(
          r'\s*RustDefaultVersion\s*=\s*"([0-9]+\.[0-9]+\.[0-9]+)".*$')
      with open(path2global, 'r') as inf:
        for line in inf:
          result = version_pat.match(line)
          if result:
            return result.group(1)
    print('WARNING: cannot find RustDefaultVersion in ' + path2global)
    # Otherwise, find the newest (largest) version number in linux_dir.
    rust_version = (0, 0, 0)  # the prebuilt version to use
    version_pat = re.compile(r'([0-9]+)\.([0-9]+)\.([0-9]+)$')
    for dir_name in os.listdir(linux_dir):
      result = version_pat.match(dir_name)
      if not result:
        continue
      version = (result.group(1), result.group(2), result.group(3))
      if version > rust_version:
        rust_version = version
    return '.'.join(rust_version)

  def find_out_files(self):
    # list1 has build.rs output for normal crates
    list1 = glob.glob(TARGET_TMP + '/*/*/build/' + self.root_pkg + '-*/out/*')
    # list2 has build.rs output for proc-macro crates
    list2 = glob.glob(TARGET_TMP + '/*/build/' + self.root_pkg + '-*/out/*')
    return list1 + list2

  def copy_out_files(self):
    """Copy build.rs output files to ./out and set up build_out_files."""
    if self.checked_out_files:
      return
    self.checked_out_files = True
    cargo_out_files = self.find_out_files()
    out_files = set()
    if cargo_out_files:
      os.makedirs('out', exist_ok=True)
    for path in cargo_out_files:
      file_name = path.split('/')[-1]
      out_files.add(file_name)
      shutil.copy(path, 'out/' + file_name)
    self.build_out_files = sorted(out_files)

  def has_used_out_dir(self):
    """Returns true if env!("OUT_DIR") is found."""
    return 0 == os.system('grep -rl --exclude build.rs --include \\*.rs' +
                          ' \'env!("OUT_DIR")\' * > /dev/null')

  def copy_out_module_name(self):
    if self.args.copy_out and self.build_out_files:
      return 'copy_' + self.root_pkg + '_build_out'
    else:
      return ''

  def read_license(self, name):
    if not os.path.isfile(name):
      return ''
    license = ''
    with open(name, 'r') as intf:
      line = intf.readline()
      # Firstly skip ANDROID_BP_HEADER
      while line.startswith('//'):
        line = intf.readline()
      # Read all lines until we see a rust_* or genrule rule.
      while line != '' and not (line.startswith('rust_') or line.startswith('genrule {')):
        license += line
        line = intf.readline()
    return license.strip()

  def dump_copy_out_module(self, outf):
    """Output the genrule module to copy out/* to $(genDir)."""
    copy_out = self.copy_out_module_name()
    if not copy_out:
      return
    outf.write('\ngenrule {\n')
    outf.write('    name: "' + copy_out + '",\n')
    outf.write('    srcs: ["out/*"],\n')
    outf.write('    cmd: "cp $(in) $(genDir)",\n')
    if len(self.build_out_files) > 1:
      outf.write('    out: [\n')
      for f in self.build_out_files:
        outf.write('        "' + f + '",\n')
      outf.write('    ],\n')
    else:
      outf.write('    out: ["' + self.build_out_files[0] + '"],\n')
    outf.write('}\n')

  def init_bp_file(self, name):
    # name could be Android.bp or sub_dir_path/Android.bp
    if name not in self.bp_files:
      self.bp_files.add(name)
      license_section = self.read_license(name)
      with open(name, 'w') as outf:
        print_args = sys.argv[1:].copy()
        if '--cargo_bin' in print_args:
          index = print_args.index('--cargo_bin')
          del print_args[index:index+2]
        outf.write(ANDROID_BP_HEADER.format(args=' '.join(print_args)))
        outf.write('\n')
        outf.write(license_section)
        outf.write('\n')
        # at most one copy_out module per .bp file
        self.dump_copy_out_module(outf)

  def try_claim_module_name(self, name, owner):
    """Reserve and return True if it has not been reserved yet."""
    if name not in self.name_owners or owner == self.name_owners[name]:
      self.name_owners[name] = owner
      return True
    return False

  def claim_module_name(self, prefix, owner, counter):
    """Return prefix if not owned yet, otherwise, prefix+str(counter)."""
    while True:
      name = prefix
      if counter > 0:
        name += '_' + str(counter)
      if self.try_claim_module_name(name, owner):
        return name
      counter += 1

  def find_root_pkg(self):
    """Read name of [package] in ./Cargo.toml."""
    if not os.path.exists('./Cargo.toml'):
      return
    with open('./Cargo.toml', 'r') as inf:
      pkg_section = re.compile(r'^ *\[package\]')
      name = re.compile('^ *name *= * "([^"]*)"')
      in_pkg = False
      for line in inf:
        if in_pkg:
          if name.match(line):
            self.root_pkg = name.match(line).group(1)
            break
        else:
          in_pkg = pkg_section.match(line) is not None

  def run_cargo(self):
    """Calls cargo -v and save its output to ./cargo.out."""
    if self.skip_cargo:
      return self
    cargo_toml = './Cargo.toml'
    cargo_out = './cargo.out'
    # Do not use Cargo.lock, because .bp rules are designed to
    # run with "latest" crates avaialable on Android.
    cargo_lock = './Cargo.lock'
    cargo_lock_saved = './cargo.lock.saved'
    had_cargo_lock = os.path.exists(cargo_lock)
    if not os.access(cargo_toml, os.R_OK):
      print('ERROR: Cannot find or read', cargo_toml)
      return self
    if not self.dry_run:
      if os.path.exists(cargo_out):
        os.remove(cargo_out)
      if not self.args.use_cargo_lock and had_cargo_lock:  # save it
        os.rename(cargo_lock, cargo_lock_saved)
    cmd_tail_target = ' --target-dir ' + TARGET_TMP
    cmd_tail_redir = ' >> ' + cargo_out + ' 2>&1'
    # set up search PATH for cargo to find the correct rustc
    saved_path = os.environ['PATH']
    os.environ['PATH'] = os.path.dirname(self.cargo_path) + ':' + saved_path
    # Add [workspace] to Cargo.toml if it is not there.
    added_workspace = False
    if self.args.add_workspace:
      with open(cargo_toml, 'r') as in_file:
        cargo_toml_lines = in_file.readlines()
      found_workspace = '[workspace]\n' in cargo_toml_lines
      if found_workspace:
        print('### WARNING: found [workspace] in Cargo.toml')
      else:
        with open(cargo_toml, 'a') as out_file:
          out_file.write('[workspace]\n')
          added_workspace = True
          if self.args.verbose:
            print('### INFO: added [workspace] to Cargo.toml')
    for c in self.cargo:
      features = ''
      if c != 'clean':
        if self.args.features is not None:
          features = ' --no-default-features'
        if self.args.features:
          features += ' --features ' + self.args.features
      cmd_v_flag = ' -vv ' if self.args.vv else ' -v '
      cmd = self.cargo_path + cmd_v_flag
      cmd += c + features + cmd_tail_target + cmd_tail_redir
      if self.args.rustflags and c != 'clean':
        cmd = 'RUSTFLAGS="' + self.args.rustflags + '" ' + cmd
      self.run_cmd(cmd, cargo_out)
    if self.args.tests:
      cmd = self.cargo_path + ' test' + features + cmd_tail_target + ' -- --list' + cmd_tail_redir
      self.run_cmd(cmd, cargo_out)
    if added_workspace:  # restore original Cargo.toml
      with open(cargo_toml, 'w') as out_file:
        out_file.writelines(cargo_toml_lines)
      if self.args.verbose:
        print('### INFO: restored original Cargo.toml')
    os.environ['PATH'] = saved_path
    if not self.dry_run:
      if not had_cargo_lock:  # restore to no Cargo.lock state
        os.remove(cargo_lock)
      elif not self.args.use_cargo_lock:  # restore saved Cargo.lock
        os.rename(cargo_lock_saved, cargo_lock)
    return self

  def run_cmd(self, cmd, cargo_out):
    if self.dry_run:
      print('Dry-run skip:', cmd)
    else:
      if self.args.verbose:
        print('Running:', cmd)
      with open(cargo_out, 'a') as out_file:
        out_file.write('### Running: ' + cmd + '\n')
      ret = os.system(cmd)
      if ret != 0:
        print('*** There was an error while running cargo.  ' +
              'See the cargo.out file for details.')

  def dump_pkg_obj2cc(self):
    """Dump debug info of the pkg_obj2cc map."""
    if not self.args.debug:
      return
    self.init_bp_file('Android.bp')
    with open('Android.bp', 'a') as outf:
      sorted_pkgs = sorted(self.pkg_obj2cc.keys())
      for pkg in sorted_pkgs:
        if not self.pkg_obj2cc[pkg]:
          continue
        outf.write('\n// obj => src for %s\n' % pkg)
        obj2cc = self.pkg_obj2cc[pkg]
        for obj in sorted(obj2cc.keys()):
          outf.write('//  ' + short_out_name(pkg, obj) + ' => ' +
                     short_out_name(pkg, obj2cc[obj].src) + '\n')

  def apply_patch(self):
    """Apply local patch file if it is given."""
    if self.args.patch:
      if self.dry_run:
        print('Dry-run skip patch file:', self.args.patch)
      else:
        if not os.path.exists(self.args.patch):
          self.append_to_bp('ERROR cannot find patch file: ' + self.args.patch)
          return self
        if self.args.verbose:
          print('### INFO: applying local patch file:', self.args.patch)
        subprocess.run(['patch', '-s', '--no-backup-if-mismatch', './Android.bp',
                        self.args.patch], check=True)
    return self

  def gen_bp(self):
    """Parse cargo.out and generate Android.bp files."""
    if self.dry_run:
      print('Dry-run skip: read', CARGO_OUT, 'write Android.bp')
    elif os.path.exists(CARGO_OUT):
      self.find_root_pkg()
      if self.args.copy_out:
        self.copy_out_files()
      elif self.find_out_files() and self.has_used_out_dir():
        print('WARNING: ' + self.root_pkg + ' has cargo output files; ' +
              'please rerun with the --copy-out flag.')
      with open(CARGO_OUT, 'r') as cargo_out:
        self.parse(cargo_out, 'Android.bp')
        self.crates.sort(key=get_module_name)
        for obj in self.cc_objects:
          obj.dump()
        self.dump_pkg_obj2cc()
        for crate in self.crates:
          crate.dump()
        dumped_libs = set()
        for lib in self.ar_objects:
          if lib.pkg == self.root_pkg:
            lib_name = file_base_name(lib.lib)
            if lib_name not in dumped_libs:
              dumped_libs.add(lib_name)
              lib.dump()
        if self.args.add_toplevel_block:
          with open(self.args.add_toplevel_block, 'r') as f:
            self.append_to_bp('\n' + f.read() + '\n')
        if self.errors:
          self.append_to_bp('\n' + ERRORS_LINE + '\n' + self.errors)
        if self.test_errors:
          self.append_to_bp('\n// Errors when listing tests:\n' + self.test_errors)
    return self

  def add_ar_object(self, obj):
    self.ar_objects.append(obj)

  def add_cc_object(self, obj):
    self.cc_objects.append(obj)

  def add_crate(self, crate):
    """Merge crate with someone in crates, or append to it. Return crates."""
    if crate.skip_crate():
      if self.args.debug:  # include debug info of all crates
        self.crates.append(crate)
    else:
      for c in self.crates:
        if c.merge(crate, 'Android.bp'):
          return
      # If not merged, decide module type and name now.
      crate.decide_module_type()
      self.crates.append(crate)

  def find_warning_owners(self):
    """For each warning file, find its owner crate."""
    missing_owner = False
    for f in self.warning_files:
      cargo_dir = ''  # find lowest crate, with longest path
      owner = None  # owner crate of this warning
      for c in self.crates:
        if (f.startswith(c.cargo_dir + '/') and
            len(cargo_dir) < len(c.cargo_dir)):
          cargo_dir = c.cargo_dir
          owner = c
      if owner:
        owner.has_warning = True
      else:
        missing_owner = True
    if missing_owner and os.path.exists('Cargo.toml'):
      # owner is the root cargo, with empty cargo_dir
      for c in self.crates:
        if not c.cargo_dir:
          c.has_warning = True

  def rustc_command(self, n, rustc_line, line, outf_name):
    """Process a rustc command line from cargo -vv output."""
    # cargo build -vv output can have multiple lines for a rustc command
    # due to '\n' in strings for environment variables.
    # strip removes leading spaces and '\n' at the end
    new_rustc = (rustc_line.strip() + line) if rustc_line else line
    # Use an heuristic to detect the completions of a multi-line command.
    # This might fail for some very rare case, but easy to fix manually.
    if not line.endswith('`\n') or (new_rustc.count('`') % 2) != 0:
      return new_rustc
    if RUSTC_VV_CMD_ARGS.match(new_rustc):
      args = RUSTC_VV_CMD_ARGS.match(new_rustc).group(1)
      self.add_crate(Crate(self, outf_name).parse(n, args))
    else:
      self.assert_empty_vv_line(new_rustc)
    return ''

  def cc_ar_command(self, n, groups, outf_name):
    pkg = groups.group(1)
    line = groups.group(3)
    if groups.group(2) == 'cc':
      self.add_cc_object(CCObject(self, outf_name).parse(pkg, n, line))
    else:
      self.add_ar_object(ARObject(self, outf_name).parse(pkg, n, line))

  def append_to_bp(self, line):
    self.init_bp_file('Android.bp')
    with open('Android.bp', 'a') as outf:
      outf.write(line)

  def assert_empty_vv_line(self, line):
    if line:  # report error if line is not empty
      self.append_to_bp('ERROR -vv line: ' + line)
    return ''

  def add_empty_test(self, name):
    if name == 'unittests':
      self.empty_unittests = True
    else:
      self.empty_tests.add(name)

  def should_ignore_test(self, src):
    # cargo test outputs the source file for integration tests but "unittests"
    # for unit tests.  To figure out to which crate this corresponds, we check
    # if the current source file is the main source of a non-test crate, e.g.,
    # a library or a binary.
    return (src in self.args.test_blocklist or src in self.empty_tests
            or (self.empty_unittests
                and src in [c.main_src for c in self.crates if c.crate_types != ['test']]))

  def parse(self, inf, outf_name):
    """Parse rustc, test, and warning messages in inf, return a list of Crates."""
    n = 0  # line number
    # We read the file in two passes, where the first simply checks for empty tests.
    # Otherwise we would add and merge tests before seeing they're empty.
    cur_test_name = None
    for line in inf:
      if CARGO_TEST_LIST_START_PAT.match(line):
        cur_test_name = CARGO_TEST_LIST_START_PAT.match(line).group(1)
      elif cur_test_name and CARGO_TEST_LIST_END_PAT.match(line):
        match = CARGO_TEST_LIST_END_PAT.match(line)
        if int(match.group(1)) + int(match.group(2)) == 0:
          self.add_empty_test(cur_test_name)
        cur_test_name = None
    inf.seek(0)
    prev_warning = False  # true if the previous line was warning: ...
    rustc_line = ''  # previous line(s) matching RUSTC_VV_PAT
    in_tests = False
    for line in inf:
      n += 1
      if line.startswith('warning: '):
        prev_warning = True
        rustc_line = self.assert_empty_vv_line(rustc_line)
        continue
      new_rustc = ''
      if RUSTC_PAT.match(line):
        args_line = RUSTC_PAT.match(line).group(1)
        self.add_crate(Crate(self, outf_name).parse(n, args_line))
        self.assert_empty_vv_line(rustc_line)
      elif rustc_line or RUSTC_VV_PAT.match(line):
        new_rustc = self.rustc_command(n, rustc_line, line, outf_name)
      elif CC_AR_VV_PAT.match(line):
        self.cc_ar_command(n, CC_AR_VV_PAT.match(line), outf_name)
      elif prev_warning and WARNING_FILE_PAT.match(line):
        self.assert_empty_vv_line(rustc_line)
        fpath = WARNING_FILE_PAT.match(line).group(1)
        if fpath[0] != '/':  # ignore absolute path
          self.warning_files.add(fpath)
      elif line.startswith('error: ') or line.startswith('error[E'):
        if not self.args.ignore_cargo_errors:
          if in_tests:
            self.test_errors += '// ' + line
          else:
            self.errors += line
      elif CARGO2ANDROID_RUNNING_PAT.match(line):
        in_tests = "cargo test" in line and "--list" in line
      prev_warning = False
      rustc_line = new_rustc
    self.find_warning_owners()


def get_parser():
  """Parse main arguments."""
  parser = argparse.ArgumentParser('cargo2android')
  parser.add_argument(
      '--add_workspace',
      action='store_true',
      default=False,
      help=('append [workspace] to Cargo.toml before calling cargo,' +
            ' to treat current directory as root of package source;' +
            ' otherwise the relative source file path in generated' +
            ' .bp file will be from the parent directory.'))
  parser.add_argument(
      '--cargo',
      action='append',
      metavar='args_string',
      help=('extra cargo build -v args in a string, ' +
            'each --cargo flag calls cargo build -v once'))
  parser.add_argument(
      '--cargo_bin',
      type=str,
      help='use cargo in the cargo_bin directory instead of the prebuilt one')
  parser.add_argument(
      '--copy-out',
      action='store_true',
      default=False,
      help=('only for root directory, ' +
            'copy build.rs output to ./out/* and add a genrule to copy ' +
            './out/* to genrule output; for crates with code pattern: ' +
            'include!(concat!(env!("OUT_DIR"), "/<some_file>.rs"))'))
  parser.add_argument(
      '--debug',
      action='store_true',
      default=False,
      help='dump debug info into Android.bp')
  parser.add_argument(
      '--dependencies',
      action='store_true',
      default=False,
      help='Deprecated. Has no effect.')
  parser.add_argument(
      '--device',
      action='store_true',
      default=False,
      help='run cargo also for a default device target')
  parser.add_argument(
      '--features',
      type=str,
      help=('pass features to cargo build, ' +
            'empty string means no default features'))
  parser.add_argument(
      '--global_defaults',
      type=str,
      help='add a defaults name to every module')
  parser.add_argument(
      '--host-first-multilib',
      action='store_true',
      default=False,
      help=('add a compile_multilib:"first" property ' +
            'to Android.bp host modules.'))
  parser.add_argument(
      '--ignore-cargo-errors',
      action='store_true',
      default=False,
      help='do not append cargo/rustc error messages to Android.bp')
  parser.add_argument(
      '--no-host',
      action='store_true',
      default=False,
      help='do not run cargo for the host; only for the device target')
  parser.add_argument(
      '--no-subdir',
      action='store_true',
      default=False,
      help='do not output anything for sub-directories')
  parser.add_argument(
      '--onefile',
      action='store_true',
      default=False,
      help=('output all into one ./Android.bp, default will generate ' +
            'one Android.bp per Cargo.toml in subdirectories'))
  parser.add_argument(
      '--patch',
      type=str,
      help='apply the given patch file to generated ./Android.bp')
  parser.add_argument(
      '--run',
      action='store_true',
      default=False,
      help='run it, default is dry-run')
  parser.add_argument('--rustflags', type=str, help='passing flags to rustc')
  parser.add_argument(
      '--skipcargo',
      action='store_true',
      default=False,
      help='skip cargo command, parse cargo.out, and generate Android.bp')
  parser.add_argument(
      '--tests',
      action='store_true',
      default=False,
      help='run cargo build --tests after normal build')
  parser.add_argument(
      '--use-cargo-lock',
      action='store_true',
      default=False,
      help=('run cargo build with existing Cargo.lock ' +
            '(used when some latest dependent crates failed)'))
  parser.add_argument(
      '--exported_c_header_dir',
      nargs='*',
      help='Directories with headers to export for C usage'
  )
  parser.add_argument(
      '--min-sdk-version',
      type=str,
      help='Minimum SDK version')
  parser.add_argument(
      '--apex-available',
      nargs='*',
      help='Mark the main library as apex_available with the given apexes.')
  parser.add_argument(
      '--native-bridge-supported',
      action='store_true',
      default=False,
      help='Mark the main library as native_bridge_supported.')
  parser.add_argument(
      '--product-available',
      action='store_true',
      default=False,
      help='Mark the main library as product_available.')
  parser.add_argument(
      '--recovery-available',
      action='store_true',
      default=False,
      help='Mark the main library as recovery_available.')
  parser.add_argument(
      '--vendor-available',
      action='store_true',
      default=False,
      help='Mark the main library as vendor_available.')
  parser.add_argument(
      '--vendor-ramdisk-available',
      action='store_true',
      default=False,
      help='Mark the main library as vendor_ramdisk_available.')
  parser.add_argument(
      '--ramdisk-available',
      action='store_true',
      default=False,
      help='Mark the main library as ramdisk_available.')
  parser.add_argument(
      '--force-rlib',
      action='store_true',
      default=False,
      help='Make the main library an rlib.')
  parser.add_argument(
      '--whole-static-libs',
      nargs='*',
      default=[],
      help='Make the given libraries (without lib prefixes) whole_static_libs.')
  parser.add_argument(
      '--no-pkg-vers',
      action='store_true',
      default=False,
      help='Do not attempt to determine the package version automatically.')
  parser.add_argument(
      '--test-data',
      nargs='*',
      default=[],
      help=('Add the given file to the given test\'s data property. ' +
            'Usage: test-path=data-path'))
  parser.add_argument(
      '--dependency-blocklist',
      nargs='*',
      default=[],
      help='Do not emit the given dependencies (without lib prefixes).')
  parser.add_argument(
      '--lib-blocklist',
      nargs='*',
      default=[],
      help='Do not emit the given C libraries as dependencies (without lib prefixes).')
  parser.add_argument(
      '--test-blocklist',
      nargs='*',
      default=[],
      help=('Do not emit the given tests. ' +
            'Pass the path to the test file to exclude.'))
  parser.add_argument(
      '--cfg-blocklist',
      nargs='*',
      default=[],
      help='Do not emit the given cfg.')
  parser.add_argument(
      '--add-toplevel-block',
      type=str,
      help=('Add the contents of the given file to the top level of the Android.bp. ' +
            'The filename should start with cargo2android to work with the updater.'))
  parser.add_argument(
      '--add-module-block',
      type=str,
      help=('Add the contents of the given file to the main module. '+
            'The filename should start with cargo2android to work with the updater.'))
  parser.add_argument(
      '--verbose',
      action='store_true',
      default=False,
      help='echo executed commands')
  parser.add_argument(
      '--vv',
      action='store_true',
      default=False,
      help='run cargo with -vv instead of default -v')
  parser.add_argument(
      '--dump-config-and-exit',
      type=str,
      help=('Dump command-line arguments (minus this flag) to a config file and exit. ' +
            'This is intended to help migrate from command line options to config files.'))
  parser.add_argument(
      '--config',
      type=str,
      help=('Load command-line options from the given config file. ' +
            'Options in this file will override those passed on the command line.'))
  return parser


def parse_args(parser):
  """Parses command-line options."""
  args = parser.parse_args()
  # Use the values specified in a config file if one was found.
  if args.config:
    with open(args.config, 'r') as f:
      config = json.load(f)
      args_dict = vars(args)
      for arg in config:
        args_dict[arg.replace('-', '_')] = config[arg]
  return args


def dump_config(parser, args):
  """Writes the non-default command-line options to the specified file."""
  args_dict = vars(args)
  # Filter out the arguments that have their default value.
  # Also filter certain "temporary" arguments.
  non_default_args = {}
  for arg in args_dict:
    if (args_dict[arg] != parser.get_default(arg) and arg != 'dump_config_and_exit'
        and arg != 'config' and arg != 'cargo_bin'):
      non_default_args[arg.replace('_', '-')] = args_dict[arg]
  # Write to the specified file.
  with open(args.dump_config_and_exit, 'w') as f:
    json.dump(non_default_args, f, indent=2, sort_keys=True)


def main():
  parser = get_parser()
  args = parse_args(parser)
  if not args.run:  # default is dry-run
    print(DRY_RUN_NOTE)
  if args.dump_config_and_exit:
    dump_config(parser, args)
  else:
    Runner(args).run_cargo().gen_bp().apply_patch()


if __name__ == '__main__':
  main()