tools/gen_android_bp

#!/usr/bin/env python
# Copyright (C) 2017 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# This tool translates a collection of BUILD.gn files into a mostly equivalent
# Android.bp file for the Android Soong build system. The input to the tool is a
# JSON description of the GN build definition generated with the following
# command:
#
#   gn desc out --format=json --all-toolchains "//*" > desc.json
#
# The tool is then given a list of GN labels for which to generate Android.bp
# build rules. The dependencies for the GN labels are squashed to the generated
# Android.bp target, except for actions which get their own genrule. Some
# libraries are also mapped to their Android equivalents -- see |builtin_deps|.

import argparse
import json
import os
import re
import sys

import gn_utils

# Default targets to translate to the blueprint file.
default_targets = [
    '//:libperfetto',
    '//:libperfetto_client_experimental',
    '//:libperfetto_android_internal',
    '//:perfetto_integrationtests',
    '//:perfetto_trace_protos',
    '//:perfetto_unittests',
    '//:perfetto',
    '//:trace_processor_shell',
    '//:traced',
    '//:traced_probes',
    '//:trace_to_text',
    '//:heapprofd_client',
    '//:heapprofd',
    '//:trigger_perfetto',
]

# Targets which are testonly but should still be a cc_binary.
non_test_binaries = [
    '//:trace_processor_shell',
]

# Defines a custom init_rc argument to be applied to the corresponding output
# blueprint target.
target_initrc = {
    '//:traced': 'perfetto.rc',
    '//:heapprofd': 'heapprofd.rc',
}

target_host_supported = [
    '//:perfetto_trace_protos',
]

target_host_only = [
    '//:trace_to_text',
]

# Arguments for the GN output directory.
gn_args = 'target_os="android" target_cpu="arm" is_debug=false perfetto_build_with_android=true'

# All module names are prefixed with this string to avoid collisions.
module_prefix = 'perfetto_'

# Shared libraries which are directly translated to Android system equivalents.
library_whitelist = [
    "android.hardware.atrace@1.0",
    'android.hardware.health@2.0',
    "android.hardware.power.stats@1.0",
    'android',
    'base',
    'binder',
    'hidlbase',
    'hidltransport',
    'hwbinder',
    'incident',
    'log',
    'services',
    'utils',
]

# Name of the module which settings such as compiler flags for all other
# modules.
defaults_module = module_prefix + 'defaults'

# Location of the project in the Android source tree.
tree_path = 'external/perfetto'

# Compiler flags which are passed through to the blueprint.
cflag_whitelist = r'^-DPERFETTO.*$'

# Compiler defines which are passed through to the blueprint.
define_whitelist = r'^(GOOGLE_PROTO.*)|(PERFETTO_BUILD_WITH_ANDROID)|(ZLIB_.*)|(USE_MMAP)|(HAVE_HIDDEN)$'

# Shared libraries which are not in PDK.
library_not_in_pdk = {
    'libandroid',
    'libservices',
}

# Additional arguments to apply to Android.bp rules.
additional_args = {
    'heapprofd_client': [
        ('include_dirs', ['bionic/libc']),
        ('static_libs', ['libasync_safe']),
    ],
    'traced_probes': [
      ('required', ['libperfetto_android_internal', 'trigger_perfetto']),
    ],
    'libperfetto_android_internal': [
      ('static_libs', ['libhealthhalutils']),
    ],
}


def enable_gmock(module):
    module.static_libs.append('libgmock')


def enable_gtest_prod(module):
    module.static_libs.append('libgtest_prod')


def enable_gtest(module):
    assert module.type == 'cc_test'


def enable_protobuf_full(module):
    module.shared_libs.append('libprotobuf-cpp-full')


def enable_protobuf_lite(module):
    module.shared_libs.append('libprotobuf-cpp-lite')


def enable_protoc_lib(module):
    module.shared_libs.append('libprotoc')

def enable_libunwindstack(module):
    module.shared_libs.append('libunwindstack')
    module.shared_libs.append('libprocinfo')
    module.shared_libs.append('libbase')

def enable_libunwind(module):
    # libunwind is disabled on Darwin so we cannot depend on it.
    pass

def enable_sqlite(module):
    if module.type == 'cc_binary_host':
        module.static_libs.append('libsqlite')
    else:
        # Copy what the sqlite3 command line tool does.
        module.android.shared_libs.append('libsqlite')
        module.android.shared_libs.append('libandroidicu')
        module.android.shared_libs.append('liblog')
        module.android.shared_libs.append('libutils')
        module.host.static_libs.append('libsqlite')

def enable_zlib(module):
    module.shared_libs.append('libz')

# Android equivalents for third-party libraries that the upstream project
# depends on.
builtin_deps = {
    '//buildtools:gmock': enable_gmock,
    '//buildtools:gtest': enable_gtest,
    '//buildtools:gtest_main': enable_gtest,
    '//buildtools:libunwind': enable_libunwind,
    '//buildtools:protobuf_full': enable_protobuf_full,
    '//buildtools:protobuf_lite': enable_protobuf_lite,
    '//buildtools:protoc_lib': enable_protoc_lib,
    '//buildtools:libunwindstack': enable_libunwindstack,
    '//buildtools:sqlite': enable_sqlite,
    '//buildtools:zlib': enable_zlib,
}

# ----------------------------------------------------------------------------
# End of configuration.
# ----------------------------------------------------------------------------


class Error(Exception):
    pass


class ThrowingArgumentParser(argparse.ArgumentParser):
    def __init__(self, context):
        super(ThrowingArgumentParser, self).__init__()
        self.context = context

    def error(self, message):
        raise Error('%s: %s' % (self.context, message))


def write_blueprint_key_value(output, name, value, sort=True):
    """Writes an Blueprint key-value pair to the output"""

    if not value:
        return
    if isinstance(value, set):
        value = sorted(value)
    if isinstance(value, list):
        output.append('  %s: [' % name)
        for item in sorted(value) if sort else value:
            output.append('    "%s",' % item)
        output.append('  ],')
        return
    if isinstance(value, bool):
        output.append('  %s: true,' % name)
        return
    if isinstance(value, Target):
        value.to_string(output)
        return
    output.append('  %s: "%s",' % (name, value))

class Target(object):
    """A target-scoped part of a module"""

    def __init__(self, name):
        self.name = name
        self.shared_libs = []
        self.static_libs = []
        self.cflags = set()

    def to_string(self, output):
        nested_out = []
        self._output_field(nested_out, 'shared_libs')
        self._output_field(nested_out, 'static_libs')
        self._output_field(nested_out, 'cflags')

        if nested_out:
            output.append('  %s: {' % self.name)
            for line in nested_out:
              output.append('  %s' % line)
            output.append('  },')

    def _output_field(self, output, name, sort=True):
        value = getattr(self, name)
        return write_blueprint_key_value(output, name, value, sort)

class Module(object):
    """A single module (e.g., cc_binary, cc_test) in a blueprint."""

    def __init__(self, mod_type, name):
        self.type = mod_type
        self.name = name
        self.srcs = []
        self.comment = None
        self.shared_libs = []
        self.static_libs = []
        self.tools = []
        self.cmd = None
        self.host_supported = False
        self.init_rc = []
        self.out = []
        self.export_include_dirs = []
        self.generated_headers = []
        self.export_generated_headers = []
        self.defaults = []
        self.cflags = set()
        self.local_include_dirs = []
        self.include_dirs = []
        self.required = []
        self.user_debug_flag = False
        self.tool_files = None
        self.android = Target('android')
        self.host = Target('host')
        self.lto = None

    def to_string(self, output):
        if self.comment:
            output.append('// %s' % self.comment)
        output.append('%s {' % self.type)
        self._output_field(output, 'name')
        self._output_field(output, 'srcs')
        self._output_field(output, 'shared_libs')
        self._output_field(output, 'static_libs')
        self._output_field(output, 'tools')
        self._output_field(output, 'cmd', sort=False)
        self._output_field(output, 'host_supported')
        self._output_field(output, 'init_rc')
        self._output_field(output, 'out')
        self._output_field(output, 'export_include_dirs')
        self._output_field(output, 'generated_headers')
        self._output_field(output, 'export_generated_headers')
        self._output_field(output, 'defaults')
        self._output_field(output, 'cflags')
        self._output_field(output, 'local_include_dirs')
        self._output_field(output, 'include_dirs')
        self._output_field(output, 'required')
        self._output_field(output, 'tool_files')

        target_out = []
        self._output_field(target_out, 'android')
        self._output_field(target_out, 'host')
        if target_out:
            output.append('  target: {')
            for line in target_out:
              output.append('  %s' % line)
            output.append('  },')

        disable_pdk = any(name in library_not_in_pdk for name in self.shared_libs)
        if self.user_debug_flag or disable_pdk:
            output.append('  product_variables: {')
            if disable_pdk:
                output.append('    pdk: {')
                output.append('      enabled: false,')
                output.append('    },')
            if self.user_debug_flag:
                output.append('    debuggable: {')
                output.append('      cflags: ["-DPERFETTO_BUILD_WITH_ANDROID_USERDEBUG"],')
                output.append('    },')
            output.append('  },')
        if self.lto is not None:
          output.append('  target: {')
          output.append('    android: {')
          output.append('      lto: {')
          output.append('        thin: %s,' % 'true' if self.lto else 'false')
          output.append('      },')
          output.append('    },')
          output.append('  },')
        output.append('}')
        output.append('')

    def _output_field(self, output, name, sort=True):
        value = getattr(self, name)
        return write_blueprint_key_value(output, name, value, sort)


class Blueprint(object):
    """In-memory representation of an Android.bp file."""

    def __init__(self):
        self.modules = {}

    def add_module(self, module):
        """Adds a new module to the blueprint, replacing any existing module
        with the same name.

        Args:
            module: Module instance.
        """
        self.modules[module.name] = module

    def to_string(self, output):
        for m in sorted(self.modules.itervalues(), key=lambda m: m.name):
            m.to_string(output)


def label_to_module_name(label):
    """Turn a GN label (e.g., //:perfetto_tests) into a module name."""
    module = re.sub(r'^//:?', '', label)
    module = re.sub(r'[^a-zA-Z0-9_]', '_', module)
    if not module.startswith(module_prefix) and label not in default_targets:
        return module_prefix + module
    return module


def is_supported_source_file(name):
    """Returns True if |name| can appear in a 'srcs' list."""
    return os.path.splitext(name)[1] in ['.c', '.cc', '.proto']


def is_generated_by_action(desc, label):
    """Checks if a label is generated by an action.

    Returns True if a GN output label |label| is an output for any action,
    i.e., the file is generated dynamically.
    """
    for target in desc.itervalues():
        if target['type'] == 'action' and label in target['outputs']:
            return True
    return False


def apply_module_dependency(blueprint, desc, module, dep_name):
    """Recursively collect dependencies for a given module.

    Walk the transitive dependencies for a GN target and apply them to a given
    module. This effectively flattens the dependency tree so that |module|
    directly contains all the sources, libraries, etc. in the corresponding GN
    dependency tree.

    Args:
        blueprint: Blueprint instance which is being generated.
        desc: JSON GN description.
        module: Module to which dependencies should be added.
        dep_name: GN target of the dependency.
    """
    # If the dependency refers to a library which we can replace with an Android
    # equivalent, stop recursing and patch the dependency in.
    if gn_utils.label_without_toolchain(dep_name) in builtin_deps:
        builtin_deps[gn_utils.label_without_toolchain(dep_name)](module)
        return

    # Similarly some shared libraries are directly mapped to Android
    # equivalents.
    target = desc[dep_name]
    for lib in target.get('libs', []):
        # Generally library names sould be mangled as 'libXXX', unless they are
        # HAL libraries (e.g., android.hardware.health@2.0).
        android_lib = lib if '@' in lib else 'lib' + lib
        if lib in library_whitelist and not android_lib in module.shared_libs:
            module.shared_libs.append(android_lib)

    type = target['type']
    if type == 'action':
        if "gen_merged_sql_metrics" in dep_name:
          dep_mod = create_merged_sql_metrics_target(blueprint, desc, dep_name)
          module.generated_headers.append(dep_mod.name)
        else:
          create_modules_from_target(blueprint, desc, dep_name)
          # Depend both on the generated sources and headers -- see
          # make_genrules_for_action.
          module.srcs.append(':' + label_to_module_name(dep_name))
          module.generated_headers.append(
              label_to_module_name(dep_name) + '_headers')
    elif type == 'static_library' and label_to_module_name(
            dep_name) != module.name:
        create_modules_from_target(blueprint, desc, dep_name)
        module.static_libs.append(label_to_module_name(dep_name))
    elif type == 'shared_library' and label_to_module_name(
            dep_name) != module.name:
        module.shared_libs.append(label_to_module_name(dep_name))
    elif type in ['group', 'source_set', 'executable', 'static_library'
                  ] and 'sources' in target:
        # Ignore source files that are generated by actions since they will be
        # implicitly added by the genrule dependencies.
        module.srcs.extend(
            gn_utils.label_to_path(src) for src in target['sources']
            if is_supported_source_file(src)
            and not is_generated_by_action(desc, src))
    module.cflags |= _get_cflags(target)


def make_genrules_for_action(blueprint, desc, target_name):
    """Generate genrules for a GN action.

    GN actions are used to dynamically generate files during the build. The
    Soong equivalent is a genrule. This function turns a specific kind of
    genrule which turns .proto files into source and header files into a pair
    equivalent genrules.

    Args:
        blueprint: Blueprint instance which is being generated.
        desc: JSON GN description.
        target_name: GN target for genrule generation.

    Returns:
        A (source_genrule, header_genrule) module tuple.
    """
    target = desc[target_name]

    # We only support genrules which call protoc (with or without a plugin) to
    # turn .proto files into header and source files.
    args = target['args']
    if '/protoc' not in args[0]:
        raise Error('Unsupported action in target %s: %s' % (target_name,
                                                             target['args']))
    parser = ThrowingArgumentParser('Action in target %s (%s)' %
                                    (target_name, ' '.join(target['args'])))
    parser.add_argument('--proto_path')
    parser.add_argument('--cpp_out')
    parser.add_argument('--plugin')
    parser.add_argument('--plugin_out')
    parser.add_argument('--descriptor_set_out')
    parser.add_argument('--include_imports', action='store_true')
    parser.add_argument('protos', nargs=argparse.REMAINDER)
    args = parser.parse_args(args[2:])

    # Depending on whether we are using the default protoc C++ generator or the
    # protozero plugin, the output dir is passed as:
    # --cpp_out=gen/xxx or
    # --plugin_out=:gen/xxx or
    # --plugin_out=wrapper_namespace=pbzero:gen/xxx
    gen_dir = args.cpp_out if args.cpp_out else args.plugin_out.split(':')[1]
    assert gen_dir.startswith('gen/')
    gen_dir = gen_dir[4:]
    cpp_out_dir = ('$(genDir)/%s/%s' % (tree_path, gen_dir)).rstrip('/')

    # TODO(skyostil): Is there a way to avoid hardcoding the tree path here?
    # TODO(skyostil): Find a way to avoid creating the directory.
    cmd = [
        'mkdir -p %s &&' % cpp_out_dir,
        '$(location aprotoc)',
        '--cpp_out=%s' % cpp_out_dir
    ]

    # We create two genrules for each action: one for the protobuf headers and
    # another for the sources. This is because the module that depends on the
    # generated files needs to declare two different types of dependencies --
    # source files in 'srcs' and headers in 'generated_headers' -- and it's not
    # valid to generate .h files from a source dependency and vice versa.
    source_module = Module('genrule', label_to_module_name(target_name))
    source_module.srcs.extend(
        gn_utils.label_to_path(src) for src in target['sources'])
    source_module.tools = ['aprotoc']

    header_module = Module('genrule',
                           label_to_module_name(target_name) + '_headers')
    header_module.srcs = source_module.srcs[:]
    header_module.tools = source_module.tools[:]
    header_module.export_include_dirs = [gen_dir or '.']

    # In GN builds the proto path is always relative to the output directory
    # (out/tmp.xxx).
    assert args.proto_path.startswith('../../')
    cmd += [ '--proto_path=%s/%s' % (tree_path, args.proto_path[6:])]

    namespaces = ['pb']
    if args.plugin:
        _, plugin = os.path.split(args.plugin)
        # TODO(skyostil): Can we detect this some other way?
        if plugin == 'ipc_plugin':
            namespaces.append('ipc')
        elif plugin == 'protozero_plugin':
            namespaces = ['pbzero']
        for dep in target['deps']:
            if desc[dep]['type'] != 'executable':
                continue
            _, executable = os.path.split(desc[dep]['outputs'][0])
            if executable == plugin:
                cmd += [
                    '--plugin=protoc-gen-plugin=$(location %s)' %
                    label_to_module_name(dep)
                ]
                source_module.tools.append(label_to_module_name(dep))
                # Also make sure the module for the tool is generated.
                create_modules_from_target(blueprint, desc, dep)
                break
        else:
            raise Error('Unrecognized protoc plugin in target %s: %s' %
                        (target_name, args[i]))
    if args.plugin_out:
        plugin_args = args.plugin_out.split(':')[0]
        cmd += ['--plugin_out=%s:%s' % (plugin_args, cpp_out_dir)]

    cmd += ['$(in)']
    source_module.cmd = ' '.join(cmd)
    header_module.cmd = source_module.cmd
    header_module.tools = source_module.tools[:]

    for ns in namespaces:
        source_module.out += [
            '%s/%s' % (tree_path, src.replace('.proto', '.%s.cc' % ns))
            for src in source_module.srcs
        ]
        header_module.out += [
            '%s/%s' % (tree_path, src.replace('.proto', '.%s.h' % ns))
            for src in header_module.srcs
        ]
    return source_module, header_module


def create_merged_sql_metrics_target(blueprint, desc, gn_target_name):
    target_desc = desc[gn_target_name]
    module = Module(
        'genrule',
        'gen_merged_sql_metrics',
    )
    module.tool_files = [
        'tools/gen_merged_sql_metrics.py',
    ]
    module.cmd = ' '.join([
        '$(location tools/gen_merged_sql_metrics.py)',
        '--cpp_out=$(out)',
        '$(in)',
    ])
    module.out = set(
        src[src.index('gen/') + len('gen/'):]
        for src in target_desc.get('outputs', [])
    )
    module.srcs.extend(
      gn_utils.label_to_path(src)
      for src in target_desc.get('inputs', [])
    )
    blueprint.add_module(module)
    return module


def _get_cflags(target):
    cflags = set(flag for flag in target.get('cflags', [])
        if re.match(cflag_whitelist, flag))
    cflags |= set("-D%s" % define for define in target.get('defines', [])
                  if re.match(define_whitelist, define))
    return cflags


def create_modules_from_target(blueprint, desc, target_name):
    """Generate module(s) for a given GN target.

    Given a GN target name, generate one or more corresponding modules into a
    blueprint.

    Args:
        blueprint: Blueprint instance which is being generated.
        desc: JSON GN description.
        target_name: GN target for module generation.
    """
    target = desc[target_name]
    if target['type'] == 'executable':
        if 'host' in target['toolchain'] or target_name in target_host_only:
            module_type = 'cc_binary_host'
        elif target.get('testonly') and target_name not in non_test_binaries:
            module_type = 'cc_test'
        else:
            module_type = 'cc_binary'
        modules = [Module(module_type, label_to_module_name(target_name))]
    elif target['type'] == 'action':
        modules = make_genrules_for_action(blueprint, desc, target_name)
    elif target['type'] == 'static_library':
        module = Module('cc_library_static', label_to_module_name(target_name))
        module.export_include_dirs = ['include']
        modules = [module]
    elif target['type'] == 'shared_library':
        modules = [
            Module('cc_library_shared', label_to_module_name(target_name))
        ]
    else:
        raise Error('Unknown target type: %s' % target['type'])

    for module in modules:
        module.comment = 'GN target: %s' % target_name
        if target_name in target_initrc:
          module.init_rc = [target_initrc[target_name]]
        if target_name in target_host_supported:
          module.host_supported = True

        # Don't try to inject library/source dependencies into genrules because
        # they are not compiled in the traditional sense.
        if module.type != 'genrule':
            module.defaults = [defaults_module]
            apply_module_dependency(blueprint, desc, module, target_name)
            for dep in resolve_dependencies(desc, target_name):
                apply_module_dependency(blueprint, desc, module, dep)

        # If the module is a static library, export all the generated headers.
        if module.type == 'cc_library_static':
            module.export_generated_headers = module.generated_headers

        # Merge in additional hardcoded arguments.
        for key, add_val in additional_args.get(module.name, []):
          curr = getattr(module, key)
          if add_val and isinstance(add_val, list) and isinstance(curr, list):
            curr.extend(add_val)
          else:
            raise Error('Unimplemented type of additional_args')

        blueprint.add_module(module)


def resolve_dependencies(desc, target_name):
    """Return the transitive set of dependent-on targets for a GN target.

    Args:
        blueprint: Blueprint instance which is being generated.
        desc: JSON GN description.

    Returns:
        A set of transitive dependencies in the form of GN targets.
    """

    if gn_utils.label_without_toolchain(target_name) in builtin_deps:
        return set()
    target = desc[target_name]
    resolved_deps = set()
    for dep in target.get('deps', []):
        resolved_deps.add(dep)
        # Ignore the transitive dependencies of actions because they are
        # explicitly converted to genrules.
        if desc[dep]['type'] == 'action':
            continue
        # Dependencies on shared libraries shouldn't propagate any transitive
        # dependencies but only depend on the shared library target
        if desc[dep]['type'] == 'shared_library':
            continue
        resolved_deps.update(resolve_dependencies(desc, dep))
    return resolved_deps


def create_blueprint_for_targets(desc, targets):
    """Generate a blueprint for a list of GN targets."""
    blueprint = Blueprint()

    # Default settings used by all modules.
    defaults = Module('cc_defaults', defaults_module)
    defaults.local_include_dirs = ['include']
    defaults.cflags = [
        '-Wno-error=return-type',
        '-Wno-sign-compare',
        '-Wno-sign-promo',
        '-Wno-unused-parameter',
        '-fvisibility=hidden',
        '-Oz',
    ]
    defaults.user_debug_flag = True
    defaults.lto = True

    blueprint.add_module(defaults)
    for target in targets:
        create_modules_from_target(blueprint, desc, target)
    return blueprint


def main():
    parser = argparse.ArgumentParser(
        description='Generate Android.bp from a GN description.')
    parser.add_argument(
        '--desc',
        help=
        'GN description (e.g., gn desc out --format=json --all-toolchains "//*"'
    )
    parser.add_argument(
        '--extras',
        help='Extra targets to include at the end of the Blueprint file',
        default=os.path.join(gn_utils.repo_root(), 'Android.bp.extras'),
    )
    parser.add_argument(
        '--output',
        help='Blueprint file to create',
        default=os.path.join(gn_utils.repo_root(), 'Android.bp'),
    )
    parser.add_argument(
        'targets',
        nargs=argparse.REMAINDER,
        help='Targets to include in the blueprint (e.g., "//:perfetto_tests")')
    args = parser.parse_args()

    if args.desc:
        with open(args.desc) as f:
            desc = json.load(f)
    else:
        desc = gn_utils.create_build_description(gn_args)

    blueprint = create_blueprint_for_targets(desc, args.targets or
                                             default_targets)
    output = [
        """// Copyright (C) 2017 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// This file is automatically generated by %s. Do not edit.
""" % (__file__)
    ]
    blueprint.to_string(output)
    with open(args.extras, 'r') as r:
        for line in r:
            output.append(line.rstrip("\n\r"))
    with open(args.output, 'w') as f:
        f.write('\n'.join(output))


if __name__ == '__main__':
    sys.exit(main())