Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (C) 2019 The Android Open Source Project |
| 3 | # |
| 4 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | # you may not use this file except in compliance with the License. |
| 6 | # You may obtain a copy of the License at |
| 7 | # |
| 8 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | # |
| 10 | # Unless required by applicable law or agreed to in writing, software |
| 11 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | # See the License for the specific language governing permissions and |
| 14 | # limitations under the License. |
| 15 | |
| 16 | # This tool uses a collection of BUILD.gn files and build targets to generate |
| 17 | # an "amalgamated" C++ header and source file pair which compiles to an |
| 18 | # equivalent program. The tool also outputs the necessary compiler and linker |
| 19 | # flags needed to compile the resulting source code. |
| 20 | |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 21 | from __future__ import print_function |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 22 | import argparse |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 23 | import os |
| 24 | import re |
| 25 | import shutil |
| 26 | import subprocess |
| 27 | import sys |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 28 | import tempfile |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 29 | |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 30 | import gn_utils |
| 31 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 32 | # Default targets to include in the result. |
Primiano Tucci | 75ae50e | 2019-08-28 13:09:55 +0200 | [diff] [blame^] | 33 | # TODO(primiano): change this script to recurse into target deps when generating |
| 34 | # headers, but only for proto targets. .pbzero.h files don't include each other |
| 35 | # and we need to list targets here individually, which is unmaintainable. |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 36 | default_targets = [ |
Primiano Tucci | 658e2d6 | 2019-06-14 10:03:32 +0100 | [diff] [blame] | 37 | '//:libperfetto_client_experimental', |
Primiano Tucci | bf92d5c | 2019-07-03 14:07:45 +0100 | [diff] [blame] | 38 | "//include/perfetto/protozero:protozero", |
Primiano Tucci | 658e2d6 | 2019-06-14 10:03:32 +0100 | [diff] [blame] | 39 | "//protos/perfetto/common:zero", |
Primiano Tucci | 5a44944 | 2019-06-15 14:31:03 +0100 | [diff] [blame] | 40 | "//protos/perfetto/config:zero", |
Primiano Tucci | 75ae50e | 2019-08-28 13:09:55 +0200 | [diff] [blame^] | 41 | "//protos/perfetto/config/gpu:zero", |
| 42 | "//protos/perfetto/trace:minimal_zero", |
| 43 | "//protos/perfetto/trace:non_minimal_zero", |
Primiano Tucci | 5a44944 | 2019-06-15 14:31:03 +0100 | [diff] [blame] | 44 | "//protos/perfetto/trace/gpu:zero", |
Primiano Tucci | d434137 | 2019-07-26 14:36:54 +0100 | [diff] [blame] | 45 | "//protos/perfetto/trace/interned_data:zero", |
| 46 | "//protos/perfetto/trace/track_event:zero", |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 47 | ] |
| 48 | |
| 49 | # Arguments for the GN output directory (unless overridden from the command |
| 50 | # line). |
Primiano Tucci | 9c41165 | 2019-08-27 07:13:59 +0200 | [diff] [blame] | 51 | gn_args = ' '.join([ |
| 52 | 'is_debug=false', |
Primiano Tucci | 7e05fc1 | 2019-08-27 17:29:47 +0200 | [diff] [blame] | 53 | 'is_perfetto_build_generator=true', |
| 54 | 'is_perfetto_embedder=true', |
| 55 | 'use_custom_libcxx=false', |
| 56 | 'enable_perfetto_ipc=true', |
Primiano Tucci | 9c41165 | 2019-08-27 07:13:59 +0200 | [diff] [blame] | 57 | ]) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 58 | |
| 59 | # Compiler flags which aren't filtered out. |
| 60 | cflag_whitelist = r'^-(W.*|fno-exceptions|fPIC|std.*|fvisibility.*)$' |
| 61 | |
| 62 | # Linker flags which aren't filtered out. |
| 63 | ldflag_whitelist = r'^-()$' |
| 64 | |
| 65 | # Libraries which are filtered out. |
| 66 | lib_blacklist = r'^(c|gcc_eh)$' |
| 67 | |
| 68 | # Macros which aren't filtered out. |
| 69 | define_whitelist = r'^(PERFETTO.*|GOOGLE_PROTOBUF.*)$' |
| 70 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 71 | # Includes which will be removed from the generated source. |
| 72 | includes_to_remove = r'^(gtest).*$' |
| 73 | |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 74 | default_cflags = [ |
| 75 | # Since we're expanding header files into the generated source file, some |
| 76 | # constant may remain unused. |
| 77 | '-Wno-unused-const-variable' |
| 78 | ] |
| 79 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 80 | # Build flags to satisfy a protobuf (lite or full) dependency. |
| 81 | protobuf_cflags = [ |
| 82 | # Note that these point to the local copy of protobuf in buildtools. In |
| 83 | # reality the user of the amalgamated result will have to provide a path to |
| 84 | # an installed copy of the exact same version of protobuf which was used to |
| 85 | # generate the amalgamated build. |
| 86 | '-isystembuildtools/protobuf/src', |
| 87 | '-Lbuildtools/protobuf/src/.libs', |
| 88 | # We also need to disable some warnings for protobuf. |
| 89 | '-Wno-missing-prototypes', |
| 90 | '-Wno-missing-variable-declarations', |
| 91 | '-Wno-sign-conversion', |
| 92 | '-Wno-unknown-pragmas', |
| 93 | '-Wno-unused-macros', |
| 94 | ] |
| 95 | |
| 96 | # A mapping of dependencies to system libraries. Libraries in this map will not |
| 97 | # be built statically but instead added as dependencies of the amalgamated |
| 98 | # project. |
| 99 | system_library_map = { |
| 100 | '//buildtools:protobuf_full': { |
| 101 | 'libs': ['protobuf'], |
| 102 | 'cflags': protobuf_cflags, |
| 103 | }, |
| 104 | '//buildtools:protobuf_lite': { |
| 105 | 'libs': ['protobuf-lite'], |
| 106 | 'cflags': protobuf_cflags, |
| 107 | }, |
| 108 | '//buildtools:protoc_lib': {'libs': ['protoc']}, |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 109 | } |
| 110 | |
| 111 | # ---------------------------------------------------------------------------- |
| 112 | # End of configuration. |
| 113 | # ---------------------------------------------------------------------------- |
| 114 | |
| 115 | tool_name = os.path.basename(__file__) |
| 116 | preamble = """// Copyright (C) 2019 The Android Open Source Project |
| 117 | // |
| 118 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 119 | // you may not use this file except in compliance with the License. |
| 120 | // You may obtain a copy of the License at |
| 121 | // |
| 122 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 123 | // |
| 124 | // Unless required by applicable law or agreed to in writing, software |
| 125 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 126 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 127 | // See the License for the specific language governing permissions and |
| 128 | // limitations under the License. |
| 129 | // |
| 130 | // This file is automatically generated by %s. Do not edit. |
| 131 | """ % tool_name |
| 132 | |
| 133 | |
| 134 | def apply_blacklist(blacklist, items): |
| 135 | return [item for item in items if not re.match(blacklist, item)] |
| 136 | |
| 137 | |
| 138 | def apply_whitelist(whitelist, items): |
| 139 | return [item for item in items if re.match(whitelist, item)] |
| 140 | |
| 141 | |
| 142 | class Error(Exception): |
| 143 | pass |
| 144 | |
| 145 | |
| 146 | class DependencyNode(object): |
| 147 | """A target in a GN build description along with its dependencies.""" |
| 148 | |
| 149 | def __init__(self, target_name): |
| 150 | self.target_name = target_name |
| 151 | self.dependencies = set() |
| 152 | |
| 153 | def add_dependency(self, target_node): |
| 154 | if target_node in self.dependencies: |
| 155 | return |
| 156 | self.dependencies.add(target_node) |
| 157 | |
| 158 | def iterate_depth_first(self): |
| 159 | for node in sorted(self.dependencies, key=lambda n: n.target_name): |
| 160 | for node in node.iterate_depth_first(): |
| 161 | yield node |
| 162 | if self.target_name: |
| 163 | yield self |
| 164 | |
| 165 | |
| 166 | class DependencyTree(object): |
| 167 | """A tree of GN build target dependencies.""" |
| 168 | |
| 169 | def __init__(self): |
| 170 | self.target_to_node_map = {} |
| 171 | self.root = self._get_or_create_node(None) |
| 172 | |
| 173 | def _get_or_create_node(self, target_name): |
| 174 | if target_name in self.target_to_node_map: |
| 175 | return self.target_to_node_map[target_name] |
| 176 | node = DependencyNode(target_name) |
| 177 | self.target_to_node_map[target_name] = node |
| 178 | return node |
| 179 | |
| 180 | def add_dependency(self, from_target, to_target): |
| 181 | from_node = self._get_or_create_node(from_target) |
| 182 | to_node = self._get_or_create_node(to_target) |
| 183 | assert from_node is not to_node |
| 184 | from_node.add_dependency(to_node) |
| 185 | |
| 186 | def iterate_depth_first(self): |
| 187 | for node in self.root.iterate_depth_first(): |
| 188 | yield node |
| 189 | |
| 190 | |
| 191 | class AmalgamatedProject(object): |
| 192 | """In-memory representation of an amalgamated source/header pair.""" |
| 193 | |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 194 | def __init__(self, desc, source_deps, compute_deps_only=False): |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 195 | """Constructor. |
| 196 | |
| 197 | Args: |
| 198 | desc: JSON build description. |
| 199 | source_deps: A map of (source file, [dependency header]) which is |
| 200 | to detect which header files are included by each source file. |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 201 | compute_deps_only: If True, the project will only be used to compute |
| 202 | dependency information. Use |get_source_files()| to retrieve |
| 203 | the result. |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 204 | """ |
| 205 | self.desc = desc |
| 206 | self.source_deps = source_deps |
| 207 | self.header = [] |
| 208 | self.source = [] |
Sami Kyostila | 5bece6e | 2019-07-25 22:44:04 +0100 | [diff] [blame] | 209 | self.source_defines = [] |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 210 | # Note that we don't support multi-arg flags. |
| 211 | self.cflags = set(default_cflags) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 212 | self.ldflags = set() |
| 213 | self.defines = set() |
| 214 | self.libs = set() |
| 215 | self._dependency_tree = DependencyTree() |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 216 | self._processed_sources = set() |
| 217 | self._processed_headers = set() |
| 218 | self._processed_source_headers = set() # Header files included from .cc |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 219 | self._include_re = re.compile(r'#include "(.*)"') |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 220 | self._compute_deps_only = compute_deps_only |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 221 | |
| 222 | def add_target(self, target_name): |
| 223 | """Include |target_name| in the amalgamated result.""" |
| 224 | self._dependency_tree.add_dependency(None, target_name) |
| 225 | self._add_target_dependencies(target_name) |
| 226 | self._add_target_flags(target_name) |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 227 | self._add_target_headers(target_name) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 228 | |
| 229 | def _iterate_dep_edges(self, target_name): |
| 230 | target = self.desc[target_name] |
| 231 | for dep in target.get('deps', []): |
| 232 | # Ignore system libraries since they will be added as build-time |
| 233 | # dependencies. |
| 234 | if dep in system_library_map: |
| 235 | continue |
| 236 | # Don't descend into build action dependencies. |
| 237 | if self.desc[dep]['type'] == 'action': |
| 238 | continue |
| 239 | for sub_target, sub_dep in self._iterate_dep_edges(dep): |
| 240 | yield sub_target, sub_dep |
| 241 | yield target_name, dep |
| 242 | |
| 243 | def _iterate_target_and_deps(self, target_name): |
| 244 | yield target_name |
| 245 | for _, dep in self._iterate_dep_edges(target_name): |
| 246 | yield dep |
| 247 | |
| 248 | def _add_target_dependencies(self, target_name): |
| 249 | for target, dep in self._iterate_dep_edges(target_name): |
| 250 | self._dependency_tree.add_dependency(target, dep) |
| 251 | |
| 252 | def process_dep(dep): |
| 253 | if dep in system_library_map: |
| 254 | self.libs.update(system_library_map[dep].get('libs', [])) |
| 255 | self.cflags.update(system_library_map[dep].get('cflags', [])) |
| 256 | self.defines.update(system_library_map[dep].get('defines', [])) |
| 257 | return True |
| 258 | |
| 259 | def walk_all_deps(target_name): |
| 260 | target = self.desc[target_name] |
| 261 | for dep in target.get('deps', []): |
| 262 | if process_dep(dep): |
| 263 | return |
| 264 | walk_all_deps(dep) |
| 265 | walk_all_deps(target_name) |
| 266 | |
| 267 | def _filter_cflags(self, cflags): |
| 268 | # Since we want to deduplicate flags, combine two-part switches (e.g., |
| 269 | # "-foo bar") into one value ("-foobar") so we can store the result as |
| 270 | # a set. |
| 271 | result = [] |
| 272 | for flag in cflags: |
| 273 | if flag.startswith('-'): |
| 274 | result.append(flag) |
| 275 | else: |
| 276 | result[-1] += flag |
| 277 | return apply_whitelist(cflag_whitelist, result) |
| 278 | |
| 279 | def _add_target_flags(self, target_name): |
| 280 | for target_name in self._iterate_target_and_deps(target_name): |
| 281 | target = self.desc[target_name] |
| 282 | self.cflags.update(self._filter_cflags(target.get('cflags', []))) |
| 283 | self.cflags.update(self._filter_cflags(target.get('cflags_cc', []))) |
| 284 | self.ldflags.update( |
| 285 | apply_whitelist(ldflag_whitelist, target.get('ldflags', []))) |
| 286 | self.libs.update( |
| 287 | apply_blacklist(lib_blacklist, target.get('libs', []))) |
| 288 | self.defines.update( |
| 289 | apply_whitelist(define_whitelist, target.get('defines', []))) |
| 290 | |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 291 | def _add_target_headers(self, target_name): |
| 292 | target = self.desc[target_name] |
| 293 | if not 'sources' in target: |
| 294 | return |
| 295 | headers = [gn_utils.label_to_path(s) |
| 296 | for s in target['sources'] if s.endswith('.h')] |
| 297 | for header in headers: |
| 298 | self._add_header(target_name, header) |
| 299 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 300 | def _get_include_dirs(self, target_name): |
| 301 | include_dirs = set() |
| 302 | for target_name in self._iterate_target_and_deps(target_name): |
| 303 | target = self.desc[target_name] |
| 304 | if 'include_dirs' in target: |
| 305 | include_dirs.update( |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 306 | [gn_utils.label_to_path(d) for d in target['include_dirs']]) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 307 | return include_dirs |
| 308 | |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 309 | def _add_source_included_header( |
| 310 | self, include_dirs, allowed_files, header_name): |
| 311 | if header_name in self._processed_source_headers: |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 312 | return |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 313 | self._processed_source_headers.add(header_name) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 314 | for include_dir in include_dirs: |
| 315 | full_path = os.path.join(include_dir, header_name) |
| 316 | if os.path.exists(full_path): |
| 317 | if not full_path in allowed_files: |
| 318 | return |
| 319 | with open(full_path) as f: |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 320 | self.source.append( |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 321 | '// %s begin header: %s' % (tool_name, full_path)) |
Primiano Tucci | 2e1ae92 | 2019-05-30 12:13:47 +0100 | [diff] [blame] | 322 | self.source.extend( |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 323 | self._process_source_includes( |
Primiano Tucci | 2e1ae92 | 2019-05-30 12:13:47 +0100 | [diff] [blame] | 324 | include_dirs, allowed_files, f)) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 325 | return |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 326 | if self._compute_deps_only: |
| 327 | return |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 328 | msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) |
| 329 | raise Error('Header file %s not found. %s' % (header_name, msg)) |
| 330 | |
| 331 | def _add_source(self, target_name, source_name): |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 332 | if source_name in self._processed_sources: |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 333 | return |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 334 | self._processed_sources.add(source_name) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 335 | include_dirs = self._get_include_dirs(target_name) |
| 336 | deps = self.source_deps[source_name] |
| 337 | if not os.path.exists(source_name): |
| 338 | raise Error('Source file %s not found' % source_name) |
| 339 | with open(source_name) as f: |
| 340 | self.source.append( |
| 341 | '// %s begin source: %s' % (tool_name, source_name)) |
| 342 | try: |
| 343 | self.source.extend(self._patch_source(source_name, |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 344 | self._process_source_includes(include_dirs, deps, f))) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 345 | except Error as e: |
| 346 | raise Error( |
| 347 | 'Failed adding source %s: %s' % (source_name, e.message)) |
| 348 | |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 349 | def _add_header_included_header(self, include_dirs, header_name): |
| 350 | if header_name in self._processed_headers: |
| 351 | return |
| 352 | self._processed_headers.add(header_name) |
| 353 | for include_dir in include_dirs: |
| 354 | full_path = os.path.join(include_dir, header_name) |
| 355 | if os.path.exists(full_path): |
| 356 | with open(full_path) as f: |
| 357 | self.header.append( |
| 358 | '// %s begin header: %s' % (tool_name, full_path)) |
Primiano Tucci | 2e1ae92 | 2019-05-30 12:13:47 +0100 | [diff] [blame] | 359 | self.header.extend( |
| 360 | self._process_header_includes(include_dirs, f)) |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 361 | return |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 362 | if self._compute_deps_only: |
| 363 | return |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 364 | msg = 'Looked in %s' % ', '.join('"%s"' % d for d in include_dirs) |
| 365 | raise Error('Header file %s not found. %s' % (header_name, msg)) |
| 366 | |
| 367 | def _add_header(self, target_name, header_name): |
| 368 | if header_name in self._processed_headers: |
| 369 | return |
| 370 | self._processed_headers.add(header_name) |
| 371 | include_dirs = self._get_include_dirs(target_name) |
| 372 | if not os.path.exists(header_name): |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 373 | if self._compute_deps_only: |
| 374 | return |
Primiano Tucci | 75ae50e | 2019-08-28 13:09:55 +0200 | [diff] [blame^] | 375 | raise Error('Header file %s not found' % header_name) |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 376 | with open(header_name) as f: |
| 377 | self.header.append( |
| 378 | '// %s begin header: %s' % (tool_name, header_name)) |
| 379 | try: |
Primiano Tucci | 2e1ae92 | 2019-05-30 12:13:47 +0100 | [diff] [blame] | 380 | self.header.extend( |
| 381 | self._process_header_includes(include_dirs, f)) |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 382 | except Error as e: |
| 383 | raise Error( |
| 384 | 'Failed adding header %s: %s' % (header_name, e.message)) |
| 385 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 386 | def _patch_source(self, source_name, lines): |
| 387 | result = [] |
| 388 | namespace = re.sub(r'[^a-z]', '_', |
| 389 | os.path.splitext(os.path.basename(source_name))[0]) |
| 390 | for line in lines: |
| 391 | # Protobuf generates an identical anonymous function into each |
| 392 | # message description. Rename all but the first occurrence to avoid |
| 393 | # duplicate symbol definitions. |
| 394 | line = line.replace('MergeFromFail', '%s_MergeFromFail' % namespace) |
| 395 | result.append(line) |
| 396 | return result |
| 397 | |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 398 | def _process_source_includes(self, include_dirs, allowed_files, file): |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 399 | result = [] |
| 400 | for line in file: |
| 401 | line = line.rstrip('\n') |
| 402 | m = self._include_re.match(line) |
| 403 | if not m: |
| 404 | result.append(line) |
| 405 | continue |
| 406 | elif re.match(includes_to_remove, m.group(1)): |
| 407 | result.append('// %s removed: %s' % (tool_name, line)) |
Sami Kyostila | fd36776 | 2019-05-22 17:25:50 +0100 | [diff] [blame] | 408 | else: |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 409 | result.append('// %s expanded: %s' % (tool_name, line)) |
Sami Kyostila | 7e8509f | 2019-05-29 12:36:24 +0100 | [diff] [blame] | 410 | self._add_source_included_header( |
| 411 | include_dirs, allowed_files, m.group(1)) |
| 412 | return result |
| 413 | |
| 414 | def _process_header_includes(self, include_dirs, file): |
| 415 | result = [] |
| 416 | for line in file: |
| 417 | line = line.rstrip('\n') |
| 418 | m = self._include_re.match(line) |
| 419 | if not m: |
| 420 | result.append(line) |
| 421 | continue |
| 422 | elif re.match(includes_to_remove, m.group(1)): |
| 423 | result.append('// %s removed: %s' % (tool_name, line)) |
| 424 | else: |
| 425 | result.append('// %s expanded: %s' % (tool_name, line)) |
| 426 | self._add_header_included_header(include_dirs, m.group(1)) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 427 | return result |
| 428 | |
| 429 | def generate(self): |
| 430 | """Prepares the output for this amalgamated project. |
| 431 | |
| 432 | Call save() to persist the result. |
| 433 | """ |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 434 | assert not self._compute_deps_only |
Sami Kyostila | 5bece6e | 2019-07-25 22:44:04 +0100 | [diff] [blame] | 435 | self.source_defines.append('// %s: predefined macros' % tool_name) |
| 436 | def add_define(name): |
| 437 | # Valued macros aren't supported for now. |
| 438 | assert '=' not in name |
| 439 | self.source_defines.append('#if !defined(%s)' % name) |
| 440 | self.source_defines.append('#define %s' % name) |
| 441 | self.source_defines.append('#endif') |
| 442 | for name in self.defines: |
| 443 | add_define(name) |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 444 | for target_name, source_name in self.get_source_files(): |
| 445 | self._add_source(target_name, source_name) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 446 | |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 447 | def get_source_files(self): |
| 448 | """Return a list of (target, [source file]) that describes the source |
| 449 | files pulled in by each target which is a dependency of this project. |
| 450 | """ |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 451 | source_files = [] |
| 452 | for node in self._dependency_tree.iterate_depth_first(): |
| 453 | target = self.desc[node.target_name] |
| 454 | if not 'sources' in target: |
| 455 | continue |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 456 | sources = [(node.target_name, gn_utils.label_to_path(s)) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 457 | for s in target['sources'] if s.endswith('.cc')] |
| 458 | source_files.extend(sources) |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 459 | return source_files |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 460 | |
| 461 | def _get_nice_path(self, prefix, format): |
| 462 | basename = os.path.basename(prefix) |
| 463 | return os.path.join( |
| 464 | os.path.relpath(os.path.dirname(prefix)), format % basename) |
| 465 | |
| 466 | def save(self, output_prefix): |
| 467 | """Save the generated header and source file pair. |
| 468 | |
| 469 | Returns a message describing the output with build instructions. |
| 470 | """ |
| 471 | header_file = self._get_nice_path(output_prefix, '%s.h') |
| 472 | source_file = self._get_nice_path(output_prefix, '%s.cc') |
| 473 | with open(header_file, 'w') as f: |
| 474 | f.write('\n'.join([preamble] + self.header + ['\n'])) |
| 475 | with open(source_file, 'w') as f: |
| 476 | include_stmt = '#include "%s"' % os.path.basename(header_file) |
Sami Kyostila | 5bece6e | 2019-07-25 22:44:04 +0100 | [diff] [blame] | 477 | f.write('\n'.join( |
| 478 | [preamble] + self.source_defines + [include_stmt] + |
| 479 | self.source + ['\n'])) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 480 | build_cmd = self.get_build_command(output_prefix) |
| 481 | |
| 482 | return """Amalgamated project written to %s and %s. |
| 483 | |
| 484 | Build settings: |
| 485 | - cflags: %s |
| 486 | - ldflags: %s |
| 487 | - libs: %s |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 488 | |
| 489 | Example build command: |
| 490 | |
| 491 | %s |
| 492 | """ % (header_file, source_file, ' '.join(self.cflags), ' '.join(self.ldflags), |
Sami Kyostila | 5bece6e | 2019-07-25 22:44:04 +0100 | [diff] [blame] | 493 | ' '.join(self.libs), ' '.join(build_cmd)) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 494 | |
| 495 | def get_build_command(self, output_prefix): |
| 496 | """Returns an example command line for building the output source.""" |
| 497 | source = self._get_nice_path(output_prefix, '%s.cc') |
| 498 | library = self._get_nice_path(output_prefix, 'lib%s.so') |
| 499 | build_cmd = ['clang++', source, '-o', library, '-shared'] + \ |
| 500 | sorted(self.cflags) + sorted(self.ldflags) |
| 501 | for lib in sorted(self.libs): |
| 502 | build_cmd.append('-l%s' % lib) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 503 | return build_cmd |
| 504 | |
| 505 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 506 | def main(): |
| 507 | parser = argparse.ArgumentParser( |
| 508 | description='Generate an amalgamated header/source pair from a GN ' |
| 509 | 'build description.') |
| 510 | parser.add_argument( |
| 511 | '--output', |
| 512 | help='Base name of files to create. A .cc/.h extension will be added', |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 513 | default=os.path.join(gn_utils.repo_root(), 'perfetto')) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 514 | parser.add_argument( |
| 515 | '--gn_args', help='GN arguments used to prepare the output directory', |
| 516 | default=gn_args) |
| 517 | parser.add_argument( |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 518 | '--keep', help='Don\'t delete the GN output directory at exit', |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 519 | action='store_true') |
| 520 | parser.add_argument( |
| 521 | '--build', help='Also compile the generated files', |
| 522 | action='store_true') |
| 523 | parser.add_argument( |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 524 | '--check', help='Don\'t keep the generated files', |
| 525 | action='store_true') |
| 526 | parser.add_argument('--quiet', help='Only report errors', |
| 527 | action='store_true') |
| 528 | parser.add_argument( |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 529 | '--dump-deps', |
| 530 | help='List all source files that the amalgamated output depends on', |
| 531 | action='store_true') |
| 532 | parser.add_argument( |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 533 | 'targets', |
| 534 | nargs=argparse.REMAINDER, |
| 535 | help='Targets to include in the output (e.g., "//:libperfetto")') |
| 536 | args = parser.parse_args() |
| 537 | targets = args.targets or default_targets |
| 538 | |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 539 | output = args.output |
| 540 | if args.check: |
| 541 | output = os.path.join(tempfile.mkdtemp(), 'perfetto_amalgamated') |
| 542 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 543 | try: |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 544 | if not args.quiet: |
| 545 | print('Building project...') |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 546 | out = gn_utils.prepare_out_directory( |
| 547 | args.gn_args, 'tmp.gen_amalgamated') |
| 548 | desc = gn_utils.load_build_description(out) |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 549 | |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 550 | # We need to build everything first so that the necessary header |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 551 | # dependencies get generated. However if we are just dumping dependency |
| 552 | # information this can be skipped, allowing cross-platform operation. |
| 553 | if not args.dump_deps: |
| 554 | gn_utils.build_targets(out, targets) |
Sami Kyostila | 3c88a1d | 2019-05-22 18:29:42 +0100 | [diff] [blame] | 555 | source_deps = gn_utils.compute_source_dependencies(out) |
Sami Kyostila | d4c857e | 2019-08-08 18:44:33 +0100 | [diff] [blame] | 556 | project = AmalgamatedProject( |
| 557 | desc, source_deps, compute_deps_only=args.dump_deps) |
| 558 | |
| 559 | for target in targets: |
| 560 | project.add_target(target) |
| 561 | |
| 562 | if args.dump_deps: |
| 563 | source_files = [ |
| 564 | source_file for _, source_file in project.get_source_files()] |
| 565 | print('\n'.join(sorted(set(source_files)))) |
| 566 | return |
| 567 | |
| 568 | project.generate() |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 569 | result = project.save(output) |
| 570 | if not args.quiet: |
| 571 | print(result) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 572 | if args.build: |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 573 | if not args.quiet: |
| 574 | sys.stdout.write('Building amalgamated project...') |
| 575 | sys.stdout.flush() |
| 576 | subprocess.check_call(project.get_build_command(output)) |
| 577 | if not args.quiet: |
| 578 | print('done') |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 579 | finally: |
| 580 | if not args.keep: |
| 581 | shutil.rmtree(out) |
Sami Kyostila | 468e61d | 2019-05-23 15:54:01 +0100 | [diff] [blame] | 582 | if args.check: |
| 583 | shutil.rmtree(os.path.dirname(output)) |
Sami Kyostila | 0a34b03 | 2019-05-16 18:28:48 +0100 | [diff] [blame] | 584 | |
| 585 | if __name__ == '__main__': |
| 586 | sys.exit(main()) |