Ben Murdoch | 61f157c | 2016-09-16 13:49:30 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | # Copyright 2016 the V8 project authors. All rights reserved. |
| 4 | # Copyright 2014 The Chromium Authors. All rights reserved. |
| 5 | # Use of this source code is governed by a BSD-style license that can be |
| 6 | # found in the LICENSE file. |
| 7 | |
| 8 | """Given the output of -t commands from a ninja build for a gyp and GN generated |
| 9 | build, report on differences between the command lines.""" |
| 10 | |
| 11 | |
| 12 | import os |
| 13 | import shlex |
| 14 | import subprocess |
| 15 | import sys |
| 16 | |
| 17 | |
| 18 | # Must be in v8/. |
| 19 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) |
| 20 | os.chdir(BASE_DIR) |
| 21 | |
| 22 | |
| 23 | g_total_differences = 0 |
| 24 | |
| 25 | |
| 26 | def FindAndRemoveArgWithValue(command_line, argname): |
| 27 | """Given a command line as a list, remove and return the value of an option |
| 28 | that takes a value as a separate entry. |
| 29 | |
| 30 | Modifies |command_line| in place. |
| 31 | """ |
| 32 | if argname not in command_line: |
| 33 | return '' |
| 34 | location = command_line.index(argname) |
| 35 | value = command_line[location + 1] |
| 36 | command_line[location:location + 2] = [] |
| 37 | return value |
| 38 | |
| 39 | |
| 40 | def MergeSpacedArgs(command_line, argname): |
| 41 | """Combine all arguments |argname| with their values, separated by a space.""" |
| 42 | i = 0 |
| 43 | result = [] |
| 44 | while i < len(command_line): |
| 45 | arg = command_line[i] |
| 46 | if arg == argname: |
| 47 | result.append(arg + ' ' + command_line[i + 1]) |
| 48 | i += 1 |
| 49 | else: |
| 50 | result.append(arg) |
| 51 | i += 1 |
| 52 | return result |
| 53 | |
| 54 | |
| 55 | def NormalizeSymbolArguments(command_line): |
| 56 | """Normalize -g arguments. |
| 57 | |
| 58 | If there's no -g args, it's equivalent to -g0. -g2 is equivalent to -g. |
| 59 | Modifies |command_line| in place. |
| 60 | """ |
| 61 | # Strip -g0 if there's no symbols. |
| 62 | have_some_symbols = False |
| 63 | for x in command_line: |
| 64 | if x.startswith('-g') and x != '-g0': |
| 65 | have_some_symbols = True |
| 66 | if not have_some_symbols and '-g0' in command_line: |
| 67 | command_line.remove('-g0') |
| 68 | |
| 69 | # Rename -g2 to -g. |
| 70 | if '-g2' in command_line: |
| 71 | command_line[command_line.index('-g2')] = '-g' |
| 72 | |
| 73 | |
| 74 | def GetFlags(lines, build_dir): |
| 75 | """Turn a list of command lines into a semi-structured dict.""" |
| 76 | is_win = sys.platform == 'win32' |
| 77 | flags_by_output = {} |
| 78 | for line in lines: |
| 79 | command_line = shlex.split(line.strip(), posix=not is_win)[1:] |
| 80 | |
| 81 | output_name = FindAndRemoveArgWithValue(command_line, '-o') |
| 82 | dep_name = FindAndRemoveArgWithValue(command_line, '-MF') |
| 83 | |
| 84 | NormalizeSymbolArguments(command_line) |
| 85 | |
| 86 | command_line = MergeSpacedArgs(command_line, '-Xclang') |
| 87 | |
| 88 | cc_file = [x for x in command_line if x.endswith('.cc') or |
| 89 | x.endswith('.c') or |
| 90 | x.endswith('.cpp')] |
| 91 | if len(cc_file) != 1: |
| 92 | print 'Skipping %s' % command_line |
| 93 | continue |
| 94 | assert len(cc_file) == 1 |
| 95 | |
| 96 | if is_win: |
| 97 | rsp_file = [x for x in command_line if x.endswith('.rsp')] |
| 98 | assert len(rsp_file) <= 1 |
| 99 | if rsp_file: |
| 100 | rsp_file = os.path.join(build_dir, rsp_file[0][1:]) |
| 101 | with open(rsp_file, "r") as open_rsp_file: |
| 102 | command_line = shlex.split(open_rsp_file, posix=False) |
| 103 | |
| 104 | defines = [x for x in command_line if x.startswith('-D')] |
| 105 | include_dirs = [x for x in command_line if x.startswith('-I')] |
| 106 | dash_f = [x for x in command_line if x.startswith('-f')] |
| 107 | warnings = \ |
| 108 | [x for x in command_line if x.startswith('/wd' if is_win else '-W')] |
| 109 | others = [x for x in command_line if x not in defines and \ |
| 110 | x not in include_dirs and \ |
| 111 | x not in dash_f and \ |
| 112 | x not in warnings and \ |
| 113 | x not in cc_file] |
| 114 | |
| 115 | for index, value in enumerate(include_dirs): |
| 116 | if value == '-Igen': |
| 117 | continue |
| 118 | path = value[2:] |
| 119 | if not os.path.isabs(path): |
| 120 | path = os.path.join(build_dir, path) |
| 121 | include_dirs[index] = '-I' + os.path.normpath(path) |
| 122 | |
| 123 | # GYP supports paths above the source root like <(DEPTH)/../foo while such |
| 124 | # paths are unsupported by gn. But gn allows to use system-absolute paths |
| 125 | # instead (paths that start with single '/'). Normalize all paths. |
| 126 | cc_file = [os.path.normpath(os.path.join(build_dir, cc_file[0]))] |
| 127 | |
| 128 | # Filter for libFindBadConstructs.so having a relative path in one and |
| 129 | # absolute path in the other. |
| 130 | others_filtered = [] |
| 131 | for x in others: |
| 132 | if x.startswith('-Xclang ') and x.endswith('libFindBadConstructs.so'): |
| 133 | others_filtered.append( |
| 134 | '-Xclang ' + |
| 135 | os.path.join(os.getcwd(), |
| 136 | os.path.normpath( |
| 137 | os.path.join('out/gn_flags', x.split(' ', 1)[1])))) |
| 138 | elif x.startswith('-B'): |
| 139 | others_filtered.append( |
| 140 | '-B' + |
| 141 | os.path.join(os.getcwd(), |
| 142 | os.path.normpath(os.path.join('out/gn_flags', x[2:])))) |
| 143 | else: |
| 144 | others_filtered.append(x) |
| 145 | others = others_filtered |
| 146 | |
| 147 | flags_by_output[cc_file[0]] = { |
| 148 | 'output': output_name, |
| 149 | 'depname': dep_name, |
| 150 | 'defines': sorted(defines), |
| 151 | 'include_dirs': sorted(include_dirs), # TODO(scottmg): This is wrong. |
| 152 | 'dash_f': sorted(dash_f), |
| 153 | 'warnings': sorted(warnings), |
| 154 | 'other': sorted(others), |
| 155 | } |
| 156 | return flags_by_output |
| 157 | |
| 158 | |
| 159 | def CompareLists(gyp, gn, name, dont_care_gyp=None, dont_care_gn=None): |
| 160 | """Return a report of any differences between gyp and gn lists, ignoring |
| 161 | anything in |dont_care_{gyp|gn}| respectively.""" |
| 162 | global g_total_differences |
| 163 | if not dont_care_gyp: |
| 164 | dont_care_gyp = [] |
| 165 | if not dont_care_gn: |
| 166 | dont_care_gn = [] |
| 167 | output = '' |
| 168 | if gyp[name] != gn[name]: |
| 169 | gyp_set = set(gyp[name]) |
| 170 | gn_set = set(gn[name]) |
| 171 | missing_in_gyp = gyp_set - gn_set |
| 172 | missing_in_gn = gn_set - gyp_set |
| 173 | missing_in_gyp -= set(dont_care_gyp) |
| 174 | missing_in_gn -= set(dont_care_gn) |
| 175 | if missing_in_gyp or missing_in_gn: |
| 176 | output += ' %s differ:\n' % name |
| 177 | if missing_in_gyp: |
| 178 | output += ' In gyp, but not in GN:\n %s' % '\n '.join( |
| 179 | sorted(missing_in_gyp)) + '\n' |
| 180 | g_total_differences += len(missing_in_gyp) |
| 181 | if missing_in_gn: |
| 182 | output += ' In GN, but not in gyp:\n %s' % '\n '.join( |
| 183 | sorted(missing_in_gn)) + '\n\n' |
| 184 | g_total_differences += len(missing_in_gn) |
| 185 | return output |
| 186 | |
| 187 | |
| 188 | def Run(command_line): |
| 189 | """Run |command_line| as a subprocess and return stdout. Raises on error.""" |
| 190 | return subprocess.check_output(command_line, shell=True) |
| 191 | |
| 192 | |
| 193 | def main(): |
| 194 | if len(sys.argv) < 4: |
| 195 | print ('usage: %s gn_outdir gyp_outdir gn_target ' |
| 196 | '[gyp_target1, gyp_target2, ...]' % __file__) |
| 197 | return 1 |
| 198 | |
| 199 | if len(sys.argv) == 4: |
| 200 | sys.argv.append(sys.argv[3]) |
| 201 | gn_out_dir = sys.argv[1] |
| 202 | print >> sys.stderr, 'Expecting gn outdir in %s...' % gn_out_dir |
| 203 | gn = Run('ninja -C %s -t commands %s' % (gn_out_dir, sys.argv[3])) |
| 204 | if sys.platform == 'win32': |
| 205 | # On Windows flags are stored in .rsp files which are created during build. |
| 206 | print >> sys.stderr, 'Building in %s...' % gn_out_dir |
| 207 | Run('ninja -C %s -d keeprsp %s' % (gn_out_dir, sys.argv[3])) |
| 208 | |
| 209 | gyp_out_dir = sys.argv[2] |
| 210 | print >> sys.stderr, 'Expecting gyp outdir in %s...' % gyp_out_dir |
| 211 | gyp = Run('ninja -C %s -t commands %s' % (gyp_out_dir, " ".join(sys.argv[4:]))) |
| 212 | if sys.platform == 'win32': |
| 213 | # On Windows flags are stored in .rsp files which are created during build. |
| 214 | print >> sys.stderr, 'Building in %s...' % gyp_out_dir |
| 215 | Run('ninja -C %s -d keeprsp %s' % (gyp_out_dir, " ".join(sys.argv[4:]))) |
| 216 | |
| 217 | all_gyp_flags = GetFlags(gyp.splitlines(), |
| 218 | os.path.join(os.getcwd(), gyp_out_dir)) |
| 219 | all_gn_flags = GetFlags(gn.splitlines(), |
| 220 | os.path.join(os.getcwd(), gn_out_dir)) |
| 221 | gyp_files = set(all_gyp_flags.keys()) |
| 222 | gn_files = set(all_gn_flags.keys()) |
| 223 | different_source_list = gyp_files != gn_files |
| 224 | if different_source_list: |
| 225 | print 'Different set of sources files:' |
| 226 | print ' In gyp, not in GN:\n %s' % '\n '.join( |
| 227 | sorted(gyp_files - gn_files)) |
| 228 | print ' In GN, not in gyp:\n %s' % '\n '.join( |
| 229 | sorted(gn_files - gyp_files)) |
| 230 | print '\nNote that flags will only be compared for files in both sets.\n' |
| 231 | file_list = gyp_files & gn_files |
| 232 | files_with_given_differences = {} |
| 233 | for filename in sorted(file_list): |
| 234 | gyp_flags = all_gyp_flags[filename] |
| 235 | gn_flags = all_gn_flags[filename] |
| 236 | differences = CompareLists(gyp_flags, gn_flags, 'dash_f') |
| 237 | differences += CompareLists(gyp_flags, gn_flags, 'defines') |
| 238 | differences += CompareLists(gyp_flags, gn_flags, 'include_dirs', |
| 239 | ['-I%s' % os.path.dirname(BASE_DIR)]) |
| 240 | differences += CompareLists(gyp_flags, gn_flags, 'warnings', |
| 241 | # More conservative warnings in GN we consider to be OK. |
| 242 | dont_care_gyp=[ |
| 243 | '/wd4091', # 'keyword' : ignored on left of 'type' when no variable |
| 244 | # is declared. |
| 245 | '/wd4456', # Declaration hides previous local declaration. |
| 246 | '/wd4457', # Declaration hides function parameter. |
| 247 | '/wd4458', # Declaration hides class member. |
| 248 | '/wd4459', # Declaration hides global declaration. |
| 249 | '/wd4702', # Unreachable code. |
| 250 | '/wd4800', # Forcing value to bool 'true' or 'false'. |
| 251 | '/wd4838', # Conversion from 'type' to 'type' requires a narrowing |
| 252 | # conversion. |
| 253 | ] if sys.platform == 'win32' else None, |
| 254 | dont_care_gn=[ |
| 255 | '-Wendif-labels', |
| 256 | '-Wextra', |
| 257 | '-Wsign-compare', |
| 258 | ] if not sys.platform == 'win32' else None) |
| 259 | differences += CompareLists(gyp_flags, gn_flags, 'other') |
| 260 | if differences: |
| 261 | files_with_given_differences.setdefault(differences, []).append(filename) |
| 262 | |
| 263 | for diff, files in files_with_given_differences.iteritems(): |
| 264 | print '\n'.join(sorted(files)) |
| 265 | print diff |
| 266 | |
| 267 | print 'Total differences:', g_total_differences |
| 268 | # TODO(scottmg): Return failure on difference once we're closer to identical. |
| 269 | return 0 |
| 270 | |
| 271 | |
| 272 | if __name__ == '__main__': |
| 273 | sys.exit(main()) |