| # |
| # Copyright 2015 Google Inc. |
| # |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| # |
| |
| #!/usr/bin/env python |
| |
| usage = ''' |
| Write buildbot spec to outfile based on the bot name: |
| $ python buildbot_spec.py outfile Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug |
| Or run self-tests: |
| $ python buildbot_spec.py test |
| ''' |
| |
| import inspect |
| import json |
| import os |
| import sys |
| |
| import builder_name_schema |
| import dm_flags |
| import nanobench_flags |
| |
| |
| CONFIG_DEBUG = 'Debug' |
| CONFIG_RELEASE = 'Release' |
| |
| |
| def lineno(): |
| caller = inspect.stack()[1] # Up one level to our caller. |
| return inspect.getframeinfo(caller[0]).lineno |
| |
| # Since we don't actually start coverage until we're in the self-test, |
| # some function def lines aren't reported as covered. Add them to this |
| # list so that we can ignore them. |
| cov_skip = [] |
| |
| cov_start = lineno()+1 # We care about coverage starting just past this def. |
| def gyp_defines(builder_dict): |
| gyp_defs = {} |
| |
| # skia_arch_type. |
| if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: |
| arch = builder_dict['target_arch'] |
| elif builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: |
| arch = None |
| else: |
| arch = builder_dict['arch'] |
| |
| arch_types = { |
| 'x86': 'x86', |
| 'x86_64': 'x86_64', |
| 'Arm7': 'arm', |
| 'Arm64': 'arm64', |
| 'Mips': 'mips32', |
| 'Mips64': 'mips64', |
| 'MipsDSP2': 'mips32', |
| } |
| if arch in arch_types: |
| gyp_defs['skia_arch_type'] = arch_types[arch] |
| |
| # housekeeper: build shared lib. |
| if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: |
| gyp_defs['skia_shared_lib'] = '1' |
| |
| # skia_gpu. |
| if builder_dict.get('cpu_or_gpu') == 'CPU': |
| gyp_defs['skia_gpu'] = '0' |
| |
| # skia_warnings_as_errors. |
| werr = False |
| if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD: |
| if 'Win' in builder_dict.get('os', ''): |
| if not ('GDI' in builder_dict.get('extra_config', '') or |
| 'Exceptions' in builder_dict.get('extra_config', '')): |
| werr = True |
| elif ('Mac' in builder_dict.get('os', '') and |
| 'Android' in builder_dict.get('extra_config', '')): |
| werr = False |
| else: |
| werr = True |
| gyp_defs['skia_warnings_as_errors'] = str(int(werr)) # True/False -> '1'/'0' |
| |
| # Win debugger. |
| if 'Win' in builder_dict.get('os', ''): |
| gyp_defs['skia_win_debuggers_path'] = 'c:/DbgHelp' |
| |
| # Qt SDK (Win). |
| if 'Win' in builder_dict.get('os', ''): |
| if builder_dict.get('os') == 'Win8': |
| gyp_defs['qt_sdk'] = 'C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/' |
| else: |
| gyp_defs['qt_sdk'] = 'C:/Qt/4.8.5/' |
| |
| # ANGLE. |
| if builder_dict.get('extra_config') == 'ANGLE': |
| gyp_defs['skia_angle'] = '1' |
| if builder_dict.get('os', '') in ('Ubuntu', 'Linux'): |
| gyp_defs['use_x11'] = '1' |
| gyp_defs['chromeos'] = '0' |
| |
| # GDI. |
| if builder_dict.get('extra_config') == 'GDI': |
| gyp_defs['skia_gdi'] = '1' |
| |
| # Build with Exceptions on Windows. |
| if ('Win' in builder_dict.get('os', '') and |
| builder_dict.get('extra_config') == 'Exceptions'): |
| gyp_defs['skia_win_exceptions'] = '1' |
| |
| # iOS. |
| if (builder_dict.get('os') == 'iOS' or |
| builder_dict.get('extra_config') == 'iOS'): |
| gyp_defs['skia_os'] = 'ios' |
| |
| # Shared library build. |
| if builder_dict.get('extra_config') == 'Shared': |
| gyp_defs['skia_shared_lib'] = '1' |
| |
| # PDF viewer in GM. |
| if (builder_dict.get('os') == 'Mac10.8' and |
| builder_dict.get('arch') == 'x86_64' and |
| builder_dict.get('configuration') == 'Release'): |
| gyp_defs['skia_run_pdfviewer_in_gm'] = '1' |
| |
| # Clang. |
| if builder_dict.get('compiler') == 'Clang': |
| gyp_defs['skia_clang_build'] = '1' |
| |
| # Valgrind. |
| if 'Valgrind' in builder_dict.get('extra_config', ''): |
| gyp_defs['skia_release_optimization_level'] = '1' |
| |
| # Link-time code generation just wastes time on compile-only bots. |
| if (builder_dict.get('role') == builder_name_schema.BUILDER_ROLE_BUILD and |
| builder_dict.get('compiler') == 'MSVC'): |
| gyp_defs['skia_win_ltcg'] = '0' |
| |
| # Mesa. |
| if (builder_dict.get('extra_config') == 'Mesa' or |
| builder_dict.get('cpu_or_gpu_value') == 'Mesa'): |
| gyp_defs['skia_mesa'] = '1' |
| |
| # VisualBench |
| if builder_dict.get('extra_config') == 'VisualBench': |
| gyp_defs['skia_use_sdl'] = '1' |
| |
| # skia_use_android_framework_defines. |
| if builder_dict.get('extra_config') == 'Android_FrameworkDefs': |
| gyp_defs['skia_use_android_framework_defines'] = '1' |
| |
| # Skia dump stats for perf tests and gpu |
| if (builder_dict.get('cpu_or_gpu') == 'GPU' and |
| builder_dict.get('role') == 'Perf'): |
| gyp_defs['skia_dump_stats'] = '1' |
| |
| return gyp_defs |
| |
| |
| cov_skip.extend([lineno(), lineno() + 1]) |
| def get_extra_env_vars(builder_dict): |
| env = {} |
| if builder_dict.get('configuration') == 'Coverage': |
| # We have to use Clang 3.6 because earlier versions do not support the |
| # compile flags we use and 3.7 and 3.8 hit asserts during compilation. |
| env['CC'] = '/usr/bin/clang-3.6' |
| env['CXX'] = '/usr/bin/clang++-3.6' |
| elif builder_dict.get('compiler') == 'Clang': |
| env['CC'] = '/usr/bin/clang' |
| env['CXX'] = '/usr/bin/clang++' |
| |
| # SKNX_NO_SIMD, SK_USE_DISCARDABLE_SCALEDIMAGECACHE, etc. |
| extra_config = builder_dict.get('extra_config', '') |
| if extra_config.startswith('SK') and extra_config.isupper(): |
| env['CPPFLAGS'] = '-D' + extra_config |
| |
| return env |
| |
| |
| cov_skip.extend([lineno(), lineno() + 1]) |
| def build_targets_from_builder_dict(builder_dict, do_test_steps, do_perf_steps): |
| """Return a list of targets to build, depending on the builder type.""" |
| if builder_dict['role'] in ('Test', 'Perf') and builder_dict['os'] == 'iOS': |
| return ['iOSShell'] |
| if builder_dict.get('extra_config') == 'Appurify': |
| return ['VisualBenchTest_APK'] |
| t = [] |
| if do_test_steps: |
| t.append('dm') |
| if do_perf_steps and builder_dict.get('extra_config') == 'VisualBench': |
| t.append('visualbench') |
| elif do_perf_steps: |
| t.append('nanobench') |
| if t: |
| return t |
| else: |
| return ['most'] |
| |
| |
| cov_skip.extend([lineno(), lineno() + 1]) |
| def device_cfg(builder_dict): |
| # Android. |
| if 'Android' in builder_dict.get('extra_config', ''): |
| if 'NoNeon' in builder_dict['extra_config']: |
| return 'arm_v7' |
| return { |
| 'Arm64': 'arm64', |
| 'x86': 'x86', |
| 'x86_64': 'x86_64', |
| 'Mips': 'mips', |
| 'Mips64': 'mips64', |
| 'MipsDSP2': 'mips_dsp2', |
| }.get(builder_dict['target_arch'], 'arm_v7_neon') |
| elif builder_dict.get('os') == 'Android': |
| return { |
| 'AndroidOne': 'arm_v7_neon', |
| 'GalaxyS3': 'arm_v7_neon', |
| 'GalaxyS4': 'arm_v7_neon', |
| 'NVIDIA_Shield': 'arm64', |
| 'Nexus10': 'arm_v7_neon', |
| 'Nexus5': 'arm_v7_neon', |
| 'Nexus6': 'arm_v7_neon', |
| 'Nexus7': 'arm_v7_neon', |
| 'Nexus9': 'arm64', |
| 'NexusPlayer': 'x86', |
| }[builder_dict['model']] |
| |
| # ChromeOS. |
| if 'CrOS' in builder_dict.get('extra_config', ''): |
| if 'Link' in builder_dict['extra_config']: |
| return 'link' |
| if 'Daisy' in builder_dict['extra_config']: |
| return 'daisy' |
| elif builder_dict.get('os') == 'ChromeOS': |
| return { |
| 'Link': 'link', |
| 'Daisy': 'daisy', |
| }[builder_dict['model']] |
| |
| return None |
| |
| |
| cov_skip.extend([lineno(), lineno() + 1]) |
| def get_builder_spec(builder_name): |
| builder_dict = builder_name_schema.DictForBuilderName(builder_name) |
| env = get_extra_env_vars(builder_dict) |
| gyp_defs = gyp_defines(builder_dict) |
| gyp_defs_list = ['%s=%s' % (k, v) for k, v in gyp_defs.iteritems()] |
| gyp_defs_list.sort() |
| env['GYP_DEFINES'] = ' '.join(gyp_defs_list) |
| rv = { |
| 'builder_cfg': builder_dict, |
| 'dm_flags': dm_flags.get_args(builder_name), |
| 'env': env, |
| 'nanobench_flags': nanobench_flags.get_args(builder_name), |
| } |
| device = device_cfg(builder_dict) |
| if device: |
| rv['device_cfg'] = device |
| |
| role = builder_dict['role'] |
| if role == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER: |
| configuration = CONFIG_RELEASE |
| else: |
| configuration = builder_dict.get( |
| 'configuration', CONFIG_DEBUG) |
| arch = (builder_dict.get('arch') or builder_dict.get('target_arch')) |
| if ('Win' in builder_dict.get('os', '') and arch == 'x86_64'): |
| configuration += '_x64' |
| rv['configuration'] = configuration |
| rv['do_test_steps'] = role == builder_name_schema.BUILDER_ROLE_TEST |
| rv['do_perf_steps'] = (role == builder_name_schema.BUILDER_ROLE_PERF or |
| (role == builder_name_schema.BUILDER_ROLE_TEST and |
| configuration == CONFIG_DEBUG)) |
| if 'Valgrind' in builder_name: |
| rv['do_perf_steps'] = True |
| if 'GalaxyS4' in builder_name: |
| rv['do_perf_steps'] = False |
| |
| rv['build_targets'] = build_targets_from_builder_dict( |
| builder_dict, rv['do_test_steps'], rv['do_perf_steps']) |
| |
| # Do we upload perf results? |
| upload_perf_results = False |
| if role == builder_name_schema.BUILDER_ROLE_PERF: |
| upload_perf_results = True |
| rv['upload_perf_results'] = upload_perf_results |
| |
| # Do we upload correctness results? |
| skip_upload_bots = [ |
| 'ASAN', |
| 'Coverage', |
| 'TSAN', |
| 'UBSAN', |
| 'Valgrind', |
| ] |
| upload_dm_results = True |
| for s in skip_upload_bots: |
| if s in builder_name: |
| upload_dm_results = False |
| break |
| rv['upload_dm_results'] = upload_dm_results |
| |
| return rv |
| |
| |
| cov_end = lineno() # Don't care about code coverage past here. |
| |
| |
| def self_test(): |
| import coverage # This way the bots don't need coverage.py to be installed. |
| args = {} |
| cases = [ |
| 'Build-Mac10.8-Clang-Arm7-Debug-Android', |
| 'Build-Win-MSVC-x86-Debug', |
| 'Build-Win-MSVC-x86-Debug-GDI', |
| 'Build-Win-MSVC-x86-Debug-Exceptions', |
| 'Build-Ubuntu-GCC-Arm7-Debug-Android_FrameworkDefs', |
| 'Build-Ubuntu-GCC-Arm7-Debug-Android_NoNeon', |
| 'Build-Ubuntu-GCC-Arm7-Debug-CrOS_Daisy', |
| 'Build-Ubuntu-GCC-x86_64-Debug-CrOS_Link', |
| 'Build-Ubuntu-GCC-x86_64-Release-Mesa', |
| 'Build-Ubuntu-GCC-x86_64-Release-ANGLE', |
| 'Housekeeper-PerCommit', |
| 'Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot', |
| 'Perf-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Release-VisualBench', |
| 'Test-Android-GCC-GalaxyS4-GPU-SGX544-Arm7-Debug', |
| 'Perf-Android-GCC-Nexus5-GPU-Adreno330-Arm7-Release-Appurify', |
| 'Test-Android-GCC-Nexus6-GPU-Adreno420-Arm7-Debug', |
| 'Test-ChromeOS-GCC-Link-CPU-AVX-x86_64-Debug', |
| 'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug', |
| 'Test-Mac10.8-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release', |
| 'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage', |
| ('Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-' |
| 'SK_USE_DISCARDABLE_SCALEDIMAGECACHE'), |
| 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD', |
| 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared', |
| 'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind', |
| 'Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86-Release-ANGLE', |
| 'Test-Win8-MSVC-ShuttleA-CPU-AVX-x86_64-Debug', |
| ] |
| |
| cov = coverage.coverage() |
| cov.start() |
| for case in cases: |
| args[case] = get_builder_spec(case) |
| cov.stop() |
| |
| this_file = os.path.basename(__file__) |
| _, _, not_run, _ = cov.analysis(this_file) |
| filtered = [line for line in not_run if |
| line > cov_start and line < cov_end and line not in cov_skip] |
| if filtered: |
| print 'Lines not covered by test cases: ', filtered |
| sys.exit(1) |
| |
| golden = this_file.replace('.py', '.json') |
| with open(os.path.join(os.path.dirname(__file__), golden), 'w') as f: |
| json.dump(args, f, indent=2, sort_keys=True) |
| |
| |
| if __name__ == '__main__': |
| if len(sys.argv) == 2 and sys.argv[1] == 'test': |
| self_test() |
| sys.exit(0) |
| |
| if len(sys.argv) != 3: |
| print usage |
| sys.exit(1) |
| |
| with open(sys.argv[1], 'w') as out: |
| json.dump(get_builder_spec(sys.argv[2]), out) |