| #!/usr/bin/env python |
| # |
| # Copyright 2016 Google Inc. |
| # |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| |
| import contextlib |
| import glob |
| import math |
| import os |
| import psutil |
| import shutil |
| import socket |
| import subprocess |
| import sys |
| import time |
| import urllib2 |
| |
| from flavor import android_flavor |
| from flavor import chromeos_flavor |
| from flavor import cmake_flavor |
| from flavor import coverage_flavor |
| from flavor import default_flavor |
| from flavor import ios_flavor |
| from flavor import valgrind_flavor |
| from flavor import xsan_flavor |
| |
| |
| CONFIG_COVERAGE = 'Coverage' |
| CONFIG_DEBUG = 'Debug' |
| CONFIG_RELEASE = 'Release' |
| VALID_CONFIGS = (CONFIG_COVERAGE, CONFIG_DEBUG, CONFIG_RELEASE) |
| |
| BUILD_PRODUCTS_WHITELIST = [ |
| 'dm', |
| 'dm.exe', |
| 'nanobench', |
| 'nanobench.exe', |
| '*.so', |
| '*.dll', |
| ] |
| |
| GM_ACTUAL_FILENAME = 'actual-results.json' |
| GM_EXPECTATIONS_FILENAME = 'expected-results.json' |
| GM_IGNORE_TESTS_FILENAME = 'ignored-tests.txt' |
| |
| GOLD_UNINTERESTING_HASHES_URL = 'https://gold.skia.org/_/hashes' |
| |
| GS_GM_BUCKET = 'chromium-skia-gm' |
| GS_SUMMARIES_BUCKET = 'chromium-skia-gm-summaries' |
| |
| GS_SUBDIR_TMPL_SK_IMAGE = 'skimage/v%s' |
| GS_SUBDIR_TMPL_SKP = 'playback_%s/skps' |
| |
| SKIA_REPO = 'https://skia.googlesource.com/skia.git' |
| INFRA_REPO = 'https://skia.googlesource.com/buildbot.git' |
| |
| SERVICE_ACCOUNT_FILE = 'service-account-skia.json' |
| SERVICE_ACCOUNT_INTERNAL_FILE = 'service-account-skia-internal.json' |
| |
| VERSION_FILE_SK_IMAGE = 'SK_IMAGE_VERSION' |
| VERSION_FILE_SKP = 'SKP_VERSION' |
| |
| |
| def is_android(bot_cfg): |
| """Determine whether the given bot is an Android bot.""" |
| return ('Android' in bot_cfg.get('extra_config', '') or |
| bot_cfg.get('os') == 'Android') |
| |
| def is_chromeos(bot_cfg): |
| return ('CrOS' in bot_cfg.get('extra_config', '') or |
| bot_cfg.get('os') == 'ChromeOS') |
| |
| def is_cmake(bot_cfg): |
| return 'CMake' in bot_cfg.get('extra_config', '') |
| |
| def is_ios(bot_cfg): |
| return ('iOS' in bot_cfg.get('extra_config', '') or |
| bot_cfg.get('os') == 'iOS') |
| |
| |
| def is_valgrind(bot_cfg): |
| return 'Valgrind' in bot_cfg.get('extra_config', '') |
| |
| |
| def is_xsan(bot_cfg): |
| return (bot_cfg.get('extra_config') == 'ASAN' or |
| bot_cfg.get('extra_config') == 'MSAN' or |
| bot_cfg.get('extra_config') == 'TSAN') |
| |
| |
| def download_dir(skia_dir, tmp_dir, version_file, gs_path_tmpl, dst_dir): |
| # Ensure that the tmp_dir exists. |
| if not os.path.isdir(tmp_dir): |
| os.makedirs(tmp_dir) |
| |
| # Get the expected version. |
| with open(os.path.join(skia_dir, version_file)) as f: |
| expected_version = f.read().rstrip() |
| |
| print 'Expected %s = %s' % (version_file, expected_version) |
| |
| # Get the actually-downloaded version, if we have one. |
| actual_version_file = os.path.join(tmp_dir, version_file) |
| try: |
| with open(actual_version_file) as f: |
| actual_version = f.read().rstrip() |
| except IOError: |
| actual_version = -1 |
| |
| print 'Actual %s = %s' % (version_file, actual_version) |
| |
| # If we don't have the desired version, download it. |
| if actual_version != expected_version: |
| if actual_version != -1: |
| os.remove(actual_version_file) |
| if os.path.isdir(dst_dir): |
| shutil.rmtree(dst_dir) |
| os.makedirs(dst_dir) |
| gs_path = 'gs://%s/%s/*' % (GS_GM_BUCKET, gs_path_tmpl % expected_version) |
| print 'Downloading from %s' % gs_path |
| subprocess.check_call(['gsutil', 'cp', '-R', gs_path, dst_dir]) |
| with open(actual_version_file, 'w') as f: |
| f.write(expected_version) |
| |
| |
| def get_uninteresting_hashes(hashes_file): |
| retries = 5 |
| timeout = 60 |
| wait_base = 15 |
| |
| socket.setdefaulttimeout(timeout) |
| for retry in range(retries): |
| try: |
| with contextlib.closing( |
| urllib2.urlopen(GOLD_UNINTERESTING_HASHES_URL, timeout=timeout)) as w: |
| hashes = w.read() |
| with open(hashes_file, 'w') as f: |
| f.write(hashes) |
| break |
| except Exception as e: |
| print >> sys.stderr, 'Failed to get uninteresting hashes from %s:\n%s' % ( |
| GOLD_UNINTERESTING_HASHES_URL, e) |
| if retry == retries: |
| raise |
| waittime = wait_base * math.pow(2, retry) |
| print 'Retry in %d seconds.' % waittime |
| time.sleep(waittime) |
| |
| |
| class BotInfo(object): |
| def __init__(self, bot_name, swarm_out_dir): |
| """Initialize the bot, given its name. |
| |
| Assumes that CWD is the directory containing this file. |
| """ |
| self.name = bot_name |
| self.skia_dir = os.path.abspath(os.path.join( |
| os.path.dirname(os.path.realpath(__file__)), |
| os.pardir, os.pardir)) |
| self.swarm_out_dir = swarm_out_dir |
| os.chdir(self.skia_dir) |
| self.build_dir = os.path.abspath(os.path.join(self.skia_dir, os.pardir)) |
| self.infrabots_dir = os.path.join(self.skia_dir, 'infra', 'bots') |
| self.home_dir = os.path.expanduser('~') |
| |
| self.spec = self.get_bot_spec(bot_name) |
| self.bot_cfg = self.spec['builder_cfg'] |
| self.out_dir = os.path.join(os.pardir, 'out') |
| self.configuration = self.spec['configuration'] |
| self.default_env = { |
| 'CHROME_HEADLESS': '1', |
| 'SKIA_OUT': self.out_dir, |
| 'BUILDTYPE': self.configuration, |
| 'PATH': os.environ['PATH'], |
| } |
| if 'Win' in self.bot_cfg['os']: |
| self.default_env['SystemRoot'] = 'C:\\Windows' |
| self.default_env['TEMP'] = os.path.join( |
| self.home_dir, 'AppData', 'Local', 'Temp') |
| self.default_env['TMP'] = self.default_env['TEMP'] |
| self.default_env.update(self.spec['env']) |
| self.build_targets = [str(t) for t in self.spec['build_targets']] |
| self.is_trybot = self.bot_cfg['is_trybot'] |
| self.upload_dm_results = self.spec['upload_dm_results'] |
| self.upload_perf_results = self.spec['upload_perf_results'] |
| self.perf_data_dir = os.path.join(self.swarm_out_dir, 'perfdata', |
| self.name, 'data') |
| self.resource_dir = os.path.join(self.skia_dir, 'resources') |
| self.images_dir = os.path.join(self.build_dir, 'images') |
| self.local_skp_dir = os.path.join(self.build_dir, 'playback', 'skps') |
| self.dm_flags = self.spec['dm_flags'] |
| self.nanobench_flags = self.spec['nanobench_flags'] |
| self._ccache = None |
| self._checked_for_ccache = False |
| self._already_ran = {} |
| self.tmp_dir = os.path.join(self.build_dir, 'tmp') |
| self.flavor = self.get_flavor(self.bot_cfg) |
| |
| # These get filled in during subsequent steps. |
| self.device_dirs = None |
| self.build_number = None |
| self.got_revision = None |
| self.master_name = None |
| self.slave_name = None |
| |
| @property |
| def ccache(self): |
| if not self._checked_for_ccache: |
| self._checked_for_ccache = True |
| if sys.platform != 'win32': |
| try: |
| result = subprocess.check_output(['which', 'ccache']) |
| self._ccache = result.rstrip() |
| except subprocess.CalledProcessError: |
| pass |
| |
| return self._ccache |
| |
| def get_bot_spec(self, bot_name): |
| """Retrieve the bot spec for this bot.""" |
| sys.path.append(self.skia_dir) |
| from tools import buildbot_spec |
| return buildbot_spec.get_builder_spec(bot_name) |
| |
| def get_flavor(self, bot_cfg): |
| """Return a flavor utils object specific to the given bot.""" |
| if is_android(bot_cfg): |
| return android_flavor.AndroidFlavorUtils(self) |
| elif is_chromeos(bot_cfg): |
| return chromeos_flavor.ChromeOSFlavorUtils(self) |
| elif is_cmake(bot_cfg): |
| return cmake_flavor.CMakeFlavorUtils(self) |
| elif is_ios(bot_cfg): |
| return ios_flavor.iOSFlavorUtils(self) |
| elif is_valgrind(bot_cfg): |
| return valgrind_flavor.ValgrindFlavorUtils(self) |
| elif is_xsan(bot_cfg): |
| return xsan_flavor.XSanFlavorUtils(self) |
| elif bot_cfg.get('configuration') == CONFIG_COVERAGE: |
| return coverage_flavor.CoverageFlavorUtils(self) |
| else: |
| return default_flavor.DefaultFlavorUtils(self) |
| |
| def run(self, cmd, env=None, cwd=None): |
| _env = {} |
| _env.update(self.default_env) |
| _env.update(env or {}) |
| cwd = cwd or self.skia_dir |
| print '============' |
| print 'CMD: %s' % cmd |
| print 'CWD: %s' % cwd |
| print 'ENV: %s' % _env |
| print '============' |
| subprocess.check_call(cmd, env=_env, cwd=cwd) |
| |
| def compile_steps(self): |
| for t in self.build_targets: |
| self.flavor.compile(t) |
| dst = os.path.join(self.swarm_out_dir, 'out', self.configuration) |
| os.makedirs(dst) |
| for pattern in BUILD_PRODUCTS_WHITELIST: |
| path = os.path.join(self.out_dir, self.configuration, pattern) |
| for f in glob.glob(path): |
| print 'Copying build product %s' % f |
| shutil.copy(f, dst) |
| self.cleanup() |
| |
| def _run_once(self, fn, *args, **kwargs): |
| if not fn.__name__ in self._already_ran: |
| self._already_ran[fn.__name__] = True |
| fn(*args, **kwargs) |
| |
| def install(self): |
| """Copy the required executables and files to the device.""" |
| self.device_dirs = self.flavor.get_device_dirs() |
| |
| # Run any device-specific installation. |
| self.flavor.install() |
| |
| # TODO(borenet): Only copy files which have changed. |
| # Resources |
| self.flavor.copy_directory_contents_to_device(self.resource_dir, |
| self.device_dirs.resource_dir) |
| |
| def _key_params(self): |
| """Build a unique key from the builder name (as a list). |
| |
| E.g. arch x86 gpu GeForce320M mode MacMini4.1 os Mac10.6 |
| """ |
| # Don't bother to include role, which is always Test. |
| # TryBots are uploaded elsewhere so they can use the same key. |
| blacklist = ['role', 'is_trybot'] |
| |
| flat = [] |
| for k in sorted(self.bot_cfg.keys()): |
| if k not in blacklist: |
| flat.append(k) |
| flat.append(self.bot_cfg[k]) |
| return flat |
| |
| def test_steps(self, got_revision, master_name, slave_name, build_number, |
| issue=None, patchset=None): |
| """Run the DM test.""" |
| self.build_number = build_number |
| self.got_revision = got_revision |
| self.master_name = master_name |
| self.slave_name = slave_name |
| self._run_once(self.install) |
| |
| use_hash_file = False |
| if self.upload_dm_results: |
| # This must run before we write anything into self.device_dirs.dm_dir |
| # or we may end up deleting our output on machines where they're the same. |
| host_dm_dir = os.path.join(self.swarm_out_dir, 'dm') |
| print 'host dm dir: %s' % host_dm_dir |
| self.flavor.create_clean_host_dir(host_dm_dir) |
| if str(host_dm_dir) != str(self.device_dirs.dm_dir): |
| self.flavor.create_clean_device_dir(self.device_dirs.dm_dir) |
| |
| # Obtain the list of already-generated hashes. |
| if not os.path.isdir(self.tmp_dir): |
| os.makedirs(self.tmp_dir) |
| hash_filename = 'uninteresting_hashes.txt' |
| host_hashes_file = os.path.join(self.tmp_dir, hash_filename) |
| hashes_file = self.flavor.device_path_join( |
| self.device_dirs.tmp_dir, hash_filename) |
| |
| try: |
| get_uninteresting_hashes(host_hashes_file) |
| except Exception: |
| pass |
| |
| if os.path.exists(host_hashes_file): |
| self.flavor.copy_file_to_device(host_hashes_file, hashes_file) |
| use_hash_file = True |
| |
| # Run DM. |
| properties = [ |
| 'gitHash', self.got_revision, |
| 'master', self.master_name, |
| 'builder', self.name, |
| 'build_number', self.build_number, |
| ] |
| if self.is_trybot: |
| if not issue: |
| raise Exception('issue is required for trybots.') |
| if not patchset: |
| raise Exception('patchset is required for trybots.') |
| properties.extend([ |
| 'issue', issue, |
| 'patchset', patchset, |
| ]) |
| |
| args = [ |
| 'dm', |
| '--undefok', # This helps branches that may not know new flags. |
| '--resourcePath', self.device_dirs.resource_dir, |
| '--skps', self.device_dirs.skp_dir, |
| '--images', self.flavor.device_path_join( |
| self.device_dirs.images_dir, 'dm'), |
| '--nameByHash', |
| '--properties' |
| ] + properties |
| |
| args.append('--key') |
| args.extend(self._key_params()) |
| if use_hash_file: |
| args.extend(['--uninterestingHashesFile', hashes_file]) |
| if self.upload_dm_results: |
| args.extend(['--writePath', self.device_dirs.dm_dir]) |
| |
| skip_flag = None |
| if self.bot_cfg.get('cpu_or_gpu') == 'CPU': |
| skip_flag = '--nogpu' |
| elif self.bot_cfg.get('cpu_or_gpu') == 'GPU': |
| skip_flag = '--nocpu' |
| if skip_flag: |
| args.append(skip_flag) |
| args.extend(self.dm_flags) |
| |
| self.flavor.run(args, env=self.default_env) |
| |
| if self.upload_dm_results: |
| # Copy images and JSON to host machine if needed. |
| self.flavor.copy_directory_contents_to_host(self.device_dirs.dm_dir, |
| host_dm_dir) |
| |
| # See skia:2789. |
| if ('Valgrind' in self.name and |
| self.bot_cfg.get('cpu_or_gpu') == 'GPU'): |
| abandonGpuContext = list(args) |
| abandonGpuContext.append('--abandonGpuContext') |
| self.flavor.run(abandonGpuContext) |
| preAbandonGpuContext = list(args) |
| preAbandonGpuContext.append('--preAbandonGpuContext') |
| self.flavor.run(preAbandonGpuContext) |
| |
| self.cleanup() |
| |
| def perf_steps(self, got_revision, master_name, slave_name, build_number, |
| issue=None, patchset=None): |
| """Run Skia benchmarks.""" |
| self.build_number = build_number |
| self.got_revision = got_revision |
| self.master_name = master_name |
| self.slave_name = slave_name |
| self._run_once(self.install) |
| if self.upload_perf_results: |
| self.flavor.create_clean_device_dir(self.device_dirs.perf_data_dir) |
| |
| # Run nanobench. |
| properties = [ |
| '--properties', |
| 'gitHash', self.got_revision, |
| 'build_number', self.build_number, |
| ] |
| if self.is_trybot: |
| if not issue: |
| raise Exception('issue is required for trybots.') |
| if not patchset: |
| raise Exception('patchset is required for trybots.') |
| properties.extend([ |
| 'issue', issue, |
| 'patchset', patchset, |
| ]) |
| |
| target = 'nanobench' |
| if 'VisualBench' in self.name: |
| target = 'visualbench' |
| args = [ |
| target, |
| '--undefok', # This helps branches that may not know new flags. |
| '-i', self.device_dirs.resource_dir, |
| '--skps', self.device_dirs.skp_dir, |
| '--images', self.flavor.device_path_join( |
| self.device_dirs.images_dir, 'dm'), # Using DM images for now. |
| ] |
| |
| skip_flag = None |
| if self.bot_cfg.get('cpu_or_gpu') == 'CPU': |
| skip_flag = '--nogpu' |
| elif self.bot_cfg.get('cpu_or_gpu') == 'GPU': |
| skip_flag = '--nocpu' |
| if skip_flag: |
| args.append(skip_flag) |
| args.extend(self.nanobench_flags) |
| |
| if self.upload_perf_results: |
| json_path = self.flavor.device_path_join( |
| self.device_dirs.perf_data_dir, |
| 'nanobench_%s.json' % self.got_revision) |
| args.extend(['--outResultsFile', json_path]) |
| args.extend(properties) |
| |
| keys_blacklist = ['configuration', 'role', 'is_trybot'] |
| args.append('--key') |
| for k in sorted(self.bot_cfg.keys()): |
| if not k in keys_blacklist: |
| args.extend([k, self.bot_cfg[k]]) |
| |
| self.flavor.run(args, env=self.default_env) |
| |
| # See skia:2789. |
| if ('Valgrind' in self.name and |
| self.bot_cfg.get('cpu_or_gpu') == 'GPU'): |
| abandonGpuContext = list(args) |
| abandonGpuContext.extend(['--abandonGpuContext', '--nocpu']) |
| self.flavor.run(abandonGpuContext, env=self.default_env) |
| |
| # Copy results to host. |
| if self.upload_perf_results: |
| if not os.path.exists(self.perf_data_dir): |
| os.makedirs(self.perf_data_dir) |
| self.flavor.copy_directory_contents_to_host( |
| self.device_dirs.perf_data_dir, self.perf_data_dir) |
| |
| self.cleanup() |
| |
| def cleanup(self): |
| if sys.platform == 'win32': |
| # Kill mspdbsrv.exe, which tends to hang around after the build finishes. |
| for p in psutil.process_iter(): |
| try: |
| if p.name == 'mspdbsrv.exe': |
| p.kill() |
| except psutil._error.AccessDenied: |
| pass |
| self.flavor.cleanup_steps() |