Add "skp" asset
Don't get rid of SKP_VERSION and associated stuff just yet.
BUG=skia:5427
GOLD_TRYBOT_URL= https://gold.skia.org/search?issue=2167763002
DOCS_PREVIEW= https://skia.org/?cl=2167763002
Review-Url: https://codereview.chromium.org/2167763002
diff --git a/infra/bots/assets/skp/common.py b/infra/bots/assets/skp/common.py
new file mode 100755
index 0000000..4920c9b
--- /dev/null
+++ b/infra/bots/assets/skp/common.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Common vars used by scripts in this directory."""
+
+
+import os
+import sys
+
+FILE_DIR = os.path.dirname(os.path.abspath(__file__))
+INFRA_BOTS_DIR = os.path.realpath(os.path.join(FILE_DIR, os.pardir, os.pardir))
+
+sys.path.insert(0, INFRA_BOTS_DIR)
+from assets import assets
+
+ASSET_NAME = os.path.basename(FILE_DIR)
+
+
+def run(cmd):
+ """Run a command, eg. "upload" or "download". """
+ assets.main([cmd, ASSET_NAME] + sys.argv[1:])
diff --git a/infra/bots/assets/skp/create.py b/infra/bots/assets/skp/create.py
new file mode 100755
index 0000000..68df7f0
--- /dev/null
+++ b/infra/bots/assets/skp/create.py
@@ -0,0 +1,83 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Create the SKP asset."""
+
+
+import argparse
+import common
+import os
+import shutil
+import subprocess
+import utils
+
+
+SKIA_TOOLS = os.path.join(common.INFRA_BOTS_DIR, os.pardir, os.pardir, 'tools')
+
+
+def create_asset(chrome_src_path, browser_executable, target_dir):
+ """Create the asset."""
+ browser_executable = os.path.realpath(browser_executable)
+ chrome_src_path = os.path.realpath(chrome_src_path)
+ target_dir = os.path.realpath(target_dir)
+
+ if not os.path.exists(target_dir):
+ os.makedirs(target_dir)
+
+ with utils.tmp_dir():
+ if os.environ.get('CHROME_HEADLESS'):
+ # Start Xvfb if running on a bot.
+ try:
+ subprocess.Popen(['sudo', 'Xvfb', ':0', '-screen', '0', '1280x1024x24'])
+ except Exception:
+ # It is ok if the above command fails, it just means that DISPLAY=:0
+ # is already up.
+ pass
+
+ webpages_playback_cmd = [
+ 'python', os.path.join(SKIA_TOOLS, 'skp', 'webpages_playback.py'),
+ '--page_sets', 'all',
+ '--browser_executable', browser_executable,
+ '--non-interactive',
+ '--output_dir', os.getcwd(),
+ '--chrome_src_path', chrome_src_path,
+ ]
+ try:
+ subprocess.check_call(webpages_playback_cmd)
+ finally:
+ # Clean up any leftover browser instances. This can happen if there are
+ # telemetry crashes, processes are not always cleaned up appropriately by
+ # the webpagereplay and telemetry frameworks.
+ procs = subprocess.check_output(['ps', 'ax'])
+ for line in procs.splitlines():
+ if browser_executable in line:
+ pid = line.strip().split(' ')[0]
+ if pid != str(os.getpid()) and not 'python' in line:
+ try:
+ subprocess.check_call(['kill', '-9', pid])
+ except subprocess.CalledProcessError as e:
+ print e
+ else:
+ print 'Refusing to kill self.'
+ src = os.path.join(os.getcwd(), 'playback', 'skps')
+ for f in os.listdir(src):
+ if f.endswith('.skp'):
+ shutil.copyfile(os.path.join(src, f), os.path.join(target_dir, f))
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--target_dir', '-t', required=True)
+ parser.add_argument('--chrome_src_path', '-c', required=True)
+ parser.add_argument('--browser_executable', '-e', required=True)
+ args = parser.parse_args()
+ create_asset(args.chrome_src_path, args.browser_executable, args.target_dir)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/infra/bots/assets/skp/create_and_upload.py b/infra/bots/assets/skp/create_and_upload.py
new file mode 100755
index 0000000..6567f80
--- /dev/null
+++ b/infra/bots/assets/skp/create_and_upload.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Create the asset and upload it."""
+
+
+import argparse
+import common
+import os
+import subprocess
+import sys
+import utils
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--chrome_src_path', '-c', required=True)
+ parser.add_argument('--browser_executable', '-e', required=True)
+ parser.add_argument('--gsutil')
+ args = parser.parse_args()
+
+ with utils.tmp_dir():
+ cwd = os.getcwd()
+ create_script = os.path.join(common.FILE_DIR, 'create.py')
+ upload_script = os.path.join(common.FILE_DIR, 'upload.py')
+
+ try:
+ subprocess.check_call(['python', create_script, '--target_dir', cwd,
+ '--chrome_src_path', args.chrome_src_path,
+ '--browser_executable', args.browser_executable])
+ cmd = ['python', upload_script, '-t', cwd]
+ if args.gsutil:
+ cmd.extend(['--gsutil', args.gsutil])
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError:
+ # Trap exceptions to avoid printing two stacktraces.
+ sys.exit(1)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/infra/bots/assets/skp/download.py b/infra/bots/assets/skp/download.py
new file mode 100755
index 0000000..96cc87d
--- /dev/null
+++ b/infra/bots/assets/skp/download.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Download the current version of the asset."""
+
+
+import common
+
+
+if __name__ == '__main__':
+ common.run('download')
diff --git a/infra/bots/assets/skp/upload.py b/infra/bots/assets/skp/upload.py
new file mode 100755
index 0000000..ba7fc8b
--- /dev/null
+++ b/infra/bots/assets/skp/upload.py
@@ -0,0 +1,16 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+"""Upload a new version of the asset."""
+
+
+import common
+
+
+if __name__ == '__main__':
+ common.run('upload')
diff --git a/infra/bots/common.py b/infra/bots/common.py
deleted file mode 100644
index 3c4a85d..0000000
--- a/infra/bots/common.py
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import os
-import shutil
-import subprocess
-
-
-GS_GM_BUCKET = 'chromium-skia-gm'
-
-GS_SUBDIR_TMPL_SKP = 'playback_%s/skps'
-
-VERSION_FILE_SKP = 'SKP_VERSION'
-
-
-def download_dir(skia_dir, tmp_dir, version_file, gs_path_tmpl, dst_dir):
- # Ensure that the tmp_dir exists.
- if not os.path.isdir(tmp_dir):
- os.makedirs(tmp_dir)
-
- # Get the expected version.
- with open(os.path.join(skia_dir, version_file)) as f:
- expected_version = f.read().rstrip()
-
- print 'Expected %s = %s' % (version_file, expected_version)
-
- # Get the actually-downloaded version, if we have one.
- actual_version_file = os.path.join(tmp_dir, version_file)
- try:
- with open(actual_version_file) as f:
- actual_version = f.read().rstrip()
- except IOError:
- actual_version = -1
-
- print 'Actual %s = %s' % (version_file, actual_version)
-
- # If we don't have the desired version, download it.
- if actual_version != expected_version:
- if actual_version != -1:
- os.remove(actual_version_file)
- if os.path.isdir(dst_dir):
- shutil.rmtree(dst_dir)
- os.makedirs(dst_dir)
- gs_path = 'gs://%s/%s/*' % (GS_GM_BUCKET, gs_path_tmpl % expected_version)
- print 'Downloading from %s' % gs_path
- subprocess.check_call(['gsutil', 'cp', '-R', gs_path, dst_dir])
- with open(actual_version_file, 'w') as f:
- f.write(expected_version)
diff --git a/infra/bots/download_skps.py b/infra/bots/download_skps.py
deleted file mode 100644
index 45b5de8..0000000
--- a/infra/bots/download_skps.py
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright 2016 Google Inc.
-#
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-import common
-import os
-import sys
-
-
-def main():
- if len(sys.argv) != 1:
- print >> sys.stderr, 'Usage: download_skps.py'
- sys.exit(1)
- skia_dir = os.path.abspath(os.path.join(
- os.path.dirname(os.path.realpath(__file__)),
- os.pardir, os.pardir))
- dst_dir = os.path.join(skia_dir, os.pardir, 'skps')
- tmp_dir = os.path.join(skia_dir, os.pardir, 'tmp')
- common.download_dir(skia_dir, tmp_dir, common.VERSION_FILE_SKP,
- common.GS_SUBDIR_TMPL_SKP, dst_dir)
-
-
-if __name__ == '__main__':
- main()
diff --git a/tools/skp/recreate_skps.py b/tools/skp/recreate_skps.py
deleted file mode 100644
index b7ca07a..0000000
--- a/tools/skp/recreate_skps.py
+++ /dev/null
@@ -1,96 +0,0 @@
-#!/usr/bin/env python
-# Copyright (c) 2014 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-
-"""Run the webpages_playback automation script."""
-
-
-import os
-import subprocess
-import sys
-
-sys.path.insert(0, os.getcwd())
-
-from common.py.utils import gs_utils
-from common.py.utils import shell_utils
-
-
-SKP_VERSION_FILE = 'SKP_VERSION'
-
-
-def _get_skp_version():
- """Find an unused SKP version."""
- current_skp_version = None
- with open(SKP_VERSION_FILE) as f:
- current_skp_version = int(f.read().rstrip())
-
- # Find the first SKP version which has no uploaded SKPs.
- new_version = current_skp_version + 1
- while True:
- gs_path = 'playback_%d/skps' % new_version
- if not gs_utils.GSUtils().does_storage_object_exist('chromium-skia-gm',
- gs_path):
- return new_version
- new_version += 1
-
-
-def main(chrome_src_path, browser_executable, dry_run):
- browser_executable = os.path.realpath(browser_executable)
- dry_run = (dry_run == 'True')
- skp_version = _get_skp_version()
- print 'SKP_VERSION=%d' % skp_version
-
- if os.environ.get('CHROME_HEADLESS'):
- # Start Xvfb if running on a bot.
- try:
- shell_utils.run('sudo Xvfb :0 -screen 0 1280x1024x24 &', shell=True)
- except Exception:
- # It is ok if the above command fails, it just means that DISPLAY=:0
- # is already up.
- pass
-
- upload_dir = 'playback_%d' % skp_version
- webpages_playback_cmd = [
- 'python', os.path.join(os.path.dirname(os.path.realpath(__file__)),
- 'webpages_playback.py'),
- '--page_sets', 'all',
- '--browser_executable', browser_executable,
- '--non-interactive',
- '--upload',
- '--alternate_upload_dir', upload_dir,
- '--chrome_src_path', chrome_src_path,
- ]
- if not dry_run:
- webpages_playback_cmd.append('--upload_to_partner_bucket')
-
- try:
- shell_utils.run(webpages_playback_cmd)
- finally:
- # Clean up any leftover browser instances. This can happen if there are
- # telemetry crashes, processes are not always cleaned up appropriately by
- # the webpagereplay and telemetry frameworks.
- procs = subprocess.check_output(['ps', 'ax'])
- for line in procs.splitlines():
- if browser_executable in line:
- pid = line.strip().split(' ')[0]
- if pid != str(os.getpid()) and not 'python' in line:
- try:
- shell_utils.run(['kill', '-9', pid])
- except shell_utils.CommandFailedException as e:
- print e
- else:
- print 'Refusing to kill self.'
-
- print 'writing %s: %s' % (SKP_VERSION_FILE, skp_version)
- with open(SKP_VERSION_FILE, 'w') as f:
- f.write(str(skp_version))
-
-
-if '__main__' == __name__:
- if len(sys.argv) != 4:
- print >> sys.stderr, ('USAGE: %s <chrome src path> <browser executable> '
- '<dry run>')
- sys.exit(1)
- main(*sys.argv[1:])
diff --git a/tools/skp/webpages_playback.py b/tools/skp/webpages_playback.py
index d5f45d6..8e78e0d 100644
--- a/tools/skp/webpages_playback.py
+++ b/tools/skp/webpages_playback.py
@@ -63,7 +63,10 @@
import time
import traceback
-sys.path.insert(0, os.getcwd())
+SKIA_DIR = os.path.abspath(os.path.join(
+ os.path.realpath(os.path.dirname(__file__)),
+ os.pardir, os.pardir))
+sys.path.insert(0, SKIA_DIR)
from common.py.utils import gs_utils
from common.py.utils import shell_utils