Swarming bots: setup for skipping download of build products
Turns out it's pretty easy to pass the compile outputs to the test task by just adding the hash to the "includes" list in the .isolated file. So the flow is:
1. Isolate skia repo
2. Run compile task, record hash of results
3. Isolate test inputs for DM. This writes a .isolated file
4. Edit the .isolated file from #3 to include the hash from #2
5. Upload the modified .isolated file to the isolate server
6. Trigger the swarming task for DM
7. Wait for DM task to finish, download results from isolate server
8. Upload results to GS as normal
I expect the swarming bots to break when this is committed due to the moved out directory. The associated recipe change will fix them.
NOTRY=true
BUG=skia:4763
GOLD_TRYBOT_URL= https://gold.skia.org/search2?unt=true&query=source_type%3Dgm&master=false&issue=1759553003
Review URL: https://codereview.chromium.org/1759553003
diff --git a/infra/bots/add_isolated_input.py b/infra/bots/add_isolated_input.py
new file mode 100644
index 0000000..70bde6d
--- /dev/null
+++ b/infra/bots/add_isolated_input.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python
+#
+# Copyright 2016 Google Inc.
+#
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+
+import argparse
+import json
+
+
+"""Add the given hash to the includes section of the given isolated file."""
+
+
+def add_isolated_hash(isolated_file, hash_str):
+ with open(isolated_file) as f:
+ isolated = json.load(f)
+ isolated['includes'].append(hash_str)
+ with open(isolated_file, 'w') as f:
+ json.dump(isolated, f, sort_keys=True)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument('--isolated_file', required=True)
+ parser.add_argument('--hash', required=True)
+ args = parser.parse_args()
+ add_isolated_hash(args.isolated_file, args.hash)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/infra/bots/common.py b/infra/bots/common.py
index b532d5d..31f1f82 100644
--- a/infra/bots/common.py
+++ b/infra/bots/common.py
@@ -155,9 +155,9 @@
self.spec = self.get_bot_spec(bot_name)
self.bot_cfg = self.spec['builder_cfg']
if self.bot_cfg['role'] == 'Build':
- self.out_dir = swarm_out_dir
+ self.out_dir = os.path.join(swarm_out_dir, 'out')
else:
- self.out_dir = os.path.join(self.skia_dir, 'out', self.name)
+ self.out_dir = os.path.join(os.pardir, 'out')
self.configuration = self.spec['configuration']
self.default_env = {
'SKIA_OUT': self.out_dir,
diff --git a/infra/bots/compile_skia.isolate b/infra/bots/compile_skia.isolate
index 5866f4c..529ca15 100644
--- a/infra/bots/compile_skia.isolate
+++ b/infra/bots/compile_skia.isolate
@@ -4,7 +4,7 @@
],
'variables': {
'command': [
- 'python', 'compile_skia.py', '--builder_name', '<(BUILDER_NAME)', '--swarm_out_dir', '${ISOLATED_OUTDIR}/out',
+ 'python', 'compile_skia.py', '--builder_name', '<(BUILDER_NAME)', '--swarm_out_dir', '${ISOLATED_OUTDIR}',
],
},
}
diff --git a/infra/bots/infrabots.isolate b/infra/bots/infrabots.isolate
new file mode 100644
index 0000000..4936c62
--- /dev/null
+++ b/infra/bots/infrabots.isolate
@@ -0,0 +1,13 @@
+{
+ 'variables': {
+ 'files': [
+ '../../tools/__init__.py',
+ '../../tools/buildbot_spec.py',
+ '../../tools/builder_name_schema.json',
+ '../../tools/builder_name_schema.py',
+ '../../tools/dm_flags.py',
+ '../../tools/nanobench_flags.py',
+ './',
+ ],
+ },
+}
diff --git a/infra/bots/perf_skia.isolate b/infra/bots/perf_skia.isolate
index 420a0d0..dd442ba 100644
--- a/infra/bots/perf_skia.isolate
+++ b/infra/bots/perf_skia.isolate
@@ -1,7 +1,8 @@
{
'includes': [
'images.isolate',
- 'skia_repo.isolate',
+ 'infrabots.isolate',
+ 'resources.isolate',
'skps.isolate',
],
'variables': {
diff --git a/infra/bots/resources.isolate b/infra/bots/resources.isolate
new file mode 100644
index 0000000..be73552
--- /dev/null
+++ b/infra/bots/resources.isolate
@@ -0,0 +1,7 @@
+{
+ 'variables': {
+ 'files': [
+ '../../resources/',
+ ],
+ },
+}
diff --git a/infra/bots/test_skia.isolate b/infra/bots/test_skia.isolate
index d9eab8c..d5bb98c 100644
--- a/infra/bots/test_skia.isolate
+++ b/infra/bots/test_skia.isolate
@@ -1,7 +1,8 @@
{
'includes': [
'images.isolate',
- 'skia_repo.isolate',
+ 'infrabots.isolate',
+ 'resources.isolate',
'skps.isolate',
],
'variables': {