Merge V8 5.4.500.40
Test: Manual - built & ran d8
Change-Id: I4edfa2853d3e565b729723645395688ece3193f4
diff --git a/tools/swarming_client/example/1_isolate_server.py b/tools/swarming_client/example/1_isolate_server.py
new file mode 100755
index 0000000..ca53e2f
--- /dev/null
+++ b/tools/swarming_client/example/1_isolate_server.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# Copyright 2012 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+"""Runs hello_world.py, through hello_world.isolated, locally in a temporary
+directory.
+
+The files are archived and fetched from the remote Isolate Server.
+"""
+
+import hashlib
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+# Pylint can't find common.py that's in the same directory as this file.
+# pylint: disable=F0401
+import common
+
+
+def main():
+ options = common.parse_args(use_isolate_server=True, use_swarming=False)
+ tempdir = tempfile.mkdtemp(prefix=u'hello_world')
+ try:
+ # All the files are put in a temporary directory. This is optional and
+ # simply done so the current directory doesn't have the following files
+ # created:
+ # - hello_world.isolated
+ # - hello_world.isolated.state
+ # - cache/
+ cachedir = os.path.join(tempdir, 'cache')
+ isolateddir = os.path.join(tempdir, 'isolated')
+ isolated = os.path.join(isolateddir, 'hello_world.isolated')
+
+ os.mkdir(isolateddir)
+
+ common.note('Archiving to %s' % options.isolate_server)
+ # TODO(maruel): Parse the output from run() to get 'isolated_sha1'.
+ common.run(
+ [
+ 'isolate.py',
+ 'archive',
+ '--isolate', os.path.join('payload', 'hello_world.isolate'),
+ '--isolated', isolated,
+ '--isolate-server', options.isolate_server,
+ '--config-variable', 'OS', 'Yours',
+ ], options.verbose)
+
+ common.note(
+ 'Downloading from %s and running in a temporary directory' %
+ options.isolate_server)
+ with open(isolated, 'rb') as f:
+ isolated_sha1 = hashlib.sha1(f.read()).hexdigest()
+ common.run(
+ [
+ 'run_isolated.py',
+ '--cache', cachedir,
+ '--isolate-server', options.isolate_server,
+ '--isolated', isolated_sha1,
+ '--no-log',
+ ], options.verbose)
+ return 0
+ except subprocess.CalledProcessError as e:
+ return e.returncode
+ finally:
+ shutil.rmtree(tempdir)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/tools/swarming_client/example/2_swarming_run.py b/tools/swarming_client/example/2_swarming_run.py
new file mode 100755
index 0000000..64f7ccc
--- /dev/null
+++ b/tools/swarming_client/example/2_swarming_run.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+# Copyright 2012 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+"""Runs hello_world.py, through hello_world.isolate, remotely on a Swarming
+bot.
+
+It first 'compiles' hello_world.isolate into hello_word.isolated, then requests
+via swarming.py to archives, run and collect results for this task.
+
+It generates example_result.json as a task summary.
+"""
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+# Pylint can't find common.py that's in the same directory as this file.
+# pylint: disable=F0401
+import common
+
+
+def main():
+ options = common.parse_args(use_isolate_server=True, use_swarming=True)
+ tempdir = tempfile.mkdtemp(prefix=u'hello_world')
+ try:
+ isolated, _ = common.isolate(
+ tempdir, options.isolate_server, options.swarming_os, options.verbose)
+ common.note(
+ 'Running the job remotely. This:\n'
+ ' - archives to %s\n'
+ ' - runs and collect results via %s' %
+ (options.isolate_server, options.swarming))
+ cmd = [
+ 'swarming.py',
+ 'run',
+ '--swarming', options.swarming,
+ '--isolate-server', options.isolate_server,
+ '--dimension', 'os', options.swarming_os,
+ '--dimension', 'pool', 'default',
+ '--task-name', options.task_name,
+ '--task-summary-json', 'example_result.json',
+ '--decorate',
+ isolated,
+ ]
+ if options.idempotent:
+ cmd.append('--idempotent')
+ if options.priority is not None:
+ cmd.extend(('--priority', str(options.priority)))
+ common.run(cmd, options.verbose)
+ with open('example_result.json', 'rb') as f:
+ print('example_result.json content:')
+ print(f.read())
+ return 0
+ except subprocess.CalledProcessError as e:
+ return e.returncode
+ finally:
+ shutil.rmtree(tempdir)
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/tools/swarming_client/example/3_swarming_trigger_collect.py b/tools/swarming_client/example/3_swarming_trigger_collect.py
new file mode 100755
index 0000000..df0852c
--- /dev/null
+++ b/tools/swarming_client/example/3_swarming_trigger_collect.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# Copyright 2012 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+"""Runs hello_world.py, through hello_world.isolate, remotely on a Swarming
+slave.
+
+It compiles and archives via 'isolate.py archive', then discard the local files.
+After, it triggers and finally collects the results.
+
+Creates 2 shards and instructs the script to produce a file in the output
+directory.
+"""
+
+import os
+import shutil
+import subprocess
+import sys
+import tempfile
+
+# Pylint can't find common.py that's in the same directory as this file.
+# pylint: disable=F0401
+import common
+
+
+def main():
+ options = common.parse_args(use_isolate_server=True, use_swarming=True)
+ try:
+ tempdir = tempfile.mkdtemp(prefix=u'hello_world')
+ try:
+ _, hashval = common.isolate(
+ tempdir, options.isolate_server, options.swarming_os, options.verbose)
+
+ json_file = os.path.join(tempdir, 'task.json')
+ common.note('Running on %s' % options.swarming)
+ cmd = [
+ 'swarming.py',
+ 'trigger',
+ '--swarming', options.swarming,
+ '--isolate-server', options.isolate_server,
+ '--dimension', 'os', options.swarming_os,
+ '--dimension', 'pool', 'default',
+ '--task-name', options.task_name,
+ '--dump-json', json_file,
+ '--isolated', hashval,
+ '--shards', '2',
+ ]
+ if options.idempotent:
+ cmd.append('--idempotent')
+ if options.priority is not None:
+ cmd.extend(('--priority', str(options.priority)))
+ cmd.extend(('--', '${ISOLATED_OUTDIR}'))
+ common.run(cmd, options.verbose)
+
+ common.note('Getting results from %s' % options.swarming)
+ common.run(
+ [
+ 'swarming.py',
+ 'collect',
+ '--swarming', options.swarming,
+ '--json', json_file,
+ '--task-output-dir', 'example_result',
+ ], options.verbose)
+ for root, _, files in os.walk('example_result'):
+ for name in files:
+ p = os.path.join(root, name)
+ with open(p, 'rb') as f:
+ print('%s content:' % p)
+ print(f.read())
+ return 0
+ finally:
+ shutil.rmtree(tempdir)
+ except subprocess.CalledProcessError as e:
+ return e.returncode
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/tools/swarming_client/example/4_swarming_run_manual_upload.py b/tools/swarming_client/example/4_swarming_run_manual_upload.py
new file mode 100755
index 0000000..6758ae5
--- /dev/null
+++ b/tools/swarming_client/example/4_swarming_run_manual_upload.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python
+# Copyright 2012 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+"""Runs hello_world.py through a manually crafted hello_world.isolated, remotely
+on a Swarming slave.
+
+No .isolate file is involved at all.
+
+It creates hello_world.isolated and archives via 'isolateserver.py archive',
+then trigger and finally collect the results.
+
+It never create a local file.
+"""
+
+import json
+import os
+import subprocess
+import sys
+import tempfile
+
+# Pylint can't find common.py that's in the same directory as this file.
+# pylint: disable=F0401
+import common
+
+
+def main():
+ options = common.parse_args(use_isolate_server=True, use_swarming=True)
+ try:
+ common.note(
+ 'Archiving directory \'payload\' to %s' % options.isolate_server)
+ payload_isolated_sha1 = common.capture(
+ [
+ 'isolateserver.py',
+ 'archive',
+ '--isolate-server', options.isolate_server,
+ 'payload',
+ ]).split()[0]
+
+ common.note(
+ 'Archiving custom .isolated file to %s' % options.isolate_server)
+ handle, isolated = tempfile.mkstemp(
+ prefix=u'hello_world', suffix=u'.isolated')
+ os.close(handle)
+ try:
+ data = {
+ 'algo': 'sha-1',
+ 'command': ['python', 'hello_world.py', 'Custom'],
+ 'includes': [payload_isolated_sha1],
+ 'version': '1.0',
+ }
+ with open(isolated, 'wb') as f:
+ json.dump(data, f, sort_keys=True, separators=(',',':'))
+ isolated_sha1 = common.capture(
+ [
+ 'isolateserver.py',
+ 'archive',
+ '--isolate-server', options.isolate_server,
+ isolated,
+ ]).split()[0]
+ finally:
+ common.note('Deleting temporary file, it is not necessary anymore.')
+ os.remove(isolated)
+
+ # Now trigger as usual. You could look at run_exmaple_swarming_involved for
+ # the involved way but use the short way here.
+
+ common.note('Running %s on %s' % (isolated_sha1, options.swarming))
+ cmd = [
+ 'swarming.py',
+ 'run',
+ '--swarming', options.swarming,
+ '--isolate-server', options.isolate_server,
+ '--dimension', 'os', options.swarming_os,
+ '--dimension', 'pool', 'default',
+ '--task-name', options.task_name,
+ isolated_sha1,
+ ]
+ if options.idempotent:
+ cmd.append('--idempotent')
+ if options.priority is not None:
+ cmd.extend(('--priority', str(options.priority)))
+ common.run(cmd, options.verbose)
+ return 0
+ except subprocess.CalledProcessError as e:
+ return e.returncode
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/tools/swarming_client/example/README.md b/tools/swarming_client/example/README.md
new file mode 100644
index 0000000..eb282c8
--- /dev/null
+++ b/tools/swarming_client/example/README.md
@@ -0,0 +1,12 @@
+# Swarming examples
+
+This directory contains examples how to use this toolset to run tasks remotely.
+
+You should look at the code and try these in the following order:
+
+* 1_isolate_server.py
+* 2_swarming_run.py
+* 3_swarming_trigger_collect.py
+* 4_swarming_run_manual_upload.py
+
+Look at the docstring of each script to see what they do.
diff --git a/tools/swarming_client/example/common.py b/tools/swarming_client/example/common.py
new file mode 100644
index 0000000..07c0bfc
--- /dev/null
+++ b/tools/swarming_client/example/common.py
@@ -0,0 +1,126 @@
+# Copyright 2012 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+import datetime
+import getpass
+import hashlib
+import optparse
+import os
+import subprocess
+import sys
+
+
+ROOT_DIR = os.path.dirname(os.path.abspath(
+ __file__.decode(sys.getfilesystemencoding())))
+sys.path.append(os.path.join(ROOT_DIR, '..', 'third_party'))
+
+import colorama
+
+
+CHROMIUM_SWARMING_OSES = {
+ 'darwin': 'Mac',
+ 'cygwin': 'Windows',
+ 'linux2': 'Ubuntu',
+ 'win32': 'Windows',
+}
+
+
+def parse_args(use_isolate_server, use_swarming):
+ """Process arguments for the example scripts."""
+ os.chdir(ROOT_DIR)
+ colorama.init()
+
+ parser = optparse.OptionParser(description=sys.modules['__main__'].__doc__)
+ if use_isolate_server:
+ parser.add_option(
+ '-I', '--isolate-server',
+ metavar='URL', default=os.environ.get('ISOLATE_SERVER', ''),
+ help='Isolate server to use')
+ if use_swarming:
+ task_name = '%s-%s-hello_world' % (
+ getpass.getuser(),
+ datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S'))
+ parser.add_option(
+ '--idempotent', action='store_true',
+ help='Tells Swarming to reused previous task result if possible')
+ parser.add_option(
+ '-S', '--swarming',
+ metavar='URL', default=os.environ.get('SWARMING_SERVER', ''),
+ help='Swarming server to use')
+ parser.add_option(
+ '-o', '--os', default=sys.platform,
+ help='Swarming slave OS to request. Should be one of the valid '
+ 'sys.platform values like darwin, linux2 or win32 default: '
+ '%default.')
+ parser.add_option(
+ '-t', '--task-name', default=task_name,
+ help='Swarming task name, default is based on time: %default')
+ parser.add_option('-v', '--verbose', action='count', default=0)
+ parser.add_option(
+ '--priority', metavar='INT', type='int', help='Priority to use')
+ options, args = parser.parse_args()
+
+ if args:
+ parser.error('Unsupported argument %s' % args)
+ if use_isolate_server and not options.isolate_server:
+ parser.error('--isolate-server is required.')
+ if use_swarming:
+ if not options.swarming:
+ parser.error('--swarming is required.')
+ options.swarming_os = CHROMIUM_SWARMING_OSES[options.os]
+ del options.os
+
+ return options
+
+
+def note(text):
+ """Prints a formatted note."""
+ print(
+ colorama.Fore.YELLOW + colorama.Style.BRIGHT + '\n-> ' + text +
+ colorama.Fore.RESET)
+
+
+def run(cmd, verbose):
+ """Prints the command it runs then run it."""
+ cmd = cmd[:]
+ cmd.extend(['--verbose'] * verbose)
+ print(
+ 'Running: %s%s%s' %
+ (colorama.Fore.GREEN, ' '.join(cmd), colorama.Fore.RESET))
+ cmd = [sys.executable, os.path.join('..', cmd[0])] + cmd[1:]
+ if sys.platform != 'win32':
+ cmd = ['time', '-p'] + cmd
+ subprocess.check_call(cmd)
+
+
+def capture(cmd):
+ """Prints the command it runs then return stdout."""
+ print(
+ 'Running: %s%s%s' %
+ (colorama.Fore.GREEN, ' '.join(cmd), colorama.Fore.RESET))
+ cmd = [sys.executable, os.path.join('..', cmd[0])] + cmd[1:]
+ return subprocess.check_output(cmd)
+
+
+def isolate(tempdir, isolate_server, swarming_os, verbose):
+ """Archives the payload."""
+ # All the files are put in a temporary directory. This is optional and
+ # simply done so the current directory doesn't have the following files
+ # created:
+ # - hello_world.isolated
+ # - hello_world.isolated.state
+ isolated = os.path.join(tempdir, 'hello_world.isolated')
+ note('Archiving to %s' % isolate_server)
+ run(
+ [
+ 'isolate.py',
+ 'archive',
+ '--isolate', os.path.join('payload', 'hello_world.isolate'),
+ '--isolated', isolated,
+ '--isolate-server', isolate_server,
+ '--config-variable', 'OS', swarming_os,
+ ], verbose)
+ with open(isolated, 'rb') as f:
+ hashval = hashlib.sha1(f.read()).hexdigest()
+ return isolated, hashval
diff --git a/tools/swarming_client/example/payload/hello_world.isolate b/tools/swarming_client/example/payload/hello_world.isolate
new file mode 100644
index 0000000..d6e5373
--- /dev/null
+++ b/tools/swarming_client/example/payload/hello_world.isolate
@@ -0,0 +1,49 @@
+# Copyright 2012 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+{
+ 'variables': {
+ 'files': [
+ 'hello_world.py',
+ ],
+ },
+ #
+ 'conditions': [
+ ['OS=="Ubuntu"', {
+ 'variables': {
+ 'command': [
+ 'python',
+ 'hello_world.py',
+ 'Ubuntu',
+ ],
+ },
+ }],
+ ['OS=="Mac"', {
+ 'variables': {
+ 'command': [
+ 'python',
+ 'hello_world.py',
+ 'OSX',
+ ],
+ },
+ }],
+ ['OS=="Yours"', {
+ 'variables': {
+ 'command': [
+ 'python',
+ 'hello_world.py',
+ 'your OS is the best',
+ ],
+ },
+ }],
+ ['OS=="Windows"', {
+ 'variables': {
+ 'command': [
+ 'python',
+ 'hello_world.py',
+ 'Windows',
+ ],
+ },
+ }],
+ ],
+}
diff --git a/tools/swarming_client/example/payload/hello_world.py b/tools/swarming_client/example/payload/hello_world.py
new file mode 100755
index 0000000..a57ba5d
--- /dev/null
+++ b/tools/swarming_client/example/payload/hello_world.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+# Copyright 2012 The LUCI Authors. All rights reserved.
+# Use of this source code is governed under the Apache License, Version 2.0
+# that can be found in the LICENSE file.
+
+"""This script is meant to be run on a Swarming slave."""
+
+import os
+import sys
+
+
+def main():
+ print('Hello world: ' + sys.argv[1])
+ if len(sys.argv) == 3:
+ # Write a file in ${ISOLATED_OUTDIR}.
+ with open(os.path.join(sys.argv[2], 'happiness.txt'), 'wb') as f:
+ f.write(
+ 'is where you look %d/%d' % (
+ int(os.environ['GTEST_SHARD_INDEX']),
+ int(os.environ['GTEST_TOTAL_SHARDS'])))
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())