Add Goma support in gn_flavor.
Using full paths in skia.h causes "file not found" with Goma. All other
Builds seem fine without the path, so I changed find_headers.py to use
the basename.
Change-Id: Ib520e91a92ebffe36a736eb53f643d359f5bb2ce
Reviewed-on: https://skia-review.googlesource.com/79360
Reviewed-by: Greg Daniel <egdaniel@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
Commit-Queue: Ben Wagner <benjaminwagner@google.com>
diff --git a/gn/find_headers.py b/gn/find_headers.py
index 1baca2f..2f20e1e 100755
--- a/gn/find_headers.py
+++ b/gn/find_headers.py
@@ -35,7 +35,7 @@
f.write('#ifndef skia_h_DEFINED\n')
f.write('#define skia_h_DEFINED\n')
for h in headers:
- f.write('#include "' + h + '"\n')
+ f.write('#include "' + os.path.basename(h) + '"\n')
f.write('#endif//skia_h_DEFINED\n')
with open(skia_h + '.deps', "w") as f:
diff --git a/infra/bots/jobs.json b/infra/bots/jobs.json
index e796d5b..a362715 100644
--- a/infra/bots/jobs.json
+++ b/infra/bots/jobs.json
@@ -65,6 +65,7 @@
"Build-Win-Clang-arm64-Release-Android",
"Build-Win-Clang-x86_64-Debug",
"Build-Win-Clang-x86_64-Debug-ANGLE",
+ "Build-Win-Clang-x86_64-Debug-Goma",
"Build-Win-Clang-x86_64-Debug-Vulkan",
"Build-Win-Clang-x86_64-Release",
"Build-Win-Clang-x86_64-Release-ANGLE",
diff --git a/infra/bots/recipe_modules/flavor/__init__.py b/infra/bots/recipe_modules/flavor/__init__.py
index de62850..aade7ce 100644
--- a/infra/bots/recipe_modules/flavor/__init__.py
+++ b/infra/bots/recipe_modules/flavor/__init__.py
@@ -5,6 +5,7 @@
DEPS = [
'builder_name_schema',
'depot_tools/bot_update',
+ 'depot_tools/cipd',
'env',
'recipe_engine/context',
'recipe_engine/file',
diff --git a/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json
new file mode 100644
index 0000000..cfad3a5
--- /dev/null
+++ b/infra/bots/recipe_modules/flavor/examples/full.expected/Build-Win-Clang-x86_64-Release-Goma.json
@@ -0,0 +1,196 @@
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[START_DIR]/tmp"
+ ],
+ "infra_step": true,
+ "name": "makedirs tmp_dir"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "\nimport os\nimport sys\nimport urllib2\n\nTOKEN_URL = (\n 'http://metadata/computeMetadata/v1/project/attributes/jwt_service_account_goma-client')\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nwith open(sys.argv[1], 'w') as f:\n f.write(contents)\n",
+ "[START_DIR]/tmp/jwt_service_account_goma-client.json"
+ ],
+ "infra_step": true,
+ "name": "download jwt_service_account_goma-client.json",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@import os@@@",
+ "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+ "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = (@@@",
+ "@@@STEP_LOG_LINE@python.inline@ 'http://metadata/computeMetadata/v1/project/attributes/jwt_service_account_goma-client')@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
+ "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
+ "@@@STEP_LOG_LINE@python.inline@@@@",
+ "@@@STEP_LOG_LINE@python.inline@with open(sys.argv[1], 'w') as f:@@@",
+ "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
+ "@@@STEP_LOG_END@python.inline@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/goma",
+ "-ensure-file",
+ "infra_internal/goma/client/linux-amd64 release",
+ "-json-output",
+ "/path/to/tmp/json",
+ "-service-account-json",
+ "[START_DIR]/tmp/jwt_service_account_goma-client.json"
+ ],
+ "name": "ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-release---------\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"infra_internal/goma/client/linux-amd64\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "goma_ctl.py",
+ "ensure_start"
+ ],
+ "cwd": "[CACHE]/goma",
+ "env": {
+ "BUILDTYPE": "Release_x64",
+ "CHROME_HEADLESS": "1",
+ "GOMA_FALLBACK": "0",
+ "GOMA_HERMETIC": "error",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
+ "GOMA_USE_LOCAL": "0",
+ "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
+ },
+ "infra_step": true,
+ "name": "start goma"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "[CUSTOM_C:\\_B_WORK]/skia/bin/fetch-gn"
+ ],
+ "cwd": "[CUSTOM_C:\\_B_WORK]/skia",
+ "env": {
+ "BUILDTYPE": "Release_x64",
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
+ },
+ "infra_step": true,
+ "name": "fetch-gn"
+ },
+ {
+ "cmd": [
+ "[CUSTOM_C:\\_B_WORK]/skia/bin/gn.exe",
+ "gen",
+ "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma/Release_x64",
+ "--args=cc=\"clang\" cc_wrapper=\"[CACHE]/goma/gomacc\" clang_win=\"[START_DIR]/clang_win\" cxx=\"clang++\" is_debug=false target_cpu=\"x86_64\" win_sdk=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/a9e1098bba66d2acccc377d5ee81265910f29272/win_sdk\" win_vc=\"[START_DIR]/t/depot_tools/win_toolchain/vs_files/a9e1098bba66d2acccc377d5ee81265910f29272/VC\""
+ ],
+ "cwd": "[CUSTOM_C:\\_B_WORK]/skia",
+ "env": {
+ "BUILDTYPE": "Release_x64",
+ "CHROME_HEADLESS": "1",
+ "GOMA_FALLBACK": "0",
+ "GOMA_HERMETIC": "error",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
+ "GOMA_USE_LOCAL": "0",
+ "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
+ },
+ "name": "gn gen"
+ },
+ {
+ "cmd": [
+ "ninja.exe",
+ "-k",
+ "0",
+ "-C",
+ "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma/Release_x64",
+ "-j",
+ "100"
+ ],
+ "cwd": "[CUSTOM_C:\\_B_WORK]/skia",
+ "env": {
+ "BUILDTYPE": "Release_x64",
+ "CHROME_HEADLESS": "1",
+ "GOMA_FALLBACK": "0",
+ "GOMA_HERMETIC": "error",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
+ "GOMA_USE_LOCAL": "0",
+ "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
+ },
+ "name": "ninja"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "goma_ctl.py",
+ "stat"
+ ],
+ "cwd": "[CACHE]/goma",
+ "env": {
+ "BUILDTYPE": "Release_x64",
+ "CHROME_HEADLESS": "1",
+ "GOMA_FALLBACK": "0",
+ "GOMA_HERMETIC": "error",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
+ "GOMA_USE_LOCAL": "0",
+ "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
+ },
+ "infra_step": true,
+ "name": "print goma stats"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "goma_ctl.py",
+ "stop"
+ ],
+ "cwd": "[CACHE]/goma",
+ "env": {
+ "BUILDTYPE": "Release_x64",
+ "CHROME_HEADLESS": "1",
+ "GOMA_FALLBACK": "0",
+ "GOMA_HERMETIC": "error",
+ "GOMA_SERVICE_ACCOUNT_JSON_FILE": "[START_DIR]/tmp/jwt_service_account_goma-client.json",
+ "GOMA_USE_LOCAL": "0",
+ "PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]",
+ "SKIA_OUT": "[CUSTOM_C:\\_B_WORK]/skia/out/Build-Win-Clang-x86_64-Release-Goma"
+ },
+ "infra_step": true,
+ "name": "stop goma"
+ },
+ {
+ "name": "$result",
+ "recipe_result": null,
+ "status_code": 0
+ }
+]
\ No newline at end of file
diff --git a/infra/bots/recipe_modules/flavor/examples/full.py b/infra/bots/recipe_modules/flavor/examples/full.py
index 6a052ed..4ec7e8f 100644
--- a/infra/bots/recipe_modules/flavor/examples/full.py
+++ b/infra/bots/recipe_modules/flavor/examples/full.py
@@ -77,6 +77,7 @@
'Build-Win-Clang-arm64-Release-Android',
'Build-Win-Clang-x86_64-Debug-GDI',
'Build-Win-Clang-x86_64-Release',
+ 'Build-Win-Clang-x86_64-Release-Goma',
'Build-Win-Clang-x86_64-Release-Vulkan',
'Build-Win-MSVC-x86-Debug-Exceptions',
'Housekeeper-PerCommit-CheckGeneratedFiles',
diff --git a/infra/bots/recipe_modules/flavor/gn_flavor.py b/infra/bots/recipe_modules/flavor/gn_flavor.py
index 62eb15f..9e8e22b 100644
--- a/infra/bots/recipe_modules/flavor/gn_flavor.py
+++ b/infra/bots/recipe_modules/flavor/gn_flavor.py
@@ -22,6 +22,36 @@
'--output-dir', self.m.vars.skia_out.join(self.m.vars.configuration),
'--no-sync', '--make-output-dir'])
+ def _get_goma_json(self):
+ json_key = 'jwt_service_account_goma-client'
+ json_filename = json_key + '.json'
+
+ # Ensure that the tmp_dir exists.
+ self.m.run.run_once(self.m.file.ensure_directory,
+ 'makedirs tmp_dir',
+ self.m.vars.tmp_dir)
+
+ json_file = self.m.vars.tmp_dir.join(json_filename)
+ self.m.python.inline(
+ 'download ' + json_filename,
+ """
+import os
+import sys
+import urllib2
+
+TOKEN_URL = (
+ 'http://metadata/computeMetadata/v1/project/attributes/%s')
+
+req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})
+contents = urllib2.urlopen(req).read()
+
+with open(sys.argv[1], 'w') as f:
+ f.write(contents)
+""" % json_key,
+ args=[json_file],
+ infra_step=True)
+ return json_file
+
def compile(self, unused_target):
"""Build Skia with GN."""
compiler = self.m.vars.builder_cfg.get('compiler', '')
@@ -30,6 +60,7 @@
os = self.m.vars.builder_cfg.get('os', '')
target_arch = self.m.vars.builder_cfg.get('target_arch', '')
+ goma_dir = None
clang_linux = str(self.m.vars.slave_dir.join('clang_linux'))
emscripten_sdk = str(self.m.vars.slave_dir.join('emscripten_sdk'))
linux_vulkan_sdk = str(self.m.vars.slave_dir.join('linux_vulkan_sdk'))
@@ -81,6 +112,8 @@
extra_ldflags.append('-L' + clang_linux + '/msan')
args = {}
+ ninja_args = ['-k', '0', '-C', self.out_dir]
+ env = {}
if configuration != 'Debug':
args['is_debug'] = 'false'
@@ -130,6 +163,21 @@
'skia_use_icu': 'false',
'skia_enable_gpu': 'false',
})
+ if 'Goma' in extra_config:
+ json_file = self._get_goma_json()
+ self.m.cipd.set_service_account_credentials(json_file)
+ goma_package = ('infra_internal/goma/client/%s' %
+ self.m.cipd.platform_suffix())
+ goma_dir = self.m.path['cache'].join('goma')
+ self.m.cipd.ensure(goma_dir, {goma_package: 'release'})
+ env['GOMA_SERVICE_ACCOUNT_JSON_FILE'] = json_file
+ env['GOMA_HERMETIC'] = 'error'
+ env['GOMA_USE_LOCAL'] = '0'
+ env['GOMA_FALLBACK'] = '0'
+ with self.m.context(cwd=goma_dir, env=env):
+ self._py('start goma', 'goma_ctl.py', args=['ensure_start'])
+ args['cc_wrapper'] = '"%s"' % goma_dir.join('gomacc')
+ ninja_args.extend(['-j', '100'])
sanitize = ''
if 'SAN' in extra_config:
@@ -159,21 +207,30 @@
ninja = 'ninja.exe' if 'Win' in os else 'ninja'
gn = self.m.vars.skia_dir.join('bin', gn)
- with self.m.context(cwd=self.m.vars.skia_dir):
- self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn'))
- env = {}
- if 'CheckGeneratedFiles' in extra_config:
- env['PATH'] = '%s:%%(PATH)s' % self.m.vars.skia_dir.join('bin')
- self._py(
- 'fetch-clang-format',
- self.m.vars.skia_dir.join('bin', 'fetch-clang-format'))
- if target_arch == 'wasm':
- fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin'
- env['PATH'] = '%s:%%(PATH)s' % fastcomp
+ try:
+ with self.m.context(cwd=self.m.vars.skia_dir):
+ self._py('fetch-gn', self.m.vars.skia_dir.join('bin', 'fetch-gn'))
+ if 'CheckGeneratedFiles' in extra_config:
+ env['PATH'] = '%s:%%(PATH)s' % self.m.vars.skia_dir.join('bin')
+ self._py(
+ 'fetch-clang-format',
+ self.m.vars.skia_dir.join('bin', 'fetch-clang-format'))
+ if target_arch == 'wasm':
+ fastcomp = emscripten_sdk + '/clang/fastcomp/build_incoming_64/bin'
+ env['PATH'] = '%s:%%(PATH)s' % fastcomp
- with self.m.env(env):
- self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args])
- self._run('ninja', [ninja, '-k', '0', '-C', self.out_dir])
+ with self.m.env(env):
+ self._run('gn gen', [gn, 'gen', self.out_dir, '--args=' + gn_args])
+ self._run('ninja', [ninja] + ninja_args)
+ finally:
+ if goma_dir:
+ with self.m.context(cwd=goma_dir, env=env):
+ self.m.run(self.m.python, 'print goma stats',
+ script='goma_ctl.py', args=['stat'], infra_step=True,
+ abort_on_failure=False, fail_build_on_failure=False)
+ self.m.run(self.m.python, 'stop goma',
+ script='goma_ctl.py', args=['stop'], infra_step=True,
+ abort_on_failure=False, fail_build_on_failure=False)
def copy_extra_build_products(self, swarming_out_dir):
configuration = self.m.vars.builder_cfg.get('configuration', '')
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index 1ebc8c3..a5dca04 100644
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -401,6 +401,12 @@
"Build-Win-Clang-x86_64-Debug-ANGLE"
]
},
+ "Build-Win-Clang-x86_64-Debug-Goma": {
+ "priority": 0.8,
+ "tasks": [
+ "Build-Win-Clang-x86_64-Debug-Goma"
+ ]
+ },
"Build-Win-Clang-x86_64-Debug-Vulkan": {
"priority": 0.8,
"tasks": [
@@ -4785,6 +4791,39 @@
"isolate": "compile_skia.isolate",
"priority": 0.8
},
+ "Build-Win-Clang-x86_64-Debug-Goma": {
+ "cipd_packages": [
+ {
+ "name": "skia/bots/clang_win",
+ "path": "clang_win",
+ "version": "version:2"
+ }
+ ],
+ "dependencies": [
+ "Housekeeper-PerCommit-IsolateWinToolchain"
+ ],
+ "dimensions": [
+ "cpu:x86-64-Haswell_GCE",
+ "gpu:none",
+ "os:Windows-2016Server-14393",
+ "pool:Skia"
+ ],
+ "extra_args": [
+ "--workdir",
+ "../../..",
+ "compile",
+ "repository=<(REPO)",
+ "buildername=Build-Win-Clang-x86_64-Debug-Goma",
+ "swarm_out_dir=${ISOLATED_OUTDIR}",
+ "revision=<(REVISION)",
+ "patch_repo=<(PATCH_REPO)",
+ "patch_storage=<(PATCH_STORAGE)",
+ "patch_issue=<(ISSUE)",
+ "patch_set=<(PATCHSET)"
+ ],
+ "isolate": "compile_skia.isolate",
+ "priority": 0.8
+ },
"Build-Win-Clang-x86_64-Debug-Vulkan": {
"cipd_packages": [
{