[G3 compile bot] Add comment to CL when there is an interesting failure
Also, create and use new service account for the G3 compile bots.
Interesting failure here refers to when the bot fails only because of
test failures and not:
* merge failures
* unknown failure reasons (eg: some flaky timeouts from the script running in borg)
* when the bot is not owned or approved by a Googler
NoTry: true
Bug: skia:9631
Change-Id: I964fd7874269116cca0e25d1b516283a00d5133f
Reviewed-on: https://skia-review.googlesource.com/c/skia/+/254278
Commit-Queue: Ravi Mistry <rmistry@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
diff --git a/infra/bots/cfg.json b/infra/bots/cfg.json
index 91dac47..ca03016 100644
--- a/infra/bots/cfg.json
+++ b/infra/bots/cfg.json
@@ -17,7 +17,6 @@
"UBSAN",
"Valgrind"
],
- "service_account_android_framework_compile": "skia-android-framework-compile@skia-swarming-bots.iam.gserviceaccount.com",
"service_account_compile": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com",
"service_account_housekeeper": "skia-external-housekeeper@skia-swarming-bots.iam.gserviceaccount.com",
"service_account_recreate_skps": "skia-recreate-skps@skia-swarming-bots.iam.gserviceaccount.com",
diff --git a/infra/bots/g3_compile/trigger_wait_g3_task.py b/infra/bots/g3_compile/trigger_wait_g3_task.py
index df0e49c..c796200 100755
--- a/infra/bots/g3_compile/trigger_wait_g3_task.py
+++ b/infra/bots/g3_compile/trigger_wait_g3_task.py
@@ -16,6 +16,7 @@
INFRA_BOTS_DIR = os.path.abspath(os.path.realpath(os.path.join(
os.path.dirname(os.path.abspath(__file__)), os.pardir)))
sys.path.insert(0, INFRA_BOTS_DIR)
+import git_utils
import utils
@@ -108,6 +109,15 @@
return task
+def _add_cl_comment(issue, comment):
+ # Depot tools needs a checkout to use "git cl" even though we are just adding
+ # a comment to a change unrelated to the checkout.
+ # TODO(rmistry): Try using the Gerrit API?
+ with git_utils.NewGitCheckout(repository=utils.SKIA_REPO):
+ add_comment_cmd = ['git', 'cl', 'comments', '-i', str(issue), '-a', comment]
+ subprocess.check_call(add_comment_cmd)
+
+
def _read_from_storage(gs_file, use_expo_retries=True):
"""Returns the contents of the specified file from storage."""
num_retries = GS_RETRIES if use_expo_retries else 1
@@ -168,6 +178,14 @@
elif ret['status'] == 'merge_conflict':
raise G3CompileException(MERGE_CONFLICT_ERROR_MSG)
elif ret['status'] == 'failure':
+ # Add a comment to the CL before throwing an exception. See skbug/9631
+ # for context.
+ msg = ('FYI: The %s experimental CQ bot failed for patchset #%s.\n'
+ 'The bot is known to be flaky and the failure might or might '
+ 'not be related to this change.\n'
+ 'Please take a quick look at http://cl/%s to verify.') % (
+ options.builder_name, task['patchset'], ret['cl'])
+ _add_cl_comment(task['issue'], msg)
raise G3CompileException(
'\n\nRun failed G3 TAP: cl/%s' % ret['cl'] + PATCHING_INFORMATION)
elif ret['status'] == 'success':
@@ -201,6 +219,9 @@
option_parser.add_option(
'', '--output_file', type=str,
help='The file to write the task to.')
+ option_parser.add_option(
+ '', '--builder_name', type=str, default='',
+ help='The builder that triggered this run.')
options, _ = option_parser.parse_args()
sys.exit(trigger_and_wait(options))
diff --git a/infra/bots/gen_tasks_logic/gen_tasks_logic.go b/infra/bots/gen_tasks_logic/gen_tasks_logic.go
index de48fc3..fceb821 100644
--- a/infra/bots/gen_tasks_logic/gen_tasks_logic.go
+++ b/infra/bots/gen_tasks_logic/gen_tasks_logic.go
@@ -207,7 +207,6 @@
Project string `json:"project"`
// Service accounts.
- ServiceAccountAndroidFrameworkCompile string `json:"service_account_android_framework_compile"`
ServiceAccountCompile string `json:"service_account_compile"`
ServiceAccountHousekeeper string `json:"service_account_housekeeper"`
ServiceAccountRecreateSKPs string `json:"service_account_recreate_skps"`
@@ -1113,7 +1112,7 @@
// the name of the last task in the generated chain of tasks, which the Job
// should add as a dependency.
func (b *builder) androidFrameworkCompile(name string) string {
- task := b.kitchenTask(name, "android_compile", "compile_android_framework.isolate", b.cfg.ServiceAccountAndroidFrameworkCompile, b.linuxGceDimensions(MACHINE_TYPE_SMALL), EXTRA_PROPS, OUTPUT_NONE)
+ task := b.kitchenTask(name, "android_compile", "compile_android_framework.isolate", "skia-android-framework-compile@skia-swarming-bots.iam.gserviceaccount.com", b.linuxGceDimensions(MACHINE_TYPE_SMALL), EXTRA_PROPS, OUTPUT_NONE)
timeout(task, 2*time.Hour)
b.usesGit(task, name)
b.MustAddTask(name, task)
@@ -1124,8 +1123,9 @@
// the name of the last task in the generated chain of tasks, which the Job
// should add as a dependency.
func (b *builder) g3FrameworkCompile(name string) string {
- task := b.kitchenTask(name, "g3_compile", "compile_g3_framework.isolate", b.cfg.ServiceAccountCompile, b.linuxGceDimensions(MACHINE_TYPE_SMALL), EXTRA_PROPS, OUTPUT_NONE)
+ task := b.kitchenTask(name, "g3_compile", "compile_g3_framework.isolate", "skia-g3-framework-compile@skia-swarming-bots.iam.gserviceaccount.com", b.linuxGceDimensions(MACHINE_TYPE_SMALL), EXTRA_PROPS, OUTPUT_NONE)
timeout(task, 3*time.Hour)
+ b.usesGit(task, name)
b.MustAddTask(name, task)
return name
}
diff --git a/infra/bots/recipes/g3_compile.expected/g3_compile_nontrybot.json b/infra/bots/recipes/g3_compile.expected/g3_compile_nontrybot.json
index e2442c8..45e39d0 100644
--- a/infra/bots/recipes/g3_compile.expected/g3_compile_nontrybot.json
+++ b/infra/bots/recipes/g3_compile.expected/g3_compile_nontrybot.json
@@ -15,7 +15,7 @@
" arg_names, **additional_args)",
" File \"RECIPE_REPO[recipe_engine]/recipe_engine/internal/property_invoker.py\", in _invoke_with_properties",
" return callable_obj(*props, **additional_args)",
- " File \"RECIPE_REPO[skia]/infra/bots/recipes/g3_compile.py\", line 20, in RunSteps",
+ " File \"RECIPE_REPO[skia]/infra/bots/recipes/g3_compile.py\", line 22, in RunSteps",
" raise Exception('%s can only be run as a trybot.' % api.vars.builder_name)",
"Exception: Build-Debian9-Clang-TAP-Presubmit-G3_Framework can only be run as a trybot."
]
diff --git a/infra/bots/recipes/g3_compile.expected/g3_compile_trybot.json b/infra/bots/recipes/g3_compile.expected/g3_compile_trybot.json
index 8562d56..22d8846 100644
--- a/infra/bots/recipes/g3_compile.expected/g3_compile_trybot.json
+++ b/infra/bots/recipes/g3_compile.expected/g3_compile_trybot.json
@@ -8,8 +8,15 @@
"--patchset",
"12",
"--output_file",
- "[CLEANUP]/g3_try_tmp_1/output_file"
+ "[CLEANUP]/g3_try_tmp_1/output_file",
+ "--builder_name",
+ "Build-Debian9-Clang-TAP-Presubmit-G3_Framework"
],
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
"name": "Trigger and wait for g3 compile task"
},
{
diff --git a/infra/bots/recipes/g3_compile.expected/g3_compile_trybot_failure.json b/infra/bots/recipes/g3_compile.expected/g3_compile_trybot_failure.json
index 4c0e5f9..dd97c86 100644
--- a/infra/bots/recipes/g3_compile.expected/g3_compile_trybot_failure.json
+++ b/infra/bots/recipes/g3_compile.expected/g3_compile_trybot_failure.json
@@ -8,8 +8,15 @@
"--patchset",
"12",
"--output_file",
- "[CLEANUP]/g3_try_tmp_1/output_file"
+ "[CLEANUP]/g3_try_tmp_1/output_file",
+ "--builder_name",
+ "Build-Debian9-Clang-TAP-Presubmit-G3_Framework"
],
+ "cwd": "[START_DIR]/skia",
+ "env": {
+ "CHROME_HEADLESS": "1",
+ "PATH": "<PATH>:RECIPE_REPO[depot_tools]"
+ },
"name": "Trigger and wait for g3 compile task",
"~followup_annotations": [
"@@@STEP_FAILURE@@@"
diff --git a/infra/bots/recipes/g3_compile.py b/infra/bots/recipes/g3_compile.py
index 1c320b6..b7256d3 100644
--- a/infra/bots/recipes/g3_compile.py
+++ b/infra/bots/recipes/g3_compile.py
@@ -4,12 +4,14 @@
DEPS = [
+ 'recipe_engine/context',
'recipe_engine/file',
'recipe_engine/json',
'recipe_engine/path',
'recipe_engine/properties',
'recipe_engine/raw_io',
'recipe_engine/step',
+ 'run',
'vars',
]
@@ -30,9 +32,11 @@
'--issue', api.vars.issue,
'--patchset', api.vars.patchset,
'--output_file', output_file,
+ '--builder_name', api.vars.builder_name,
]
try:
- api.step('Trigger and wait for g3 compile task', cmd=cmd)
+ with api.context(cwd=api.path['start_dir'].join('skia')):
+ api.run(api.step, 'Trigger and wait for g3 compile task', cmd=cmd)
except api.step.StepFailure as e:
# Add CL link if it exists in the output_file.
task_json = api.file.read_json(
diff --git a/infra/bots/tasks.json b/infra/bots/tasks.json
index f94f638..9ab8afa 100755
--- a/infra/bots/tasks.json
+++ b/infra/bots/tasks.json
@@ -3860,6 +3860,18 @@
{
"name": "vpython",
"path": "cache/vpython"
+ },
+ {
+ "name": "git",
+ "path": "cache/git"
+ },
+ {
+ "name": "git_cache",
+ "path": "cache/git_cache"
+ },
+ {
+ "name": "work",
+ "path": "cache/work"
}
],
"cipd_packages": [
@@ -3877,6 +3889,21 @@
"name": "infra/tools/luci/vpython/${platform}",
"path": "cipd_bin_packages",
"version": "git_revision:0e171233385f11fd2b7458728c8ee439d4db53f1"
+ },
+ {
+ "name": "infra/git/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "version:2.24.0.chromium16"
+ },
+ {
+ "name": "infra/tools/git/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:0e171233385f11fd2b7458728c8ee439d4db53f1"
+ },
+ {
+ "name": "infra/tools/luci/git-credential-luci/${platform}",
+ "path": "cipd_bin_packages",
+ "version": "git_revision:0e171233385f11fd2b7458728c8ee439d4db53f1"
}
],
"command": [
@@ -3913,7 +3940,7 @@
"io_timeout_ns": 10800000000000,
"isolate": "compile_g3_framework.isolate",
"max_attempts": 1,
- "service_account": "skia-external-compile-tasks@skia-swarming-bots.iam.gserviceaccount.com"
+ "service_account": "skia-g3-framework-compile@skia-swarming-bots.iam.gserviceaccount.com"
},
"Build-Debian9-Clang-arm-Debug-Android": {
"caches": [