Switch RecreateSKPs and Bookmaker bots to use service accounts
Bug: skia:
Change-Id: I59aefe7c7e7a6861b00f0e22310e7ceaff3d6d0e
Reviewed-on: https://skia-review.googlesource.com/122944
Reviewed-by: Ravi Mistry <rmistry@google.com>
diff --git a/infra/bots/recipe_modules/infra/api.py b/infra/bots/recipe_modules/infra/api.py
index 3b9b387..ee86cd9 100644
--- a/infra/bots/recipe_modules/infra/api.py
+++ b/infra/bots/recipe_modules/infra/api.py
@@ -65,21 +65,6 @@
UPDATE_GO_ATTEMPTS,
cmd=[self.go_exe, 'get', '-u', '-t', '%s/...' % INFRA_GO_PKG])
- class DownloadGitCookies(object):
- """Class to download gitcookies from GS."""
- def __init__(self, gs_path, local_path, api):
- self._gs_path = gs_path
- self._local_path = local_path
- self._api = api
-
- def __enter__(self):
- cmd = ['gsutil', 'cp', self._gs_path, self._local_path]
- self._api.step('download gitcookies', cmd=cmd, infra_step=True)
-
- def __exit__(self, exc_type, _value, _traceback):
- if self._api.path.exists(self._local_path):
- self._api.file.remove('remove %s' % self._local_path, self._local_path)
-
class MetadataFetch():
def __init__(self, api, metadata_key, local_file, **kwargs):
self.m = api
diff --git a/infra/bots/recipe_modules/infra/examples/full.expected/failed_one_update.json b/infra/bots/recipe_modules/infra/examples/full.expected/failed_one_update.json
index d8df1eb..99966e3 100644
--- a/infra/bots/recipe_modules/infra/examples/full.expected/failed_one_update.json
+++ b/infra/bots/recipe_modules/infra/examples/full.expected/failed_one_update.json
@@ -71,16 +71,6 @@
},
{
"cmd": [
- "gsutil",
- "cp",
- "/test/gspath",
- "[START_DIR]/localpath"
- ],
- "infra_step": true,
- "name": "download gitcookies"
- },
- {
- "cmd": [
"python",
"-u",
"\nimport os\nimport urllib2\n\nTOKEN_FILE = 'file'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
diff --git a/infra/bots/recipe_modules/infra/examples/full.expected/infra_tests.json b/infra/bots/recipe_modules/infra/examples/full.expected/infra_tests.json
index c69a50a..6c88554 100644
--- a/infra/bots/recipe_modules/infra/examples/full.expected/infra_tests.json
+++ b/infra/bots/recipe_modules/infra/examples/full.expected/infra_tests.json
@@ -49,29 +49,6 @@
},
{
"cmd": [
- "gsutil",
- "cp",
- "/test/gspath",
- "[START_DIR]/localpath"
- ],
- "infra_step": true,
- "name": "download gitcookies"
- },
- {
- "cmd": [
- "python",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "remove",
- "[START_DIR]/localpath"
- ],
- "infra_step": true,
- "name": "remove [START_DIR]/localpath"
- },
- {
- "cmd": [
"python",
"-u",
"\nimport os\nimport urllib2\n\nTOKEN_FILE = 'file'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/key'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
diff --git a/infra/bots/recipe_modules/infra/examples/full.py b/infra/bots/recipe_modules/infra/examples/full.py
index 38b4740..84b2e2c 100644
--- a/infra/bots/recipe_modules/infra/examples/full.py
+++ b/infra/bots/recipe_modules/infra/examples/full.py
@@ -22,9 +22,6 @@
def RunSteps(api):
api.vars.setup()
api.infra.update_go_deps()
- with api.infra.DownloadGitCookies(
- '/test/gspath', api.path['start_dir'].join('localpath'), api):
- pass
with api.infra.MetadataFetch(api, 'key', 'file'):
pass
diff --git a/infra/bots/recipes/bookmaker.expected/nightly_bookmaker.json b/infra/bots/recipes/bookmaker.expected/nightly_bookmaker.json
index 69a60c3..863a924 100644
--- a/infra/bots/recipes/bookmaker.expected/nightly_bookmaker.json
+++ b/infra/bots/recipes/bookmaker.expected/nightly_bookmaker.json
@@ -219,30 +219,12 @@
},
{
"cmd": [
- "gsutil",
- "cp",
- "gs://skia-buildbots/artifacts/server/.gitcookies_update-docs",
- "[START_DIR]/update_docs.git_cookies"
- ],
- "cwd": "[CUSTOM_/_B_WORK]/skia",
- "env": {
- "GOPATH": "[START_DIR]/gopath",
- "GOROOT": "[START_DIR]/go/go",
- "PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>"
- },
- "infra_step": true,
- "name": "download gitcookies"
- },
- {
- "cmd": [
"python",
"[CUSTOM_/_B_WORK]/skia/infra/bots/upload_md.py",
"--bookmaker_binary",
"[START_DIR]/build/out/Release/bookmaker",
"--fiddlecli_output",
- "[START_DIR]/fiddleout.json",
- "--gitcookies",
- "[START_DIR]/update_docs.git_cookies"
+ "[START_DIR]/fiddleout.json"
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
@@ -256,25 +238,6 @@
"name": "Generate and Upload Markdown files"
},
{
- "cmd": [
- "python",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "remove",
- "[START_DIR]/update_docs.git_cookies"
- ],
- "cwd": "[CUSTOM_/_B_WORK]/skia",
- "env": {
- "GOPATH": "[START_DIR]/gopath",
- "GOROOT": "[START_DIR]/go/go",
- "PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>"
- },
- "infra_step": true,
- "name": "remove [START_DIR]/update_docs.git_cookies"
- },
- {
"name": "$result",
"recipe_result": null,
"status_code": 0
diff --git a/infra/bots/recipes/bookmaker.expected/nightly_failed_upload.json b/infra/bots/recipes/bookmaker.expected/nightly_failed_upload.json
index c39f12d..3d29663 100644
--- a/infra/bots/recipes/bookmaker.expected/nightly_failed_upload.json
+++ b/infra/bots/recipes/bookmaker.expected/nightly_failed_upload.json
@@ -183,30 +183,12 @@
},
{
"cmd": [
- "gsutil",
- "cp",
- "gs://skia-buildbots/artifacts/server/.gitcookies_update-docs",
- "[START_DIR]/update_docs.git_cookies"
- ],
- "cwd": "[CUSTOM_/_B_WORK]/skia",
- "env": {
- "GOPATH": "[START_DIR]/gopath",
- "GOROOT": "[START_DIR]/go/go",
- "PATH": "[START_DIR]/go/go/bin:[START_DIR]/gopath:<PATH>"
- },
- "infra_step": true,
- "name": "download gitcookies"
- },
- {
- "cmd": [
"python",
"[CUSTOM_/_B_WORK]/skia/infra/bots/upload_md.py",
"--bookmaker_binary",
"[START_DIR]/build/out/Release/bookmaker",
"--fiddlecli_output",
- "[START_DIR]/fiddleout.json",
- "--gitcookies",
- "[START_DIR]/update_docs.git_cookies"
+ "[START_DIR]/fiddleout.json"
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
diff --git a/infra/bots/recipes/bookmaker.py b/infra/bots/recipes/bookmaker.py
index f0f0587..7f52786 100644
--- a/infra/bots/recipes/bookmaker.py
+++ b/infra/bots/recipes/bookmaker.py
@@ -24,10 +24,6 @@
'vars',
]
-UPDATE_DOCS_GITCOOKIES_FILE = 'update_docs.git_cookies'
-UPDATE_DOCS_GITCOOKIES_GS_PATH = (
- 'gs://skia-buildbots/artifacts/server/.gitcookies_update-docs')
-
def go_get_fiddlecli(api):
env = api.context.env
@@ -128,17 +124,12 @@
# Step 4: Update docs in site/user/api/ with the output of fiddlecli.
# If there are any new changes then upload and commit the changes.
- update_docs_gitcookies = api.path['start_dir'].join(
- UPDATE_DOCS_GITCOOKIES_FILE)
cmd = ['python',
api.vars.skia_dir.join('infra', 'bots', 'upload_md.py'),
'--bookmaker_binary', bookmaker_binary,
- '--fiddlecli_output', fiddlecli_output,
- '--gitcookies', str(update_docs_gitcookies)]
- with api.infra.DownloadGitCookies(
- UPDATE_DOCS_GITCOOKIES_GS_PATH, update_docs_gitcookies, api):
- with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
- api.run(api.step, 'Generate and Upload Markdown files', cmd=cmd)
+ '--fiddlecli_output', fiddlecli_output]
+ with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
+ api.run(api.step, 'Generate and Upload Markdown files', cmd=cmd)
def GenTests(api):
@@ -285,8 +276,7 @@
path_config='kitchen',
fiddleout_test_data=fiddleout_no_errors_test_data,
swarm_out_dir='[SWARM_OUT_DIR]') +
- api.path.exists(api.path['start_dir'].join('fiddleout.json'),
- api.path['start_dir'].join(UPDATE_DOCS_GITCOOKIES_FILE))
+ api.path.exists(api.path['start_dir'].join('fiddleout.json'))
)
yield (
diff --git a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
index c1af635..dfc0b42 100644
--- a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
+++ b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
@@ -253,6 +253,19 @@
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
+ "rmtree",
+ "[START_DIR]/skp_output"
+ ],
+ "infra_step": true,
+ "name": "rmtree skp_output"
+ },
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
@@ -331,22 +344,10 @@
},
{
"cmd": [
- "gsutil",
- "cp",
- "gs://skia-buildbots/artifacts/server/.gitcookies_update-skps",
- "[START_DIR]/update_skps.git_cookies"
- ],
- "infra_step": true,
- "name": "download gitcookies"
- },
- {
- "cmd": [
"python",
"[CUSTOM_/_B_WORK]/skia/infra/bots/upload_skps.py",
"--target_dir",
- "[START_DIR]/skp_output",
- "--gitcookies",
- "[START_DIR]/update_skps.git_cookies"
+ "[START_DIR]/skp_output"
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
@@ -360,19 +361,6 @@
"name": "Upload SKPs"
},
{
- "cmd": [
- "python",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
- "remove",
- "[START_DIR]/update_skps.git_cookies"
- ],
- "infra_step": true,
- "name": "remove [START_DIR]/update_skps.git_cookies"
- },
- {
"name": "$result",
"recipe_result": null,
"status_code": 0
diff --git a/infra/bots/recipes/recreate_skps.expected/failed_upload.json b/infra/bots/recipes/recreate_skps.expected/failed_upload.json
index ace10c0..51c84ac 100644
--- a/infra/bots/recipes/recreate_skps.expected/failed_upload.json
+++ b/infra/bots/recipes/recreate_skps.expected/failed_upload.json
@@ -344,22 +344,10 @@
},
{
"cmd": [
- "gsutil",
- "cp",
- "gs://skia-buildbots/artifacts/server/.gitcookies_update-skps",
- "[START_DIR]/update_skps.git_cookies"
- ],
- "infra_step": true,
- "name": "download gitcookies"
- },
- {
- "cmd": [
"python",
"[CUSTOM_/_B_WORK]/skia/infra/bots/upload_skps.py",
"--target_dir",
- "[START_DIR]/skp_output",
- "--gitcookies",
- "[START_DIR]/update_skps.git_cookies"
+ "[START_DIR]/skp_output"
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
diff --git a/infra/bots/recipes/recreate_skps.py b/infra/bots/recipes/recreate_skps.py
index df30338..0915595 100644
--- a/infra/bots/recipes/recreate_skps.py
+++ b/infra/bots/recipes/recreate_skps.py
@@ -32,12 +32,6 @@
}
-UPDATE_SKPS_GITCOOKIES_FILE = 'update_skps.git_cookies'
-
-UPDATE_SKPS_GITCOOKIES_GS_PATH = (
- 'gs://skia-buildbots/artifacts/server/.gitcookies_update-skps')
-
-
def RunSteps(api):
# Check out Chrome.
api.core.setup()
@@ -78,16 +72,11 @@
# Upload the SKPs.
if 'Canary' not in api.properties['buildername']:
api.infra.update_go_deps()
- update_skps_gitcookies = api.path['start_dir'].join(
- UPDATE_SKPS_GITCOOKIES_FILE)
cmd = ['python',
api.vars.skia_dir.join('infra', 'bots', 'upload_skps.py'),
- '--target_dir', output_dir,
- '--gitcookies', str(update_skps_gitcookies)]
- with api.infra.DownloadGitCookies(
- UPDATE_SKPS_GITCOOKIES_GS_PATH, update_skps_gitcookies, api):
- with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
- api.run(api.step, 'Upload SKPs', cmd=cmd)
+ '--target_dir', output_dir]
+ with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
+ api.run(api.step, 'Upload SKPs', cmd=cmd)
def GenTests(api):
@@ -110,8 +99,7 @@
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') +
- api.path.exists(api.path['start_dir'].join('skp_output')) +
- api.path.exists(api.path['start_dir'].join(UPDATE_SKPS_GITCOOKIES_FILE))
+ api.path.exists(api.path['start_dir'].join('skp_output'))
)
yield (
diff --git a/infra/bots/upload_md.py b/infra/bots/upload_md.py
index 6ee5ba1..2632a32 100644
--- a/infra/bots/upload_md.py
+++ b/infra/bots/upload_md.py
@@ -13,70 +13,57 @@
import git_utils
-SKIA_COMMITTER_EMAIL = 'update-docs@skia.org'
-SKIA_COMMITTER_NAME = 'Update Docs'
SKIA_REPO = 'https://skia.googlesource.com/skia.git'
COMMIT_MSG = '''Update markdown files
Automatic commit by the Housekeeper-Nightly-Bookmaker bot.
-TBR=%s
+TBR=rmistry@google.com
NO_MERGE_BUILDS
-''' % SKIA_COMMITTER_EMAIL
+'''
CC_LIST = ['rmistry@google.com', 'caryclark@google.com']
def main():
parser = argparse.ArgumentParser()
- parser.add_argument("--gitcookies")
parser.add_argument("--bookmaker_binary")
parser.add_argument("--fiddlecli_output")
args = parser.parse_args()
with git_utils.NewGitCheckout(repository=SKIA_REPO):
- config_dict = {
- 'user.name': SKIA_COMMITTER_NAME,
- 'user.email': SKIA_COMMITTER_EMAIL,
- 'http.cookiefile': args.gitcookies,
- }
- # Skip GCE Auth in depot_tools/gerrit_utils.py. Use gitcookies instead.
- os.environ['SKIP_GCE_AUTH_FOR_GIT'] = 'True'
- os.environ['GIT_COOKIES_PATH'] = args.gitcookies
+ with git_utils.GitBranch(branch_name='update_md_files',
+ commit_msg=COMMIT_MSG,
+ commit_queue=True,
+ upload=False,
+ cc_list=CC_LIST) as git_branch:
+ # Run bookmaker binary.
+ cmd = [args.bookmaker_binary,
+ '-b', 'docs',
+ '-f', args.fiddlecli_output,
+ '-r', 'site/user/api',
+ ]
+ try:
+ subprocess.check_call(cmd)
+ except subprocess.CalledProcessError as e:
+ print >> sys.stderr, (
+ 'Running %s failed, not uploading markdowns update:\n\n%s' % (
+ cmd, e.output))
+ sys.exit(1)
- with git_utils.GitLocalConfig(config_dict):
- with git_utils.GitBranch(branch_name='update_md_files',
- commit_msg=COMMIT_MSG,
- commit_queue=True,
- upload=False,
- cc_list=CC_LIST) as git_branch:
- # Run bookmaker binary.
- cmd = [args.bookmaker_binary,
- '-b', 'docs',
- '-f', args.fiddlecli_output,
- '-r', 'site/user/api',
- ]
- try:
- subprocess.check_call(cmd)
- except subprocess.CalledProcessError as e:
+ # Verify that only files in the expected directory are going to be
+ # committed and uploaded.
+ diff_files = subprocess.check_output(['git', 'diff', '--name-only'])
+ for diff_file in diff_files.split():
+ if not diff_file.startswith('site/user/api/'):
print >> sys.stderr, (
- 'Running %s failed, not uploading markdowns update:\n\n%s' % (
- cmd, e.output))
+ 'Some files in %s were not in the site/user/api dir. '
+ 'Not uploading them' % diff_files)
sys.exit(1)
-
- # Verify that only files in the expected directory are going to be
- # committed and uploaded.
- diff_files = subprocess.check_output(['git', 'diff', '--name-only'])
- for diff_file in diff_files.split():
- if not diff_file.startswith('site/user/api/'):
- print >> sys.stderr, (
- 'Some files in %s were not in the site/user/api dir. '
- 'Not uploading them' % diff_files)
- sys.exit(1)
- if diff_files:
- subprocess.check_call(['git', 'add', '-u'])
- git_branch.commit_and_upload(True)
- else:
- print 'No changes so nothing to upload.'
+ if diff_files:
+ subprocess.check_call(['git', 'add', '-u'])
+ git_branch.commit_and_upload(True)
+ else:
+ print 'No changes so nothing to upload.'
if '__main__' == __name__:
diff --git a/infra/bots/upload_skps.py b/infra/bots/upload_skps.py
index dab9130..b7bb0cf 100644
--- a/infra/bots/upload_skps.py
+++ b/infra/bots/upload_skps.py
@@ -13,19 +13,18 @@
import git_utils
-SKIA_COMMITTER_EMAIL = 'update-skps@skia.org'
-SKIA_COMMITTER_NAME = 'UpdateSKPs'
+
COMMIT_MSG = '''Update SKP version
Automatic commit by the RecreateSKPs bot.
-TBR=%s
+TBR=rmistry@google.com
NO_MERGE_BUILDS
-''' % SKIA_COMMITTER_EMAIL
+'''
SKIA_REPO = 'https://skia.googlesource.com/skia.git'
-def main(target_dir, gitcookies):
+def main(target_dir):
with git_utils.NewGitCheckout(repository=SKIA_REPO):
# First verify that there are no gen_tasks diffs.
gen_tasks = os.path.join(os.getcwd(), 'infra', 'bots', 'gen_tasks.go')
@@ -36,31 +35,20 @@
'gen_tasks.go failed, not uploading SKP update:\n\n%s' % e.output)
sys.exit(1)
- # Skip GCE Auth in depot_tools/gerrit_utils.py. Use gitcookies instead.
- os.environ['SKIP_GCE_AUTH_FOR_GIT'] = 'True'
- os.environ['GIT_COOKIES_PATH'] = gitcookies
- os.environ['USE_CIPD_GCE_AUTH'] = 'True'
# Upload the new version, land the update CL as the update-skps user.
- config_dict = {
- 'user.name': SKIA_COMMITTER_NAME,
- 'user.email': SKIA_COMMITTER_EMAIL,
- 'http.cookiefile': gitcookies,
- }
- with git_utils.GitLocalConfig(config_dict):
- with git_utils.GitBranch(branch_name='update_skp_version',
- commit_msg=COMMIT_MSG,
- commit_queue=True):
- upload_script = os.path.join(
- os.getcwd(), 'infra', 'bots', 'assets', 'skp', 'upload.py')
- subprocess.check_call(['python', upload_script, '-t', target_dir])
- subprocess.check_call(['go', 'run', gen_tasks])
- subprocess.check_call([
- 'git', 'add', os.path.join('infra', 'bots', 'tasks.json')])
+ with git_utils.GitBranch(branch_name='update_skp_version',
+ commit_msg=COMMIT_MSG,
+ commit_queue=True):
+ upload_script = os.path.join(
+ os.getcwd(), 'infra', 'bots', 'assets', 'skp', 'upload.py')
+ subprocess.check_call(['python', upload_script, '-t', target_dir])
+ subprocess.check_call(['go', 'run', gen_tasks])
+ subprocess.check_call([
+ 'git', 'add', os.path.join('infra', 'bots', 'tasks.json')])
if '__main__' == __name__:
parser = argparse.ArgumentParser()
parser.add_argument("--target_dir")
- parser.add_argument("--gitcookies")
args = parser.parse_args()
- main(args.target_dir, args.gitcookies)
+ main(args.target_dir)