Download gitcookies from GS for RecreateSKPs bot
NoTry: true
Bug: skia:6797
Change-Id: Id7005ba5252c5f1e3f85a8bff92371e0562ab87b
Reviewed-on: https://skia-review.googlesource.com/22726
Commit-Queue: Ravi Mistry <rmistry@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
diff --git a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
index 4928045..4119fbf 100644
--- a/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
+++ b/infra/bots/recipes/recreate_skps.expected/Housekeeper-Weekly-RecreateSKPs.json
@@ -154,19 +154,6 @@
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
- "rmtree",
- "[START_DIR]/skp_output"
- ],
- "infra_step": true,
- "name": "rmtree skp_output"
- },
- {
- "cmd": [
- "python",
- "-u",
- "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
- "--json-output",
- "/path/to/tmp/json",
"ensure-directory",
"--mode",
"0777",
@@ -247,27 +234,14 @@
"cmd": [
"python",
"-u",
- "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py",
+ "--",
+ "cp",
+ "gs://skia-buildbots/artifacts/server/.gitcookies_update-skps",
+ "[START_DIR]/update_skps.git_cookies"
],
- "name": "download update_skps.git_cookies",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@import os@@@",
- "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_skps.git_cookies'@@@",
- "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
- "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
- "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
- "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
- "@@@STEP_LOG_END@python.inline@@@"
- ]
+ "infra_step": true,
+ "name": "gsutil cp"
},
{
"cmd": [
@@ -276,7 +250,7 @@
"--target_dir",
"[START_DIR]/skp_output",
"--gitcookies",
- "[HOME]/update_skps.git_cookies"
+ "[START_DIR]/update_skps.git_cookies"
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
@@ -293,23 +267,14 @@
"cmd": [
"python",
"-u",
- "\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "remove",
+ "[START_DIR]/update_skps.git_cookies"
],
- "name": "cleanup update_skps.git_cookies",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@import os@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_skps.git_cookies'@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
- "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
- "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
- "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
- "@@@STEP_LOG_END@python.inline@@@"
- ]
+ "infra_step": true,
+ "name": "remove [START_DIR]/update_skps.git_cookies"
},
{
"name": "$result",
diff --git a/infra/bots/recipes/recreate_skps.expected/failed_upload.json b/infra/bots/recipes/recreate_skps.expected/failed_upload.json
index e8ada2c..0647f27 100644
--- a/infra/bots/recipes/recreate_skps.expected/failed_upload.json
+++ b/infra/bots/recipes/recreate_skps.expected/failed_upload.json
@@ -247,27 +247,14 @@
"cmd": [
"python",
"-u",
- "\nimport os\nimport urllib2\n\nTOKEN_FILE = 'update_skps.git_cookies'\nTOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'\n\nreq = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})\ncontents = urllib2.urlopen(req).read()\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\n\nwith open(token_file, 'w') as f:\n f.write(contents)\n"
+ "RECIPE_PACKAGE_REPO[depot_tools]/gsutil.py",
+ "--",
+ "cp",
+ "gs://skia-buildbots/artifacts/server/.gitcookies_update-skps",
+ "[START_DIR]/update_skps.git_cookies"
],
- "name": "download update_skps.git_cookies",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@import os@@@",
- "@@@STEP_LOG_LINE@python.inline@import urllib2@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_skps.git_cookies'@@@",
- "@@@STEP_LOG_LINE@python.inline@TOKEN_URL = 'http://metadata/computeMetadata/v1/project/attributes/update_skps_git_cookies'@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@req = urllib2.Request(TOKEN_URL, headers={'Metadata-Flavor': 'Google'})@@@",
- "@@@STEP_LOG_LINE@python.inline@contents = urllib2.urlopen(req).read()@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
- "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@with open(token_file, 'w') as f:@@@",
- "@@@STEP_LOG_LINE@python.inline@ f.write(contents)@@@",
- "@@@STEP_LOG_END@python.inline@@@"
- ]
+ "infra_step": true,
+ "name": "gsutil cp"
},
{
"cmd": [
@@ -276,7 +263,7 @@
"--target_dir",
"[START_DIR]/skp_output",
"--gitcookies",
- "[HOME]/update_skps.git_cookies"
+ "[START_DIR]/update_skps.git_cookies"
],
"cwd": "[CUSTOM_/_B_WORK]/skia",
"env": {
@@ -294,28 +281,6 @@
]
},
{
- "cmd": [
- "python",
- "-u",
- "\nimport os\n\n\nTOKEN_FILE = 'update_skps.git_cookies'\n\n\nhome = os.path.expanduser('~')\ntoken_file = os.path.join(home, TOKEN_FILE)\nif os.path.isfile(token_file):\n os.remove(token_file)\n"
- ],
- "name": "cleanup update_skps.git_cookies",
- "~followup_annotations": [
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@import os@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@TOKEN_FILE = 'update_skps.git_cookies'@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@@@@",
- "@@@STEP_LOG_LINE@python.inline@home = os.path.expanduser('~')@@@",
- "@@@STEP_LOG_LINE@python.inline@token_file = os.path.join(home, TOKEN_FILE)@@@",
- "@@@STEP_LOG_LINE@python.inline@if os.path.isfile(token_file):@@@",
- "@@@STEP_LOG_LINE@python.inline@ os.remove(token_file)@@@",
- "@@@STEP_LOG_END@python.inline@@@"
- ]
- },
- {
"name": "$result",
"reason": "Step('Upload SKPs') failed with return_code 1",
"recipe_result": null,
diff --git a/infra/bots/recipes/recreate_skps.py b/infra/bots/recipes/recreate_skps.py
index 8da1ed6..60b44b6 100644
--- a/infra/bots/recipes/recreate_skps.py
+++ b/infra/bots/recipes/recreate_skps.py
@@ -9,6 +9,7 @@
DEPS = [
'core',
'depot_tools/gclient',
+ 'depot_tools/gsutil',
'infra',
'recipe_engine/context',
'recipe_engine/file',
@@ -33,7 +34,26 @@
UPDATE_SKPS_GITCOOKIES_FILE = 'update_skps.git_cookies'
-UPDATE_SKPS_KEY = 'update_skps_git_cookies'
+
+UPDATE_SKPS_GITCOOKIES_GS_PATH = (
+ 'gs://skia-buildbots/artifacts/server/.gitcookies_update-skps')
+
+
+class DownloadGitCookies(object):
+ """Class to download gitcookies from GS."""
+ def __init__(self, gs_path, local_path, api):
+ self._gs_path = gs_path
+ self._local_path = local_path
+ self._api = api
+
+ def __enter__(self):
+ gsutil_args = ['cp', self._gs_path, self._local_path]
+ self._api.gsutil(gsutil_args, use_retry_wrapper=False)
+
+ def __exit__(self, exc_type, _value, _traceback):
+ if self._api.path.exists(self._local_path):
+ self._api.file.remove('remove %s' % self._local_path, self._local_path)
+
def RunSteps(api):
@@ -76,14 +96,14 @@
# Upload the SKPs.
if 'Canary' not in api.properties['buildername']:
api.infra.update_go_deps()
- update_skps_gitcookies = api.path.join(api.path.expanduser('~'),
- UPDATE_SKPS_GITCOOKIES_FILE)
+ update_skps_gitcookies = api.path['start_dir'].join(
+ UPDATE_SKPS_GITCOOKIES_FILE)
cmd = ['python',
api.vars.skia_dir.join('infra', 'bots', 'upload_skps.py'),
'--target_dir', output_dir,
'--gitcookies', str(update_skps_gitcookies)]
- with api.infra.MetadataFetch(
- api, UPDATE_SKPS_KEY, UPDATE_SKPS_GITCOOKIES_FILE):
+ with DownloadGitCookies(
+ UPDATE_SKPS_GITCOOKIES_GS_PATH, update_skps_gitcookies, api):
with api.context(cwd=api.vars.skia_dir, env=api.infra.go_env):
api.run(api.step, 'Upload SKPs', cmd=cmd)
@@ -108,7 +128,8 @@
revision='abc123',
path_config='kitchen',
swarm_out_dir='[SWARM_OUT_DIR]') +
- api.path.exists(api.path['start_dir'].join('skp_output'))
+ api.path.exists(api.path['start_dir'].join('skp_output')) +
+ api.path.exists(api.path['start_dir'].join(UPDATE_SKPS_GITCOOKIES_FILE))
)
yield (