Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 1 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 5 | from distutils import version |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 6 | import json |
Chris Masone | bafbbb0 | 2012-05-16 13:41:36 -0700 | [diff] [blame] | 7 | import logging |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 8 | import os |
Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 9 | import urllib2 |
Chris Masone | f70650c | 2012-05-16 08:52:12 -0700 | [diff] [blame] | 10 | import HTMLParser |
Chris Masone | acbf4e0 | 2012-05-21 16:38:14 -0700 | [diff] [blame] | 11 | import cStringIO |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 12 | import re |
Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 13 | |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 14 | from autotest_lib.client.common_lib import global_config |
Chris Masone | 6f10908 | 2012-07-18 14:21:38 -0700 | [diff] [blame] | 15 | from autotest_lib.client.common_lib.cros import retry |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 16 | from autotest_lib.site_utils.graphite import stats |
Chris Masone | bafbbb0 | 2012-05-16 13:41:36 -0700 | [diff] [blame] | 17 | # TODO(cmasone): redo this class using requests module; http://crosbug.com/30107 |
Scott Zawalski | 01310f9 | 2012-05-08 09:13:07 -0700 | [diff] [blame] | 18 | |
Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 19 | |
| 20 | CONFIG = global_config.global_config |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 21 | # This file is generated at build time and specifies, per suite and per test, |
| 22 | # the DEPENDENCIES list specified in each control file. It's a dict of dicts: |
| 23 | # {'bvt': {'/path/to/autotest/control/site_tests/test1/control': ['dep1']} |
| 24 | # 'suite': {'/path/to/autotest/control/site_tests/test2/control': ['dep2']} |
| 25 | # 'power': {'/path/to/autotest/control/site_tests/test1/control': ['dep1'], |
| 26 | # '/path/to/autotest/control/site_tests/test3/control': ['dep3']} |
| 27 | # } |
| 28 | DEPENDENCIES_FILE = 'test_suites/dependency_info' |
Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 29 | |
| 30 | |
Chris Masone | f70650c | 2012-05-16 08:52:12 -0700 | [diff] [blame] | 31 | class MarkupStripper(HTMLParser.HTMLParser): |
| 32 | """HTML parser that strips HTML tags, coded characters like & |
| 33 | |
| 34 | Works by, basically, not doing anything for any tags, and only recording |
| 35 | the content of text nodes in an internal data structure. |
| 36 | """ |
| 37 | def __init__(self): |
| 38 | self.reset() |
| 39 | self.fed = [] |
| 40 | |
| 41 | |
| 42 | def handle_data(self, d): |
| 43 | """Consume content of text nodes, store it away.""" |
| 44 | self.fed.append(d) |
| 45 | |
| 46 | |
| 47 | def get_data(self): |
| 48 | """Concatenate and return all stored data.""" |
| 49 | return ''.join(self.fed) |
| 50 | |
| 51 | |
Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 52 | def _get_image_storage_server(): |
| 53 | return CONFIG.get_config_value('CROS', 'image_storage_server', type=str) |
| 54 | |
| 55 | |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 56 | def _get_dev_server_list(): |
| 57 | return CONFIG.get_config_value('CROS', 'dev_server', type=list, default=[]) |
Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 58 | |
| 59 | |
Alex Miller | 761341e | 2012-07-18 16:34:01 -0700 | [diff] [blame] | 60 | def _get_crash_server_list(): |
| 61 | return CONFIG.get_config_value('CROS', 'crash_server', type=list, |
| 62 | default=[]) |
| 63 | |
| 64 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 65 | def remote_devserver_call(timeout_min=30): |
Chris Sosa | 6b288c8 | 2012-03-29 15:31:06 -0700 | [diff] [blame] | 66 | """A decorator to use with remote devserver calls. |
| 67 | |
Chris Masone | f70650c | 2012-05-16 08:52:12 -0700 | [diff] [blame] | 68 | This decorator converts urllib2.HTTPErrors into DevServerExceptions with |
| 69 | any embedded error info converted into plain text. |
Chris Sosa | 6b288c8 | 2012-03-29 15:31:06 -0700 | [diff] [blame] | 70 | """ |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 71 | #pylint: disable=C0111 |
| 72 | def inner_decorator(method): |
Chris Sosa | 6b288c8 | 2012-03-29 15:31:06 -0700 | [diff] [blame] | 73 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 74 | @retry.retry(urllib2.URLError, timeout_min=timeout_min) |
| 75 | def wrapper(*args, **kwargs): |
| 76 | """This wrapper actually catches the HTTPError.""" |
| 77 | try: |
| 78 | return method(*args, **kwargs) |
| 79 | except urllib2.HTTPError as e: |
| 80 | error_markup = e.read() |
| 81 | strip = MarkupStripper() |
| 82 | try: |
| 83 | strip.feed(error_markup.decode('utf_32')) |
| 84 | except UnicodeDecodeError: |
| 85 | strip.feed(error_markup) |
| 86 | raise DevServerException(strip.get_data()) |
| 87 | |
| 88 | return wrapper |
| 89 | |
| 90 | return inner_decorator |
Chris Sosa | 6b288c8 | 2012-03-29 15:31:06 -0700 | [diff] [blame] | 91 | |
| 92 | |
Chris Masone | f70650c | 2012-05-16 08:52:12 -0700 | [diff] [blame] | 93 | class DevServerException(Exception): |
| 94 | """Raised when the dev server returns a non-200 HTTP response.""" |
| 95 | pass |
| 96 | |
| 97 | |
Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 98 | class DevServer(object): |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 99 | """Base class for all DevServer-like server stubs. |
| 100 | |
| 101 | This is the base class for interacting with all Dev Server-like servers. |
| 102 | A caller should instantiate a sub-class of DevServer with: |
| 103 | |
| 104 | host = SubClassServer.resolve(build) |
| 105 | server = SubClassServer(host) |
| 106 | """ |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 107 | _MIN_FREE_DISK_SPACE_GB = 20 |
| 108 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 109 | def __init__(self, devserver): |
| 110 | self._devserver = devserver |
Alex Miller | 761341e | 2012-07-18 16:34:01 -0700 | [diff] [blame] | 111 | |
Chris Sosa | 24b3a02 | 2012-07-31 14:27:59 -0700 | [diff] [blame] | 112 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 113 | def url(self): |
| 114 | """Returns the url for this devserver.""" |
| 115 | return self._devserver |
Alex Miller | 761341e | 2012-07-18 16:34:01 -0700 | [diff] [blame] | 116 | |
Chris Sosa | 24b3a02 | 2012-07-31 14:27:59 -0700 | [diff] [blame] | 117 | |
Chris Masone | 859fdec | 2012-01-30 08:38:09 -0800 | [diff] [blame] | 118 | @staticmethod |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 119 | def devserver_healthy(devserver, timeout_min=0.1): |
| 120 | """Returns True if the |devserver| is healthy to stage build. |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 121 | |
| 122 | @param devserver: url of the devserver. |
| 123 | @param timeout_min: How long to wait in minutes before deciding the |
| 124 | the devserver is not up (float). |
| 125 | """ |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 126 | server_name = re.sub(r':\d+$', '', devserver.lstrip('http://')) |
| 127 | # statsd treats |.| as path separator. |
| 128 | server_name = server_name.replace('.', '_') |
| 129 | call = DevServer._build_call(devserver, 'check_health') |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 130 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 131 | @remote_devserver_call(timeout_min=timeout_min) |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 132 | def make_call(): |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 133 | """Inner method that makes the call.""" |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 134 | return urllib2.urlopen(call).read() |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 135 | |
| 136 | try: |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 137 | result_dict = json.load(cStringIO.StringIO(make_call())) |
| 138 | free_disk = result_dict['free_disk'] |
| 139 | stats.Gauge(server_name).send('free_disk', free_disk) |
| 140 | |
| 141 | if free_disk < DevServer._MIN_FREE_DISK_SPACE_GB: |
| 142 | logging.error('Devserver check_health failed. Free disk space ' |
| 143 | 'is low. Only %dGB is available.', free_disk) |
| 144 | stats.Counter(server_name +'.devserver_not_healthy').increment() |
| 145 | return False |
| 146 | |
| 147 | # This counter indicates the load of a devserver. By comparing the |
| 148 | # value of this counter for all devservers, we can evaluate the |
| 149 | # load balancing across all devservers. |
| 150 | stats.Counter(server_name + '.devserver_healthy').increment() |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 151 | return True |
beeps | 9e9a4a1 | 2013-04-24 13:50:01 -0700 | [diff] [blame] | 152 | except Exception as e: |
| 153 | logging.error('Devserver call failed: "%s", timeout: %s seconds,' |
| 154 | ' Error: %s', call, timeout_min*60, str(e)) |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 155 | stats.Counter(server_name + '.devserver_not_healthy').increment() |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 156 | return False |
Alex Miller | 761341e | 2012-07-18 16:34:01 -0700 | [diff] [blame] | 157 | |
| 158 | |
Chris Sosa | 24b3a02 | 2012-07-31 14:27:59 -0700 | [diff] [blame] | 159 | @staticmethod |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 160 | def _build_call(host, method, **kwargs): |
| 161 | """Build a URL to |host| that calls |method|, passing |kwargs|. |
Chris Sosa | 24b3a02 | 2012-07-31 14:27:59 -0700 | [diff] [blame] | 162 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 163 | Builds a URL that calls |method| on the dev server defined by |host|, |
| 164 | passing a set of key/value pairs built from the dict |kwargs|. |
Chris Sosa | 24b3a02 | 2012-07-31 14:27:59 -0700 | [diff] [blame] | 165 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 166 | @param host: a string that is the host basename e.g. http://server:90. |
Chris Masone | 8b76425 | 2012-01-17 11:12:51 -0800 | [diff] [blame] | 167 | @param method: the dev server method to call. |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 168 | @param kwargs: a dict mapping arg names to arg values. |
| 169 | @return the URL string. |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 170 | """ |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 171 | argstr = '&'.join(map(lambda x: "%s=%s" % x, kwargs.iteritems())) |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 172 | return "%(host)s/%(method)s?%(argstr)s" % dict( |
| 173 | host=host, method=method, argstr=argstr) |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 174 | |
Chris Sosa | 24b3a02 | 2012-07-31 14:27:59 -0700 | [diff] [blame] | 175 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 176 | def build_call(self, method, **kwargs): |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 177 | """Builds a devserver RPC string that can be invoked using urllib.open. |
| 178 | |
| 179 | @param method: remote devserver method to call. |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 180 | """ |
| 181 | return self._build_call(self._devserver, method, **kwargs) |
| 182 | |
| 183 | |
| 184 | @classmethod |
| 185 | def build_all_calls(cls, method, **kwargs): |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 186 | """Builds a list of URLs that makes RPC calls on all devservers. |
| 187 | |
| 188 | Build a URL that calls |method| on the dev server, passing a set |
| 189 | of key/value pairs built from the dict |kwargs|. |
| 190 | |
| 191 | @param method: the dev server method to call. |
Chris Masone | 8b76425 | 2012-01-17 11:12:51 -0800 | [diff] [blame] | 192 | @param kwargs: a dict mapping arg names to arg values |
| 193 | @return the URL string |
| 194 | """ |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 195 | calls = [] |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 196 | # Note we use cls.servers as servers is class specific. |
| 197 | for server in cls.servers(): |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 198 | if cls.devserver_healthy(server): |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 199 | calls.append(cls._build_call(server, method, **kwargs)) |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 200 | |
| 201 | return calls |
Chris Masone | 8b76425 | 2012-01-17 11:12:51 -0800 | [diff] [blame] | 202 | |
| 203 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 204 | @staticmethod |
| 205 | def servers(): |
| 206 | """Returns a list of servers that can serve as this type of server.""" |
| 207 | raise NotImplementedError() |
Chris Sosa | 6b288c8 | 2012-03-29 15:31:06 -0700 | [diff] [blame] | 208 | |
| 209 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 210 | @classmethod |
| 211 | def resolve(cls, build): |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 212 | """"Resolves a build to a devserver instance. |
| 213 | |
| 214 | @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514). |
| 215 | """ |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 216 | devservers = cls.servers() |
| 217 | while devservers: |
| 218 | hash_index = hash(build) % len(devservers) |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 219 | devserver = devservers.pop(hash_index) |
Dan Shi | 6a868dd | 2013-04-25 16:02:32 -0700 | [diff] [blame] | 220 | if cls.devserver_healthy(devserver): |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 221 | return cls(devserver) |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 222 | else: |
| 223 | logging.error('All devservers are currently down!!!') |
| 224 | raise DevServerException('All devservers are currently down!!!') |
Chris Masone | fb08a77 | 2012-01-12 15:57:41 -0800 | [diff] [blame] | 225 | |
| 226 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 227 | class CrashServer(DevServer): |
| 228 | """Class of DevServer that symbolicates crash dumps.""" |
| 229 | @staticmethod |
| 230 | def servers(): |
| 231 | return _get_crash_server_list() |
Scott Zawalski | 136dc4b | 2012-03-20 16:53:25 -0400 | [diff] [blame] | 232 | |
Chris Sosa | 6b288c8 | 2012-03-29 15:31:06 -0700 | [diff] [blame] | 233 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 234 | @remote_devserver_call() |
Chris Masone | bafbbb0 | 2012-05-16 13:41:36 -0700 | [diff] [blame] | 235 | def symbolicate_dump(self, minidump_path, build): |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 236 | """Ask the devserver to symbolicate the dump at minidump_path. |
Chris Masone | bafbbb0 | 2012-05-16 13:41:36 -0700 | [diff] [blame] | 237 | |
| 238 | Stage the debug symbols for |build| and, if that works, ask the |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 239 | devserver to symbolicate the dump at |minidump_path|. |
Chris Masone | bafbbb0 | 2012-05-16 13:41:36 -0700 | [diff] [blame] | 240 | |
| 241 | @param minidump_path: the on-disk path of the minidump. |
| 242 | @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) |
| 243 | whose debug symbols are needed for symbolication. |
| 244 | @return The contents of the stack trace |
| 245 | @raise DevServerException upon any return code that's not HTTP OK. |
| 246 | """ |
| 247 | try: |
| 248 | import requests |
| 249 | except ImportError: |
| 250 | logging.warning("Can't 'import requests' to connect to dev server.") |
| 251 | return '' |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 252 | |
Chris Masone | bafbbb0 | 2012-05-16 13:41:36 -0700 | [diff] [blame] | 253 | # Symbolicate minidump. |
Chris Sosa | 7d7b9eb | 2013-02-14 16:12:01 -0800 | [diff] [blame] | 254 | call = self.build_call('symbolicate_dump', |
| 255 | archive_url=_get_image_storage_server() + build) |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 256 | request = requests.post( |
| 257 | call, files={'minidump': open(minidump_path, 'rb')}) |
Chris Masone | bafbbb0 | 2012-05-16 13:41:36 -0700 | [diff] [blame] | 258 | if request.status_code == requests.codes.OK: |
| 259 | return request.text |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 260 | |
Chris Masone | acbf4e0 | 2012-05-21 16:38:14 -0700 | [diff] [blame] | 261 | error_fd = cStringIO.StringIO(request.text) |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 262 | raise urllib2.HTTPError( |
| 263 | call, request.status_code, request.text, request.headers, |
| 264 | error_fd) |
| 265 | |
| 266 | |
| 267 | class ImageServer(DevServer): |
| 268 | """Class for DevServer that handles image-related RPCs.""" |
| 269 | @staticmethod |
| 270 | def servers(): |
| 271 | return _get_dev_server_list() |
| 272 | |
| 273 | |
| 274 | @classmethod |
J. Richard Barnette | 31b2e31 | 2013-04-04 16:05:22 -0700 | [diff] [blame] | 275 | def devserver_url_for_servo(cls, board): |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 276 | """Returns the devserver url for use with servo recovery. |
| 277 | |
J. Richard Barnette | 31b2e31 | 2013-04-04 16:05:22 -0700 | [diff] [blame] | 278 | @param board: The board (e.g. 'x86-mario'). |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 279 | """ |
J. Richard Barnette | 31b2e31 | 2013-04-04 16:05:22 -0700 | [diff] [blame] | 280 | # Ideally, for load balancing we'd select the server based |
| 281 | # on the board. For now, to simplify manual steps on the |
| 282 | # server side, we ignore the board type and hard-code the |
| 283 | # server as first in the list. |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 284 | # |
| 285 | # TODO(jrbarnette) Once we have automated selection of the |
| 286 | # build for recovery, we should revisit this. |
J. Richard Barnette | 31b2e31 | 2013-04-04 16:05:22 -0700 | [diff] [blame] | 287 | url_pattern = CONFIG.get_config_value('CROS', |
| 288 | 'servo_url_pattern', |
| 289 | type=str) |
| 290 | return url_pattern % (cls.servers()[0], board) |
Chris Masone | bafbbb0 | 2012-05-16 13:41:36 -0700 | [diff] [blame] | 291 | |
| 292 | |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 293 | class ArtifactUrls(object): |
| 294 | """A container for URLs of staged artifacts. |
| 295 | |
| 296 | Attributes: |
| 297 | full_payload: URL for downloading a staged full release update |
| 298 | mton_payload: URL for downloading a staged M-to-N release update |
| 299 | nton_payload: URL for downloading a staged N-to-N release update |
| 300 | |
| 301 | """ |
| 302 | def __init__(self, full_payload=None, mton_payload=None, |
| 303 | nton_payload=None): |
| 304 | self.full_payload = full_payload |
| 305 | self.mton_payload = mton_payload |
| 306 | self.nton_payload = nton_payload |
| 307 | |
| 308 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 309 | @remote_devserver_call() |
Chris Sosa | 7d7b9eb | 2013-02-14 16:12:01 -0800 | [diff] [blame] | 310 | def stage_artifacts(self, image, artifacts): |
| 311 | """Tell the devserver to download and stage |artifacts| from |image|. |
| 312 | |
| 313 | This is the main call point for staging any specific artifacts for a |
| 314 | given build. To see the list of artifacts one can stage see: |
| 315 | |
| 316 | ~src/platfrom/dev/artifact_info.py. |
| 317 | |
| 318 | This is maintained along with the actual devserver code. |
| 319 | |
| 320 | @param image: the image to fetch and stage. |
| 321 | @param artifacts: A list of artifacts. |
| 322 | |
| 323 | @raise DevServerException upon any return code that's not HTTP OK. |
| 324 | """ |
| 325 | call = self.build_call('stage', |
| 326 | archive_url=_get_image_storage_server() + image, |
| 327 | artifacts=','.join(artifacts)) |
| 328 | response = urllib2.urlopen(call) |
| 329 | if not response.read() == 'Success': |
| 330 | raise DevServerException("staging artifacts %s for %s failed;" |
| 331 | "HTTP OK not accompanied by 'Success'." % |
| 332 | (' '.join(artifacts), image)) |
| 333 | |
| 334 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 335 | @remote_devserver_call() |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 336 | def trigger_download(self, image, synchronous=True): |
| 337 | """Tell the devserver to download and stage |image|. |
Scott Zawalski | 136dc4b | 2012-03-20 16:53:25 -0400 | [diff] [blame] | 338 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 339 | Tells the devserver to fetch |image| from the image storage server |
| 340 | named by _get_image_storage_server(). |
| 341 | |
| 342 | If |synchronous| is True, waits for the entire download to finish |
| 343 | staging before returning. Otherwise only the artifacts necessary |
| 344 | to start installing images onto DUT's will be staged before returning. |
| 345 | A caller can then call finish_download to guarantee the rest of the |
| 346 | artifacts have finished staging. |
| 347 | |
| 348 | @param image: the image to fetch and stage. |
| 349 | @param synchronous: if True, waits until all components of the image are |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 350 | staged before returning. |
| 351 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 352 | @raise DevServerException upon any return code that's not HTTP OK. |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 353 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 354 | """ |
| 355 | call = self.build_call( |
| 356 | 'download', archive_url=_get_image_storage_server() + image) |
| 357 | response = urllib2.urlopen(call) |
| 358 | was_successful = response.read() == 'Success' |
| 359 | if was_successful and synchronous: |
| 360 | self.finish_download(image) |
| 361 | elif not was_successful: |
| 362 | raise DevServerException("trigger_download for %s failed;" |
| 363 | "HTTP OK not accompanied by 'Success'." % |
| 364 | image) |
| 365 | |
| 366 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 367 | @remote_devserver_call() |
Simran Basi | 833814b | 2013-01-29 13:13:43 -0800 | [diff] [blame] | 368 | def setup_telemetry(self, build): |
| 369 | """Tell the devserver to setup telemetry for this build. |
| 370 | |
| 371 | The devserver will stage autotest and then extract the required files |
| 372 | for telemetry. |
| 373 | |
| 374 | @param build: the build to setup telemetry for. |
| 375 | |
| 376 | @returns path on the devserver that telemetry is installed to. |
| 377 | """ |
| 378 | call = self.build_call( |
| 379 | 'setup_telemetry', |
| 380 | archive_url=_get_image_storage_server() + build) |
| 381 | response = urllib2.urlopen(call) |
| 382 | return response.read() |
| 383 | |
| 384 | |
| 385 | @remote_devserver_call() |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 386 | def finish_download(self, image): |
| 387 | """Tell the devserver to finish staging |image|. |
| 388 | |
| 389 | If trigger_download is called with synchronous=False, it will return |
| 390 | before all artifacts have been staged. This method contacts the |
| 391 | devserver and blocks until all staging is completed and should be |
| 392 | called after a call to trigger_download. |
| 393 | |
| 394 | @param image: the image to fetch and stage. |
| 395 | @raise DevServerException upon any return code that's not HTTP OK. |
| 396 | """ |
| 397 | call = self.build_call('wait_for_status', |
| 398 | archive_url=_get_image_storage_server() + image) |
| 399 | if urllib2.urlopen(call).read() != 'Success': |
| 400 | raise DevServerException("finish_download for %s failed;" |
| 401 | "HTTP OK not accompanied by 'Success'." % |
| 402 | image) |
| 403 | |
Chris Sosa | b76e0ee | 2013-05-22 16:55:41 -0700 | [diff] [blame^] | 404 | def get_update_url(self, image): |
| 405 | """Returns the url that should be passed to the updater. |
| 406 | |
| 407 | @param image: the image that was fetched. |
| 408 | """ |
| 409 | url_pattern = CONFIG.get_config_value('CROS', 'image_url_pattern', |
| 410 | type=str) |
| 411 | return (url_pattern % (self.url(), image)) |
| 412 | |
| 413 | |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 414 | def _get_image_url(self, image): |
| 415 | """Returns the url of the directory for this image on the devserver. |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 416 | |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 417 | @param image: the image that was fetched. |
| 418 | """ |
| 419 | url_pattern = CONFIG.get_config_value('CROS', 'image_url_pattern', |
| 420 | type=str) |
| 421 | return (url_pattern % (self.url(), image)).replace( |
| 422 | 'update', 'static/archive') |
| 423 | |
| 424 | |
| 425 | def get_delta_payload_url(self, payload_type, image): |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 426 | """Returns a URL to a staged delta payload. |
| 427 | |
| 428 | @param payload_type: either 'mton' or 'nton' |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 429 | @param image: the image that was fetched. |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 430 | |
| 431 | @return A fully qualified URL that can be used for downloading the |
| 432 | payload. |
| 433 | |
| 434 | @raise DevServerException if payload type argument is invalid. |
| 435 | |
| 436 | """ |
| 437 | if payload_type not in ('mton', 'nton'): |
| 438 | raise DevServerException('invalid delta payload type: %s' % |
| 439 | payload_type) |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 440 | version = os.path.basename(image) |
| 441 | base_url = self._get_image_url(image) |
Chris Sosa | a2def28 | 2013-04-11 14:54:25 -0700 | [diff] [blame] | 442 | return base_url + '/au/%s_%s/update.gz' % (version, payload_type) |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 443 | |
| 444 | |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 445 | def get_full_payload_url(self, image): |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 446 | """Returns a URL to a staged full payload. |
| 447 | |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 448 | @param image: the image that was fetched. |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 449 | |
| 450 | @return A fully qualified URL that can be used for downloading the |
| 451 | payload. |
| 452 | |
| 453 | """ |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 454 | return self._get_image_url(image) + '/update.gz' |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 455 | |
| 456 | |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 457 | def get_test_image_url(self, image): |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 458 | """Returns a URL to a staged test image. |
| 459 | |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 460 | @param image: the image that was fetched. |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 461 | |
| 462 | @return A fully qualified URL that can be used for downloading the |
| 463 | image. |
| 464 | |
| 465 | """ |
Chris Sosa | b09376d | 2013-03-18 11:15:12 -0700 | [diff] [blame] | 466 | return self._get_image_url(image) + '/chromiumos_test_image.bin' |
Gilad Arnold | 0ed760c | 2012-11-05 23:42:53 -0800 | [diff] [blame] | 467 | |
| 468 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 469 | @remote_devserver_call() |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 470 | def list_control_files(self, build): |
| 471 | """Ask the devserver to list all control files for |build|. |
| 472 | |
| 473 | @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) |
| 474 | whose control files the caller wants listed. |
| 475 | @return None on failure, or a list of control file paths |
| 476 | (e.g. server/site_tests/autoupdate/control) |
| 477 | @raise DevServerException upon any return code that's not HTTP OK. |
| 478 | """ |
| 479 | call = self.build_call('controlfiles', build=build) |
| 480 | response = urllib2.urlopen(call) |
| 481 | return [line.rstrip() for line in response] |
| 482 | |
| 483 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 484 | @remote_devserver_call() |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 485 | def get_control_file(self, build, control_path): |
| 486 | """Ask the devserver for the contents of a control file. |
| 487 | |
| 488 | @param build: The build (e.g. x86-mario-release/R18-1586.0.0-a1-b1514) |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 489 | whose control file the caller wants to fetch. |
| 490 | @param control_path: The file to fetch |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 491 | (e.g. server/site_tests/autoupdate/control) |
| 492 | @return The contents of the desired file. |
| 493 | @raise DevServerException upon any return code that's not HTTP OK. |
| 494 | """ |
| 495 | call = self.build_call('controlfiles', build=build, |
| 496 | control_path=control_path) |
| 497 | return urllib2.urlopen(call).read() |
| 498 | |
| 499 | |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 500 | @remote_devserver_call() |
Chris Masone | 8906ab1 | 2012-07-23 15:37:56 -0700 | [diff] [blame] | 501 | def get_dependencies_file(self, build): |
| 502 | """Ask the dev server for the contents of the suite dependencies file. |
| 503 | |
| 504 | Ask the dev server at |self._dev_server| for the contents of the |
| 505 | pre-processed suite dependencies file (at DEPENDENCIES_FILE) |
| 506 | for |build|. |
| 507 | |
| 508 | @param build: The build (e.g. x86-mario-release/R21-2333.0.0) |
| 509 | whose dependencies the caller is interested in. |
| 510 | @return The contents of the dependencies file, which should eval to |
| 511 | a dict of dicts, as per site_utils/suite_preprocessor.py. |
| 512 | @raise DevServerException upon any return code that's not HTTP OK. |
| 513 | """ |
| 514 | call = self.build_call('controlfiles', |
| 515 | build=build, control_path=DEPENDENCIES_FILE) |
| 516 | return urllib2.urlopen(call).read() |
| 517 | |
| 518 | |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 519 | @classmethod |
Chris Sosa | ae6acd9 | 2013-02-06 15:08:04 -0800 | [diff] [blame] | 520 | @remote_devserver_call() |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 521 | def get_latest_build(cls, target, milestone=''): |
| 522 | """Ask all the devservers for the latest build for a given target. |
Scott Zawalski | 136dc4b | 2012-03-20 16:53:25 -0400 | [diff] [blame] | 523 | |
| 524 | @param target: The build target, typically a combination of the board |
| 525 | and the type of build e.g. x86-mario-release. |
Scott Zawalski | 9d7955f | 2012-03-20 20:30:51 -0400 | [diff] [blame] | 526 | @param milestone: For latest build set to '', for builds only in a |
Scott Zawalski | 136dc4b | 2012-03-20 16:53:25 -0400 | [diff] [blame] | 527 | specific milestone set to a str of format Rxx |
Scott Zawalski | 9d7955f | 2012-03-20 20:30:51 -0400 | [diff] [blame] | 528 | (e.g. R16). Default: ''. Since we are dealing with a |
| 529 | webserver sending an empty string, '', ensures that |
| 530 | the variable in the URL is ignored as if it was set |
| 531 | to None. |
Chris Masone | f70650c | 2012-05-16 08:52:12 -0700 | [diff] [blame] | 532 | @return A string of the returned build e.g. R20-2226.0.0. |
| 533 | @raise DevServerException upon any return code that's not HTTP OK. |
Scott Zawalski | 136dc4b | 2012-03-20 16:53:25 -0400 | [diff] [blame] | 534 | """ |
Chris Sosa | accb5ce | 2012-08-30 17:29:15 -0700 | [diff] [blame] | 535 | calls = cls.build_all_calls('latestbuild', target=target, |
| 536 | milestone=milestone) |
Chris Sosa | e327ec8 | 2012-06-29 15:02:34 -0700 | [diff] [blame] | 537 | latest_builds = [] |
| 538 | for call in calls: |
| 539 | latest_builds.append(urllib2.urlopen(call).read()) |
| 540 | |
| 541 | return max(latest_builds, key=version.LooseVersion) |