Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 1 | # Copyright (c) 2012 The Chromium OS Authors. All rights reserved. |
Dale Curtis | 386eea7 | 2011-09-21 18:43:04 -0700 | [diff] [blame] | 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
| 4 | |
Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 5 | import logging |
Eric Li | a11a045 | 2010-01-07 22:01:58 -0800 | [diff] [blame] | 6 | import os |
Christopher Wiley | 809301c | 2013-07-02 18:00:31 -0700 | [diff] [blame^] | 7 | import tempfile |
Alex Miller | 4f34170 | 2013-03-25 12:39:12 -0700 | [diff] [blame] | 8 | import urllib2 |
Christopher Wiley | 809301c | 2013-07-02 18:00:31 -0700 | [diff] [blame^] | 9 | |
Chris Sosa | 3ee5d5c | 2012-02-23 11:18:41 -0800 | [diff] [blame] | 10 | from autotest_lib.client.common_lib import error, global_config |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 11 | from autotest_lib.client.common_lib.cros import dev_server |
Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 12 | from autotest_lib.server import installable_object, autoserv_parser |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 13 | from autotest_lib.server.cros.dynamic_suite import tools |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 14 | from autotest_lib.server.cros.dynamic_suite.constants import JOB_REPO_URL |
Eric Li | 2eb800f | 2011-04-21 16:52:58 -0700 | [diff] [blame] | 15 | |
| 16 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 17 | _CONFIG = global_config.global_config |
| 18 | _PARSER = autoserv_parser.autoserv_parser |
Eric Li | 2eb800f | 2011-04-21 16:52:58 -0700 | [diff] [blame] | 19 | |
Eric Li | a11a045 | 2010-01-07 22:01:58 -0800 | [diff] [blame] | 20 | |
| 21 | class SiteAutotest(installable_object.InstallableObject): |
| 22 | |
Chris Sosa | 3ee5d5c | 2012-02-23 11:18:41 -0800 | [diff] [blame] | 23 | def get(self, location=None): |
Eric Li | a11a045 | 2010-01-07 22:01:58 -0800 | [diff] [blame] | 24 | if not location: |
| 25 | location = os.path.join(self.serverdir, '../client') |
| 26 | location = os.path.abspath(location) |
| 27 | installable_object.InstallableObject.get(self, location) |
| 28 | self.got = True |
| 29 | |
Eric Li | c7444bd | 2011-02-15 17:12:10 -0800 | [diff] [blame] | 30 | |
Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 31 | def _get_fetch_location_from_host_attribute(self): |
| 32 | """Get repo to use for packages from host attribute, if possible. |
| 33 | |
| 34 | Hosts are tagged with an attribute containing the URL |
| 35 | from which to source packages when running a test on that host. |
| 36 | If self.host is set, attempt to look this attribute up by calling out |
| 37 | to the AFE. |
| 38 | |
| 39 | @returns value of the 'job_repo_url' host attribute, if present. |
| 40 | """ |
| 41 | try: |
| 42 | from autotest_lib.server import frontend |
| 43 | if self.host: |
| 44 | afe = frontend.AFE(debug=False) |
| 45 | hosts = afe.get_hosts(hostname=self.host.hostname) |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 46 | if hosts and JOB_REPO_URL in hosts[0].attributes: |
| 47 | return hosts[0].attributes[JOB_REPO_URL] |
| 48 | logging.warning("No %s for %s", JOB_REPO_URL, self.host) |
Alex Miller | 4f34170 | 2013-03-25 12:39:12 -0700 | [diff] [blame] | 49 | except (ImportError, urllib2.URLError): |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 50 | logging.warning('Not attempting to look for %s', JOB_REPO_URL) |
Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 51 | pass |
| 52 | return None |
| 53 | |
| 54 | |
Eric Li | 2eb800f | 2011-04-21 16:52:58 -0700 | [diff] [blame] | 55 | def get_fetch_location(self): |
Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 56 | """Generate list of locations where autotest can look for packages. |
| 57 | |
| 58 | Old n' busted: Autotest packages are always stored at a URL that can |
| 59 | be derived from the one passed via the voodoo magic --image argument. |
| 60 | New hotness: Hosts are tagged with an attribute containing the URL |
| 61 | from which to source packages when running a test on that host. |
| 62 | |
| 63 | @returns the list of candidate locations to check for packages. |
| 64 | """ |
Dale Curtis | 386eea7 | 2011-09-21 18:43:04 -0700 | [diff] [blame] | 65 | repos = super(SiteAutotest, self).get_fetch_location() |
Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 66 | |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 67 | if _PARSER.options.image: |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 68 | image_opt = _PARSER.options.image |
| 69 | if image_opt.startswith('http://'): |
| 70 | # A devserver HTTP url was specified, set that as the repo_url. |
| 71 | repos.append(image_opt.replace( |
| 72 | 'update', 'static/archive').rstrip('/') + '/autotest') |
| 73 | else: |
| 74 | # An image_name like stumpy-release/R27-3437.0.0 was specified, |
| 75 | # set this as the repo_url for the host. If an AFE is not being |
| 76 | # run, this will ensure that the installed build uses the |
| 77 | # associated artifacts for the test specified when running |
| 78 | # autoserv with --image. However, any subsequent tests run on |
| 79 | # the host will no longer have the context of the image option |
| 80 | # and will revert back to utilizing test code/artifacts that are |
| 81 | # currently present in the users source checkout. |
| 82 | devserver_url = dev_server.ImageServer.resolve(image_opt).url() |
| 83 | repo_url = tools.get_package_url(devserver_url, image_opt) |
| 84 | repos.append(repo_url) |
Chris Masone | be78eb0 | 2012-02-23 14:28:19 -0800 | [diff] [blame] | 85 | else: |
Scott Zawalski | eadbf70 | 2013-03-14 09:23:06 -0400 | [diff] [blame] | 86 | # No --image option was specified, look for the repo url via |
| 87 | # the host attribute. If we are not running with a full AFE |
| 88 | # autoserv will fall back to serving packages itself from whatever |
| 89 | # source version it is sync'd to rather than using the proper |
| 90 | # artifacts for the build on the host. |
Chris Masone | be78eb0 | 2012-02-23 14:28:19 -0800 | [diff] [blame] | 91 | found_repo = self._get_fetch_location_from_host_attribute() |
| 92 | if found_repo is not None: |
| 93 | # Add our new repo to the end, the package manager will |
| 94 | # later reverse the list of repositories resulting in ours |
| 95 | # being first |
| 96 | repos.append(found_repo) |
Eric Li | 2eb800f | 2011-04-21 16:52:58 -0700 | [diff] [blame] | 97 | |
Dale Curtis | 386eea7 | 2011-09-21 18:43:04 -0700 | [diff] [blame] | 98 | return repos |
Eric Li | 2eb800f | 2011-04-21 16:52:58 -0700 | [diff] [blame] | 99 | |
| 100 | |
Chris Masone | 8abb6fc | 2012-01-31 09:27:36 -0800 | [diff] [blame] | 101 | def install(self, host=None, autodir=None): |
| 102 | """Install autotest. If |host| is not None, stores it in |self.host|. |
| 103 | |
| 104 | @param host A Host instance on which autotest will be installed |
| 105 | @param autodir Location on the remote host to install to |
| 106 | """ |
| 107 | if host: |
| 108 | self.host = host |
| 109 | |
| 110 | super(SiteAutotest, self).install(host=host, autodir=autodir) |
| 111 | |
| 112 | |
Christopher Wiley | 809301c | 2013-07-02 18:00:31 -0700 | [diff] [blame^] | 113 | def _install(self, host=None, autodir=None, use_autoserv=True, |
| 114 | use_packaging=True): |
| 115 | """ |
| 116 | Install autotest. If get() was not called previously, an |
| 117 | attempt will be made to install from the autotest svn |
| 118 | repository. |
| 119 | |
| 120 | @param host A Host instance on which autotest will be installed |
| 121 | @param autodir Location on the remote host to install to |
| 122 | @param use_autoserv Enable install modes that depend on the client |
| 123 | running with the autoserv harness |
| 124 | @param use_packaging Enable install modes that use the packaging system |
| 125 | |
| 126 | @exception AutoservError if a tarball was not specified and |
| 127 | the target host does not have svn installed in its path |
| 128 | """ |
| 129 | # TODO(milleral): http://crbug.com/258161 |
| 130 | super(SiteAutotest, self)._install(host, autodir, use_autoserv, |
| 131 | use_packaging) |
| 132 | # Send over the most recent global_config.ini after installation if one |
| 133 | # is available. |
| 134 | # This code is a bit duplicated from |
| 135 | # _BaseRun._create_client_config_file, but oh well. |
| 136 | if self.installed and self.source_material: |
| 137 | logging.info('Installing updated global_config.ini.') |
| 138 | destination = os.path.join(self.host.get_autodir(), |
| 139 | 'global_config.ini') |
| 140 | with tempfile.NamedTemporaryFile() as client_config: |
| 141 | config = global_config.global_config |
| 142 | client_section = config.get_section_values('CLIENT') |
| 143 | client_section.write(client_config) |
| 144 | client_config.flush() |
| 145 | self.host.send_file(client_config.name, destination) |
| 146 | |
| 147 | |
Chris Sosa | f4d43ff | 2012-10-30 11:21:05 -0700 | [diff] [blame] | 148 | def run_static_method(self, module, method, results_dir='.', host=None, |
| 149 | *args): |
| 150 | """Runs a non-instance method with |args| from |module| on the client. |
| 151 | |
| 152 | This method runs a static/class/module autotest method on the client. |
| 153 | For example: |
| 154 | run_static_method("autotest_lib.client.cros.cros_ui", "reboot") |
| 155 | |
| 156 | Will run autotest_lib.client.cros.cros_ui.reboot() on the client. |
| 157 | |
| 158 | @param module: module name as you would refer to it when importing in a |
| 159 | control file. e.g. autotest_lib.client.common_lib.module_name. |
| 160 | @param method: the method you want to call. |
| 161 | @param results_dir: A str path where the results should be stored |
| 162 | on the local filesystem. |
| 163 | @param host: A Host instance on which the control file should |
| 164 | be run. |
| 165 | @param args: args to pass to the method. |
| 166 | """ |
| 167 | control = "\n".join(["import %s" % module, |
| 168 | "%s.%s(%s)\n" % (module, method, |
| 169 | ','.join(map(repr, args)))]) |
| 170 | self.run(control, results_dir=results_dir, host=host) |
| 171 | |
| 172 | |
Chris Sosa | 3ee5d5c | 2012-02-23 11:18:41 -0800 | [diff] [blame] | 173 | class SiteClientLogger(object): |
| 174 | """Overrides default client logger to allow for using a local package cache. |
| 175 | """ |
| 176 | |
| 177 | def _process_line(self, line): |
| 178 | """Returns the package checksum file if it exists.""" |
| 179 | logging.debug(line) |
| 180 | fetch_package_match = self.fetch_package_parser.search(line) |
| 181 | if fetch_package_match: |
| 182 | pkg_name, dest_path, fifo_path = fetch_package_match.groups() |
Chris Masone | 44e4d6c | 2012-08-15 14:25:53 -0700 | [diff] [blame] | 183 | serve_packages = _CONFIG.get_config_value( |
Chris Sosa | 3ee5d5c | 2012-02-23 11:18:41 -0800 | [diff] [blame] | 184 | "PACKAGES", "serve_packages_from_autoserv", type=bool) |
| 185 | if serve_packages and pkg_name == 'packages.checksum': |
Chris Sosa | 3ee5d5c | 2012-02-23 11:18:41 -0800 | [diff] [blame] | 186 | try: |
| 187 | checksum_file = os.path.join( |
| 188 | self.job.pkgmgr.pkgmgr_dir, 'packages', pkg_name) |
| 189 | if os.path.exists(checksum_file): |
| 190 | self.host.send_file(checksum_file, dest_path) |
Chris Sosa | 3ee5d5c | 2012-02-23 11:18:41 -0800 | [diff] [blame] | 191 | except error.AutoservRunError: |
| 192 | msg = "Package checksum file not found, continuing anyway" |
| 193 | logging.exception(msg) |
| 194 | |
Chris Sosa | 2cc4da0 | 2012-07-09 16:18:24 -0700 | [diff] [blame] | 195 | try: |
| 196 | # When fetching a package, the client expects to be |
| 197 | # notified when the fetching is complete. Autotest |
| 198 | # does this pushing a B to a fifo queue to the client. |
| 199 | self.host.run("echo B > %s" % fifo_path) |
| 200 | except error.AutoservRunError: |
| 201 | msg = "Checksum installation failed, continuing anyway" |
| 202 | logging.exception(msg) |
| 203 | finally: |
| 204 | return |
Chris Sosa | 3ee5d5c | 2012-02-23 11:18:41 -0800 | [diff] [blame] | 205 | |
| 206 | # Fall through to process the line using the default method. |
| 207 | super(SiteClientLogger, self)._process_line(line) |
| 208 | |
| 209 | |
| 210 | def _send_tarball(self, pkg_name, remote_dest): |
| 211 | """Uses tarballs in package manager by default.""" |
| 212 | try: |
| 213 | server_package = os.path.join(self.job.pkgmgr.pkgmgr_dir, |
| 214 | 'packages', pkg_name) |
| 215 | if os.path.exists(server_package): |
| 216 | self.host.send_file(server_package, remote_dest) |
| 217 | return |
| 218 | |
| 219 | except error.AutoservRunError: |
| 220 | msg = ("Package %s could not be sent from the package cache." % |
| 221 | pkg_name) |
| 222 | logging.exception(msg) |
| 223 | |
| 224 | # Fall through to send tarball the default method. |
| 225 | super(SiteClientLogger, self)._send_tarball(pkg_name, remote_dest) |
| 226 | |
| 227 | |
Eric Li | c7444bd | 2011-02-15 17:12:10 -0800 | [diff] [blame] | 228 | class _SiteRun(object): |
| 229 | pass |