mbligh | 38e9d78 | 2008-05-01 20:49:41 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
Dan Shi | 404c25e | 2017-08-17 23:11:00 -0700 | [diff] [blame] | 3 | import cgi, os, socket, sys, urllib2 |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 4 | import common |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 5 | from multiprocessing import pool |
Dan Shi | 8e33105 | 2015-01-13 11:25:50 -0800 | [diff] [blame] | 6 | from autotest_lib.frontend import setup_django_environment |
| 7 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 8 | from autotest_lib.client.common_lib import global_config |
mbligh | 062ed15 | 2009-01-13 00:57:14 +0000 | [diff] [blame] | 9 | from autotest_lib.client.bin import utils |
showard | ef6fe02 | 2009-03-27 20:55:16 +0000 | [diff] [blame] | 10 | from autotest_lib.frontend.afe.json_rpc import serviceHandler |
Dan Shi | 114e172 | 2016-01-10 18:12:53 -0800 | [diff] [blame] | 11 | from autotest_lib.server import system_utils |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 12 | from autotest_lib.server import utils as server_utils |
Dan Shi | 114e172 | 2016-01-10 18:12:53 -0800 | [diff] [blame] | 13 | |
mbligh | 38e9d78 | 2008-05-01 20:49:41 +0000 | [diff] [blame] | 14 | |
showard | ef6fe02 | 2009-03-27 20:55:16 +0000 | [diff] [blame] | 15 | _PAGE = """\ |
showard | a80823b | 2008-07-24 16:33:35 +0000 | [diff] [blame] | 16 | Status: 302 Found |
mbligh | e0869d0 | 2008-06-04 20:04:28 +0000 | [diff] [blame] | 17 | Content-Type: text/plain |
showard | 77f95db | 2008-07-17 16:59:38 +0000 | [diff] [blame] | 18 | Location: %s\r\n\r |
mbligh | 38e9d78 | 2008-05-01 20:49:41 +0000 | [diff] [blame] | 19 | """ |
| 20 | |
Shuhei Takahashi | 751a4af | 2018-05-25 11:45:56 +0900 | [diff] [blame] | 21 | VIEWER_PREFIX = 'stainless.corp.google.com/browse/' |
Simran Basi | 6e58d97 | 2014-09-18 14:18:44 -0700 | [diff] [blame] | 22 | |
showard | ef6fe02 | 2009-03-27 20:55:16 +0000 | [diff] [blame] | 23 | # Define function for retrieving logs |
mbligh | 062ed15 | 2009-01-13 00:57:14 +0000 | [diff] [blame] | 24 | def _retrieve_logs_dummy(job_path): |
| 25 | pass |
| 26 | |
mbligh | bafd9a7 | 2009-07-28 23:27:42 +0000 | [diff] [blame] | 27 | site_retrieve_logs = utils.import_site_function(__file__, |
| 28 | "autotest_lib.tko.site_retrieve_logs", "site_retrieve_logs", |
mbligh | 062ed15 | 2009-01-13 00:57:14 +0000 | [diff] [blame] | 29 | _retrieve_logs_dummy) |
| 30 | |
mbligh | bafd9a7 | 2009-07-28 23:27:42 +0000 | [diff] [blame] | 31 | site_find_repository_host = utils.import_site_function(__file__, |
| 32 | "autotest_lib.tko.site_retrieve_logs", "site_find_repository_host", |
| 33 | _retrieve_logs_dummy) |
| 34 | |
mbligh | 38e9d78 | 2008-05-01 20:49:41 +0000 | [diff] [blame] | 35 | form = cgi.FieldStorage(keep_blank_values=True) |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 36 | # determine if this is a JSON-RPC request. we support both so that the new TKO |
showard | ef6fe02 | 2009-03-27 20:55:16 +0000 | [diff] [blame] | 37 | # client can use its RPC client code, but the old TKO can still use simple GET |
| 38 | # params. |
| 39 | _is_json_request = form.has_key('callback') |
| 40 | |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 41 | # if this key exists, we check if requested log exists in local machine, |
| 42 | # and do not return Google Storage URL when the log doesn't exist. |
| 43 | _local_only = form.has_key('localonly') |
| 44 | |
| 45 | |
showard | ef6fe02 | 2009-03-27 20:55:16 +0000 | [diff] [blame] | 46 | def _get_requested_path(): |
| 47 | if _is_json_request: |
| 48 | request_data = form['request'].value |
| 49 | request = serviceHandler.ServiceHandler.translateRequest(request_data) |
| 50 | parameters = request['params'][0] |
| 51 | return parameters['path'] |
| 52 | |
| 53 | return form['job'].value |
mbligh | 38e9d78 | 2008-05-01 20:49:41 +0000 | [diff] [blame] | 54 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 55 | |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 56 | def _check_result(args): |
| 57 | host = args['host'] |
| 58 | job_path = args['job_path'] |
| 59 | shard = args['shard'] |
| 60 | if shard: |
| 61 | http_path = 'http://%s/tko/retrieve_logs.cgi?localonly&job=%s' % ( |
| 62 | host, job_path) |
| 63 | else: |
| 64 | http_path = 'http://%s%s' % (host, job_path) |
| 65 | |
| 66 | try: |
Prathmesh Prabhu | 1f9ef09 | 2018-04-24 13:52:29 -0700 | [diff] [blame] | 67 | # HACK: This urlopen call isn't forwarding HTTP headers correctly. This |
| 68 | # leads to uberproxy sitting between master (orignator of this request) |
| 69 | # and shard (target of the request) to redirect to the the login page. |
| 70 | # We detect this condition and reject the target shard as a viable |
| 71 | # redirect. The implication is that we will not redirect to the shard |
| 72 | # even if the user could themselves access the shard with the correct |
| 73 | # credentials. |
| 74 | u = utils.urlopen(http_path) |
| 75 | redirected_url = u.geturl() |
| 76 | if 'accounts.google.com' in redirected_url: |
| 77 | return None |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 78 | |
| 79 | # On Vms the shard name is set to the default gateway but the |
| 80 | # browser used to navigate frontends (that runs on the host of |
| 81 | # the vms) is immune to the same NAT routing the vms have, so we |
| 82 | # need to replace the gateway with 'localhost'. |
| 83 | if utils.DEFAULT_VM_GATEWAY in host: |
| 84 | normalized_host = host.replace(utils.DEFAULT_VM_GATEWAY, 'localhost') |
| 85 | else: |
Dan Shi | 404c25e | 2017-08-17 23:11:00 -0700 | [diff] [blame] | 86 | try: |
| 87 | normalized_host = utils.normalize_hostname(host) |
| 88 | except socket.herror: |
| 89 | # Ignore error: 'socket.herror: [Errno 1] Unknown host' |
| 90 | # This can happen when reverse name lookup is not stable. |
| 91 | normalized_host = host |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 92 | return 'http', normalized_host, job_path |
| 93 | except urllib2.URLError: |
| 94 | return None |
| 95 | |
| 96 | |
| 97 | def _get_tpool_args(hosts, job_path, is_shard, host_set): |
| 98 | """Get a list of arguments to be passed to multiprocessing.pool.ThreadPool. |
| 99 | |
| 100 | @param hosts: a list of host names. |
| 101 | @param job_path: a requested job path. |
| 102 | @param is_shard: True if hosts are shards, False otherwise. |
| 103 | @param host_set: a Set to filter out duplicated hosts. |
| 104 | |
| 105 | @return: a list of dictionaries to be used as input of _check_result(). |
| 106 | """ |
| 107 | args = [] |
| 108 | for host in hosts: |
| 109 | host = host.strip() |
| 110 | if host and host != 'localhost' and host not in host_set: |
| 111 | host_set.add(host) |
| 112 | arg = {'host': host, 'job_path': job_path, 'shard': is_shard} |
| 113 | args.append(arg) |
| 114 | return args |
| 115 | |
| 116 | |
showard | 9484c31 | 2009-01-07 21:07:28 +0000 | [diff] [blame] | 117 | def find_repository_host(job_path): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 118 | """Find the machine holding the given logs and return a URL to the logs""" |
Dale Curtis | 74a314b | 2011-06-23 14:55:46 -0700 | [diff] [blame] | 119 | site_repo_info = site_find_repository_host(job_path) |
| 120 | if site_repo_info is not None: |
| 121 | return site_repo_info |
| 122 | |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 123 | # This cgi script is run only in master (cautotest) and shards. |
| 124 | # Drones do not run this script when receiving '/results/...' request. |
| 125 | # Only master should check drones and shards for the requested log. |
Fang Deng | 18699fe | 2015-12-04 16:40:27 -0800 | [diff] [blame] | 126 | # Also restricted users do not have access to drones or shards, |
| 127 | # always point them to localhost or google storage. |
| 128 | if (not server_utils.is_shard() and |
| 129 | not server_utils.is_restricted_user(os.environ.get('REMOTE_USER'))): |
Dan Shi | 114e172 | 2016-01-10 18:12:53 -0800 | [diff] [blame] | 130 | drones = system_utils.get_drones() |
| 131 | shards = system_utils.get_shards() |
Prashanth Balasubramanian | c85d74f | 2014-12-09 18:13:57 -0800 | [diff] [blame] | 132 | |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 133 | host_set = set() |
| 134 | tpool_args = _get_tpool_args(drones, job_path, False, host_set) |
| 135 | tpool_args += _get_tpool_args(shards, job_path, True, host_set) |
mbligh | ebf8164 | 2009-12-02 19:02:28 +0000 | [diff] [blame] | 136 | |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 137 | tpool = pool.ThreadPool() |
| 138 | for result_path in tpool.imap_unordered(_check_result, tpool_args): |
| 139 | if result_path: |
| 140 | return result_path |
mbligh | ebf8164 | 2009-12-02 19:02:28 +0000 | [diff] [blame] | 141 | |
Simran Basi | 6e58d97 | 2014-09-18 14:18:44 -0700 | [diff] [blame] | 142 | # If the URL requested is a test result, it is now either on the local |
| 143 | # host or in Google Storage. |
| 144 | if job_path.startswith('/results/'): |
| 145 | # We only care about the path after '/results/'. |
| 146 | job_relative_path = job_path[9:] |
MK Ryu | a50e70e | 2015-07-14 11:34:25 -0700 | [diff] [blame] | 147 | if not _local_only and not os.path.exists( |
| 148 | os.path.join('/usr/local/autotest/results', |
| 149 | job_relative_path)): |
Simran Basi | 6e58d97 | 2014-09-18 14:18:44 -0700 | [diff] [blame] | 150 | gsuri = utils.get_offload_gsuri().split('gs://')[1] |
Shuhei Takahashi | 751a4af | 2018-05-25 11:45:56 +0900 | [diff] [blame] | 151 | return ['https', VIEWER_PREFIX, gsuri + job_relative_path] |
Simran Basi | 6e58d97 | 2014-09-18 14:18:44 -0700 | [diff] [blame] | 152 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 153 | |
Dale Curtis | 74a314b | 2011-06-23 14:55:46 -0700 | [diff] [blame] | 154 | def get_full_url(info, log_path): |
| 155 | if info is not None: |
| 156 | protocol, host, path = info |
| 157 | prefix = '%s://%s' % (protocol, host) |
showard | 9484c31 | 2009-01-07 21:07:28 +0000 | [diff] [blame] | 158 | else: |
| 159 | prefix = '' |
Dale Curtis | 74a314b | 2011-06-23 14:55:46 -0700 | [diff] [blame] | 160 | path = log_path |
showard | 9484c31 | 2009-01-07 21:07:28 +0000 | [diff] [blame] | 161 | |
showard | ef6fe02 | 2009-03-27 20:55:16 +0000 | [diff] [blame] | 162 | if _is_json_request: |
| 163 | return '%s/tko/jsonp_fetcher.cgi?%s' % (prefix, |
| 164 | os.environ['QUERY_STRING']) |
showard | 9484c31 | 2009-01-07 21:07:28 +0000 | [diff] [blame] | 165 | else: |
| 166 | return prefix + path |
| 167 | |
| 168 | |
showard | ef6fe02 | 2009-03-27 20:55:16 +0000 | [diff] [blame] | 169 | log_path = _get_requested_path() |
Dale Curtis | 74a314b | 2011-06-23 14:55:46 -0700 | [diff] [blame] | 170 | info = find_repository_host(log_path) |
mbligh | bafd9a7 | 2009-07-28 23:27:42 +0000 | [diff] [blame] | 171 | site_retrieve_logs(log_path) |
Dale Curtis | 74a314b | 2011-06-23 14:55:46 -0700 | [diff] [blame] | 172 | print _PAGE % get_full_url(info, log_path) |