Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | |
| 3 | # Copyright (c) 2014 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
| 6 | |
| 7 | # This file defines script for getting host_history for DUTs in Autotest. |
| 8 | |
| 9 | """Script for checking host history for a selected group of hosts. |
| 10 | |
| 11 | Currently only supports aggregating stats for each host. |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 12 | |
| 13 | Example usage: |
Dan Shi | 17ecbbf | 2014-10-06 13:56:34 -0700 | [diff] [blame] | 14 | python host_history.py -n 10000 -l 24 --board=daisy |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 15 | |
| 16 | Output: |
| 17 | |
| 18 | trying to get all duts... |
| 19 | making the query... |
| 20 | found all duts. Time to get host_history. |
| 21 | usage stats for host: chromeos2-row5-rack1-host6 |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 22 | 2014-07-24 10:24:07 - 2014-07-25 10:24:07 |
| 23 | Verifying: 0.00 % |
| 24 | Running: 0.00 % |
| 25 | Ready: 100.00 % |
| 26 | Repairing: 0.00 % |
| 27 | Repair Failed: 0.00 % |
| 28 | Cleaning: 0.00 % |
| 29 | Pending: 0.00 % |
| 30 | Resetting: 0.00 % |
| 31 | Provisioning: 0.00 % |
| 32 | Locked: 0.00 % |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 33 | - -- --- ---- ----- ---- --- -- - |
| 34 | |
| 35 | Example usage2: more than one host: |
Dan Shi | 17ecbbf | 2014-10-06 13:56:34 -0700 | [diff] [blame] | 36 | python host_history.py -n 1000 -l 2 \ |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 37 | --hosts chromeos2-row5-rack4-host6 chromeos4-row12-rack11-host2 |
| 38 | |
| 39 | ['chromeos2-row5-rack4-host6', 'chromeos4-row12-rack11-host2'] |
| 40 | found all duts. Time to get host_history. |
| 41 | usage stats for host: chromeos2-row5-rack4-host6 |
| 42 | 2014-07-25 13:02:22 - 2014-07-25 15:02:22 |
| 43 | Num entries found in this interval: 0 |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 44 | Verifying: 0.00 % |
| 45 | Running: 0.00 % |
| 46 | Ready: 100.00 % |
| 47 | Repairing: 0.00 % |
| 48 | Repair Failed: 0.00 % |
| 49 | Cleaning: 0.00 % |
| 50 | Pending: 0.00 % |
| 51 | Resetting: 0.00 % |
| 52 | Provisioning: 0.00 % |
| 53 | Locked: 0.00 % |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 54 | - -- --- ---- ----- ---- --- -- - |
| 55 | |
| 56 | usage stats for host: chromeos4-row12-rack11-host2 |
| 57 | 2014-07-25 13:02:22 - 2014-07-25 15:02:22 |
| 58 | Num entries found in this interval: 138 |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 59 | Verifying: 0.00 % |
| 60 | Running: 70.45 % |
| 61 | Ready: 17.79 % |
| 62 | Repairing: 0.00 % |
| 63 | Repair Failed: 0.00 % |
| 64 | Cleaning: 0.00 % |
| 65 | Pending: 1.24 % |
| 66 | Resetting: 10.78 % |
| 67 | Provisioning: 0.00 % |
| 68 | Locked: 0.00 % |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 69 | - -- --- ---- ----- ---- --- -- - |
| 70 | """ |
| 71 | |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 72 | import argparse |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 73 | import time |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 74 | import traceback |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 75 | |
| 76 | import common |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 77 | from autotest_lib.client.common_lib import time_utils |
Michael Liang | 9e0be9d | 2014-08-07 09:48:42 -0700 | [diff] [blame] | 78 | from autotest_lib.site_utils import host_history_utils |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 79 | |
| 80 | |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 81 | def print_all_stats(results, labels, t_start, t_end): |
| 82 | """Prints overall stats followed by stats for each host. |
| 83 | |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 84 | @param results: A list of tuples of three elements. |
| 85 | 1st element: String representing report for individual host. |
| 86 | 2nd element: An ordered dictionary with |
| 87 | key as (t_start, t_end) and value as (status, metadata) |
| 88 | status = status of the host. e.g. 'Repair Failed' |
| 89 | t_start is the beginning of the interval where the DUT's has |
| 90 | that status |
| 91 | t_end is the end of the interval where the DUT has that |
| 92 | status |
| 93 | metadata: A dictionary of other metadata, e.g., |
| 94 | {'task_id':123, 'task_name':'Reset'} |
| 95 | 3rd element: hostname of the dut. |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 96 | @param labels: A list of labels useful for describing the group |
| 97 | of hosts these overall stats represent. |
| 98 | @param t_start: beginning of time period we are interested in. |
| 99 | @param t_end: end of time period we are interested in. |
| 100 | """ |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 101 | result_strs, stat_intervals_lst, hostname = zip(*results) |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 102 | overall_report_str = host_history_utils.get_overall_report( |
| 103 | labels, t_start, t_end, stat_intervals_lst) |
| 104 | # Print the overall stats |
| 105 | print overall_report_str |
| 106 | # Print the stats for each individual host. |
| 107 | for result_str in result_strs: |
| 108 | print result_str |
| 109 | |
| 110 | |
| 111 | def get_host_history(input): |
| 112 | """Gets the host history. |
| 113 | |
| 114 | @param input: A dictionary of input arguments to |
| 115 | host_history_utils.host_history_stats. |
| 116 | Must contain these keys: |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 117 | 't_start', |
| 118 | 't_end', |
| 119 | 'hostname', |
| 120 | 'size,' |
| 121 | 'print_each_interval' |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 122 | @returns: |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 123 | result_str: String reporting history for specific host. |
| 124 | stat_intervals: A ordered dictionary with |
| 125 | key as (t_start, t_end) and value as (status, metadata) |
| 126 | status = status of the host. e.g. 'Repair Failed' |
| 127 | t_start is the beginning of the interval where the DUT's has |
| 128 | that status |
| 129 | t_end is the end of the interval where the DUT has that |
| 130 | status |
| 131 | metadata: A dictionary of other metadata, e.g., |
| 132 | {'task_id':123, 'task_name':'Reset'} |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 133 | """ |
| 134 | try: |
| 135 | result_str, stat_intervals = host_history_utils.get_report_for_host( |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 136 | **input) |
| 137 | return result_str, stat_intervals, input['hostname'] |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 138 | except Exception as e: |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 139 | # In case any process throws an Exception, we want to see it. |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 140 | print traceback.print_exc() |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 141 | return None, None, None |
| 142 | |
| 143 | |
Dan Shi | 17ecbbf | 2014-10-06 13:56:34 -0700 | [diff] [blame] | 144 | def get_results(start_time, end_time, hosts=None, board=None, pool=None, |
| 145 | verbose=False): |
| 146 | """Get history results of specified hosts or board/pool. |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 147 | |
| 148 | If hosts is set to None, all hosts are used, filtered by the board and pool |
| 149 | constraints. If board is not provided, all boards are included. If pool is |
| 150 | not provided, all pools are included. |
| 151 | If a list of hosts is provided, the board and pool constraints are ignored. |
| 152 | |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 153 | @param hosts: A list of hosts to search for history. Default is None. |
| 154 | @param board: board type of hosts. Default is None. |
| 155 | @param pool: pool type of hosts. Default is None. |
| 156 | @param start_time: start time to search for history, can be string value or |
| 157 | epoch time. |
| 158 | @param end_time: end time to search for history, can be string value or |
| 159 | epoch time. |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 160 | @param verbose: True to print out detail intervals of host history. |
Dan Shi | 17ecbbf | 2014-10-06 13:56:34 -0700 | [diff] [blame] | 161 | |
| 162 | @returns: A dictionary of host history. |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 163 | """ |
| 164 | assert start_time and end_time |
| 165 | start_time = time_utils.to_epoch_time(start_time) |
| 166 | end_time = time_utils.to_epoch_time(end_time) |
| 167 | assert start_time < end_time |
| 168 | |
Dan Shi | 17ecbbf | 2014-10-06 13:56:34 -0700 | [diff] [blame] | 169 | return host_history_utils.get_report(t_start=start_time, t_end=end_time, |
| 170 | hosts=hosts, board=board, pool=pool, |
| 171 | print_each_interval=verbose) |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 172 | |
| 173 | |
| 174 | def get_history_details(start_time, end_time, hosts=None, board=None, |
Dan Shi | 17ecbbf | 2014-10-06 13:56:34 -0700 | [diff] [blame] | 175 | pool=None): |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 176 | """Get the details of host history. |
| 177 | |
| 178 | The return is a dictionary of host history for each host, for example, |
| 179 | {'172.22.33.51': [{'status': 'Resetting' |
| 180 | 'start_time': '2014-08-07 10:02:16', |
| 181 | 'end_time': '2014-08-07 10:03:16', |
| 182 | 'log_url': 'http://autotest/reset-546546/debug', |
| 183 | 'task_id': 546546}, |
| 184 | {'status': 'Running' |
| 185 | 'start_time': '2014-08-07 10:03:18', |
| 186 | 'end_time': '2014-08-07 10:13:00', |
| 187 | 'log_url': ('http://%s/tko/retrieve_logs.cgi?job=/' |
| 188 | 'results/16853-debug/172.22.33.51'), |
| 189 | 'job_id': 16853} |
| 190 | ] |
| 191 | } |
| 192 | @param start_time: start time to search for history, can be string value or |
| 193 | epoch time. |
| 194 | @param end_time: end time to search for history, can be string value or |
| 195 | epoch time. |
| 196 | @param hosts: A list of hosts to search for history. Default is None. |
| 197 | @param board: board type of hosts. Default is None. |
| 198 | @param pool: pool type of hosts. Default is None. |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 199 | @returns: A dictionary of the host history details. |
| 200 | """ |
Dan Shi | 17ecbbf | 2014-10-06 13:56:34 -0700 | [diff] [blame] | 201 | results = get_results(start_time=start_time, end_time=end_time, hosts=hosts, |
| 202 | board=board, pool=pool) |
Dan Shi | 1ccb652 | 2014-09-19 17:15:30 -0700 | [diff] [blame] | 203 | if not results: |
| 204 | # No host found. |
| 205 | return None |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 206 | all_history = {} |
| 207 | for result_str, status_intervals, hostname in results: |
| 208 | if hostname: |
Dan Shi | 1ccb652 | 2014-09-19 17:15:30 -0700 | [diff] [blame] | 209 | all_history[hostname] = host_history_utils.build_history( |
| 210 | hostname, status_intervals) |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 211 | return all_history |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 212 | |
| 213 | |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 214 | def main(): |
| 215 | """main script. """ |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 216 | t_now = time.time() |
| 217 | t_now_minus_one_day = t_now - 3600 * 24 |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 218 | parser = argparse.ArgumentParser() |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 219 | parser.add_argument('-v', action='store_true', dest='verbose', |
| 220 | default=False, |
Michael Liang | 9e0be9d | 2014-08-07 09:48:42 -0700 | [diff] [blame] | 221 | help='-v to print out ALL entries.') |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 222 | parser.add_argument('-l', type=float, dest='last', |
| 223 | help='last hours to search results across', |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 224 | default=None) |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 225 | parser.add_argument('--board', type=str, dest='board', |
| 226 | help='restrict query by board, not implemented yet', |
| 227 | default=None) |
| 228 | parser.add_argument('--pool', type=str, dest='pool', |
| 229 | help='restrict query by pool, not implemented yet', |
| 230 | default=None) |
| 231 | parser.add_argument('--hosts', nargs='+', dest='hosts', |
| 232 | help='Enter space deliminated hostnames', |
| 233 | default=[]) |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 234 | parser.add_argument('--start', type=str, dest='start', |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 235 | help=('Enter start time as: yyyy-mm-dd hh:mm:ss,' |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 236 | 'defualts to 24h ago.'), |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 237 | default=time_utils.epoch_time_to_date_string( |
Michael Liang | 329153d | 2014-08-06 15:31:25 -0700 | [diff] [blame] | 238 | t_now_minus_one_day)) |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 239 | parser.add_argument('--end', type=str, dest='end', |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 240 | help=('Enter end time in as: yyyy-mm-dd hh:mm:ss,' |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 241 | 'defualts to current time.'), |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 242 | default=time_utils.epoch_time_to_date_string(t_now)) |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 243 | options = parser.parse_args() |
| 244 | |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 245 | if options.last: |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 246 | start_time = t_now - 3600 * options.last |
| 247 | end_time = t_now |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 248 | else: |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 249 | start_time = time_utils.to_epoch_time(options.start) |
| 250 | end_time = time_utils.to_epoch_time(options.end) |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 251 | |
Dan Shi | 17ecbbf | 2014-10-06 13:56:34 -0700 | [diff] [blame] | 252 | results = get_results(hosts=options.hosts, |
| 253 | board=options.board, |
| 254 | pool=options.pool, |
| 255 | start_time=start_time, |
| 256 | end_time=end_time, |
| 257 | verbose=options.verbose) |
Michael Liang | 93b4ba4 | 2014-07-30 15:09:44 -0700 | [diff] [blame] | 258 | labels = [] |
| 259 | if options.board: |
| 260 | labels.append('board:%s' % (options.board)) |
| 261 | if options.pool: |
| 262 | labels.append('pool:%s' % (options.pool)) |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 263 | print_all_stats(results, labels, start_time, end_time) |
Michael Liang | 5952fbe | 2014-07-24 13:51:26 -0700 | [diff] [blame] | 264 | |
| 265 | |
| 266 | if __name__ == '__main__': |
Dan Shi | dfea368 | 2014-08-10 23:38:40 -0700 | [diff] [blame] | 267 | main() |