Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 1 | import json |
lmr | 6f80e7a | 2010-02-04 03:18:28 +0000 | [diff] [blame] | 2 | import os |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 3 | |
Allen Li | 5977e5c | 2018-10-18 17:44:46 -0700 | [diff] [blame] | 4 | from autotest_lib.server.hosts import file_store |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 5 | from autotest_lib.client.common_lib import utils |
Shuhei Takahashi | 0ae6c4d | 2018-07-31 16:36:08 +0900 | [diff] [blame] | 6 | from autotest_lib.tko import tast |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 7 | from autotest_lib.tko import utils as tko_utils |
jadmanski | 6e8bf75 | 2008-05-14 00:17:48 +0000 | [diff] [blame] | 8 | |
| 9 | |
mbligh | e79ebb0 | 2008-04-17 15:39:22 +0000 | [diff] [blame] | 10 | class job(object): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 11 | """Represents a job.""" |
| 12 | |
jadmanski | f7fa2cc | 2008-10-01 14:13:23 +0000 | [diff] [blame] | 13 | def __init__(self, dir, user, label, machine, queued_time, started_time, |
showard | 71b9431 | 2009-08-20 23:40:02 +0000 | [diff] [blame] | 14 | finished_time, machine_owner, machine_group, aborted_by, |
showard | c1a98d1 | 2010-01-15 00:22:22 +0000 | [diff] [blame] | 15 | aborted_on, keyval_dict): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 16 | self.dir = dir |
| 17 | self.tests = [] |
| 18 | self.user = user |
| 19 | self.label = label |
| 20 | self.machine = machine |
| 21 | self.queued_time = queued_time |
| 22 | self.started_time = started_time |
| 23 | self.finished_time = finished_time |
| 24 | self.machine_owner = machine_owner |
showard | 71b9431 | 2009-08-20 23:40:02 +0000 | [diff] [blame] | 25 | self.machine_group = machine_group |
jadmanski | f7fa2cc | 2008-10-01 14:13:23 +0000 | [diff] [blame] | 26 | self.aborted_by = aborted_by |
| 27 | self.aborted_on = aborted_on |
showard | c1a98d1 | 2010-01-15 00:22:22 +0000 | [diff] [blame] | 28 | self.keyval_dict = keyval_dict |
Michael Tang | 5f74ffd | 2016-10-31 10:34:53 -0700 | [diff] [blame] | 29 | self.afe_parent_job_id = None |
| 30 | self.build_version = None |
| 31 | self.suite = None |
| 32 | self.board = None |
Prathmesh Prabhu | c2a8a6a | 2018-04-19 16:23:32 -0700 | [diff] [blame] | 33 | self.job_idx = None |
Prathmesh Prabhu | f5030d3 | 2018-04-19 16:08:55 -0700 | [diff] [blame] | 34 | # id of the corresponding tko_task_references entry. |
| 35 | # This table is used to refer to skylab task / afe job corresponding to |
| 36 | # this tko_job. |
| 37 | self.task_reference_id = None |
mbligh | e79ebb0 | 2008-04-17 15:39:22 +0000 | [diff] [blame] | 38 | |
jadmanski | a8e302a | 2008-09-25 19:49:38 +0000 | [diff] [blame] | 39 | @staticmethod |
| 40 | def read_keyval(dir): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 41 | """ |
| 42 | Read job keyval files. |
| 43 | |
| 44 | @param dir: String name of directory containing job keyval files. |
| 45 | |
| 46 | @return A dictionary containing job keyvals. |
| 47 | |
| 48 | """ |
jadmanski | a8e302a | 2008-09-25 19:49:38 +0000 | [diff] [blame] | 49 | dir = os.path.normpath(dir) |
| 50 | top_dir = tko_utils.find_toplevel_job_dir(dir) |
| 51 | if not top_dir: |
| 52 | top_dir = dir |
| 53 | assert(dir.startswith(top_dir)) |
| 54 | |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 55 | # Pull in and merge all the keyval files, with higher-level |
| 56 | # overriding values in the lower-level ones. |
jadmanski | a8e302a | 2008-09-25 19:49:38 +0000 | [diff] [blame] | 57 | keyval = {} |
| 58 | while True: |
| 59 | try: |
jadmanski | fc4e338 | 2008-10-02 16:19:19 +0000 | [diff] [blame] | 60 | upper_keyval = utils.read_keyval(dir) |
| 61 | # HACK: exclude hostname from the override - this is a special |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 62 | # case where we want lower to override higher. |
| 63 | if 'hostname' in upper_keyval and 'hostname' in keyval: |
| 64 | del upper_keyval['hostname'] |
jadmanski | 8e00afa | 2008-10-02 16:22:04 +0000 | [diff] [blame] | 65 | keyval.update(upper_keyval) |
jadmanski | a8e302a | 2008-09-25 19:49:38 +0000 | [diff] [blame] | 66 | except IOError: |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 67 | pass # If the keyval can't be read just move on to the next. |
jadmanski | a8e302a | 2008-09-25 19:49:38 +0000 | [diff] [blame] | 68 | if dir == top_dir: |
| 69 | break |
| 70 | else: |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 71 | assert(dir != '/') |
jadmanski | a8e302a | 2008-09-25 19:49:38 +0000 | [diff] [blame] | 72 | dir = os.path.dirname(dir) |
| 73 | return keyval |
| 74 | |
| 75 | |
mbligh | e79ebb0 | 2008-04-17 15:39:22 +0000 | [diff] [blame] | 76 | class kernel(object): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 77 | """Represents a kernel.""" |
| 78 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 79 | def __init__(self, base, patches, kernel_hash): |
| 80 | self.base = base |
| 81 | self.patches = patches |
| 82 | self.kernel_hash = kernel_hash |
mbligh | e79ebb0 | 2008-04-17 15:39:22 +0000 | [diff] [blame] | 83 | |
| 84 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 85 | @staticmethod |
| 86 | def compute_hash(base, hashes): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 87 | """Compute a hash given the base string and hashes for each patch. |
| 88 | |
| 89 | @param base: A string representing the kernel base. |
| 90 | @param hashes: A list of hashes, where each hash is associated with a |
| 91 | patch of this kernel. |
| 92 | |
| 93 | @return A string representing the computed hash. |
| 94 | |
| 95 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 96 | key_string = ','.join([base] + hashes) |
lmr | 6f80e7a | 2010-02-04 03:18:28 +0000 | [diff] [blame] | 97 | return utils.hash('md5', key_string).hexdigest() |
jadmanski | 6e8bf75 | 2008-05-14 00:17:48 +0000 | [diff] [blame] | 98 | |
| 99 | |
mbligh | e79ebb0 | 2008-04-17 15:39:22 +0000 | [diff] [blame] | 100 | class test(object): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 101 | """Represents a test.""" |
| 102 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 103 | def __init__(self, subdir, testname, status, reason, test_kernel, |
| 104 | machine, started_time, finished_time, iterations, |
Dennis Jeffrey | 368c54b | 2013-07-24 11:19:03 -0700 | [diff] [blame] | 105 | attributes, perf_values, labels): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 106 | self.subdir = subdir |
| 107 | self.testname = testname |
| 108 | self.status = status |
| 109 | self.reason = reason |
| 110 | self.kernel = test_kernel |
| 111 | self.machine = machine |
| 112 | self.started_time = started_time |
| 113 | self.finished_time = finished_time |
| 114 | self.iterations = iterations |
| 115 | self.attributes = attributes |
Dennis Jeffrey | 368c54b | 2013-07-24 11:19:03 -0700 | [diff] [blame] | 116 | self.perf_values = perf_values |
jadmanski | 9b6babf | 2009-04-21 17:57:40 +0000 | [diff] [blame] | 117 | self.labels = labels |
mbligh | e79ebb0 | 2008-04-17 15:39:22 +0000 | [diff] [blame] | 118 | |
| 119 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 120 | @staticmethod |
| 121 | def load_iterations(keyval_path): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 122 | """Abstract method to load a list of iterations from a keyval file. |
| 123 | |
| 124 | @param keyval_path: String path to a keyval file. |
| 125 | |
| 126 | @return A list of iteration objects. |
| 127 | |
| 128 | """ |
Dennis Jeffrey | 05eb4b1 | 2013-07-17 11:11:52 -0700 | [diff] [blame] | 129 | raise NotImplementedError |
| 130 | |
| 131 | |
| 132 | @staticmethod |
| 133 | def load_perf_values(perf_values_file): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 134 | """Loads perf values from a perf measurements file. |
| 135 | |
| 136 | @param perf_values_file: The string path to a perf measurements file. |
| 137 | |
| 138 | @return A list of perf_value_iteration objects. |
| 139 | |
| 140 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 141 | raise NotImplementedError |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 142 | |
| 143 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 144 | @classmethod |
| 145 | def parse_test(cls, job, subdir, testname, status, reason, test_kernel, |
jadmanski | 74eebf3 | 2008-07-15 20:04:42 +0000 | [diff] [blame] | 146 | started_time, finished_time, existing_instance=None): |
Dennis Jeffrey | 05eb4b1 | 2013-07-17 11:11:52 -0700 | [diff] [blame] | 147 | """ |
| 148 | Parse test result files to construct a complete test instance. |
| 149 | |
| 150 | Given a job and the basic metadata about the test that can be |
| 151 | extracted from the status logs, parse the test result files (keyval |
| 152 | files and perf measurement files) and use them to construct a complete |
| 153 | test instance. |
| 154 | |
| 155 | @param job: A job object. |
| 156 | @param subdir: The string subdirectory name for the given test. |
| 157 | @param testname: The name of the test. |
| 158 | @param status: The status of the test. |
| 159 | @param reason: The reason string for the test. |
| 160 | @param test_kernel: The kernel of the test. |
| 161 | @param started_time: The start time of the test. |
| 162 | @param finished_time: The finish time of the test. |
| 163 | @param existing_instance: An existing test instance. |
| 164 | |
| 165 | @return A test instance that has the complete information. |
| 166 | |
| 167 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 168 | tko_utils.dprint("parsing test %s %s" % (subdir, testname)) |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 169 | |
Shuhei Takahashi | 0ae6c4d | 2018-07-31 16:36:08 +0900 | [diff] [blame] | 170 | if tast.is_tast_test(testname): |
| 171 | attributes, perf_values = tast.load_tast_test_aux_results(job, |
| 172 | testname) |
| 173 | iterations = [] |
| 174 | elif subdir: |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 175 | # Grab iterations from the results keyval. |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 176 | iteration_keyval = os.path.join(job.dir, subdir, |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 177 | 'results', 'keyval') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 178 | iterations = cls.load_iterations(iteration_keyval) |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 179 | |
Dennis Jeffrey | 05eb4b1 | 2013-07-17 11:11:52 -0700 | [diff] [blame] | 180 | # Grab perf values from the perf measurements file. |
| 181 | perf_values_file = os.path.join(job.dir, subdir, |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 182 | 'results', 'results-chart.json') |
| 183 | perf_values = {} |
| 184 | if os.path.exists(perf_values_file): |
| 185 | with open(perf_values_file, 'r') as fp: |
| 186 | contents = fp.read() |
| 187 | if contents: |
| 188 | perf_values = json.loads(contents) |
Dennis Jeffrey | 05eb4b1 | 2013-07-17 11:11:52 -0700 | [diff] [blame] | 189 | |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 190 | # Grab test attributes from the subdir keyval. |
| 191 | test_keyval = os.path.join(job.dir, subdir, 'keyval') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 192 | attributes = test.load_attributes(test_keyval) |
| 193 | else: |
| 194 | iterations = [] |
Keith Haddow | 1e5c701 | 2016-03-09 16:05:37 -0800 | [diff] [blame] | 195 | perf_values = {} |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 196 | attributes = {} |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 197 | |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 198 | # Grab test+host attributes from the host keyval. |
showard | 71b9431 | 2009-08-20 23:40:02 +0000 | [diff] [blame] | 199 | host_keyval = cls.parse_host_keyval(job.dir, job.machine) |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 200 | attributes.update(dict(('host-%s' % k, v) |
showard | 71b9431 | 2009-08-20 23:40:02 +0000 | [diff] [blame] | 201 | for k, v in host_keyval.iteritems())) |
jadmanski | b591fba | 2008-09-10 16:19:22 +0000 | [diff] [blame] | 202 | |
jadmanski | 74eebf3 | 2008-07-15 20:04:42 +0000 | [diff] [blame] | 203 | if existing_instance: |
| 204 | def constructor(*args, **dargs): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 205 | """Initializes an existing test instance.""" |
jadmanski | 74eebf3 | 2008-07-15 20:04:42 +0000 | [diff] [blame] | 206 | existing_instance.__init__(*args, **dargs) |
| 207 | return existing_instance |
| 208 | else: |
| 209 | constructor = cls |
Dennis Jeffrey | 368c54b | 2013-07-24 11:19:03 -0700 | [diff] [blame] | 210 | |
jadmanski | 74eebf3 | 2008-07-15 20:04:42 +0000 | [diff] [blame] | 211 | return constructor(subdir, testname, status, reason, test_kernel, |
| 212 | job.machine, started_time, finished_time, |
Dennis Jeffrey | 368c54b | 2013-07-24 11:19:03 -0700 | [diff] [blame] | 213 | iterations, attributes, perf_values, []) |
jadmanski | 74eebf3 | 2008-07-15 20:04:42 +0000 | [diff] [blame] | 214 | |
| 215 | |
| 216 | @classmethod |
| 217 | def parse_partial_test(cls, job, subdir, testname, reason, test_kernel, |
| 218 | started_time): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 219 | """ |
| 220 | Create a test instance representing a partial test result. |
| 221 | |
| 222 | Given a job and the basic metadata available when a test is |
jadmanski | 74eebf3 | 2008-07-15 20:04:42 +0000 | [diff] [blame] | 223 | started, create a test instance representing the partial result. |
| 224 | Assume that since the test is not complete there are no results files |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 225 | actually available for parsing. |
jadmanski | 74eebf3 | 2008-07-15 20:04:42 +0000 | [diff] [blame] | 226 | |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 227 | @param job: A job object. |
| 228 | @param subdir: The string subdirectory name for the given test. |
| 229 | @param testname: The name of the test. |
| 230 | @param reason: The reason string for the test. |
| 231 | @param test_kernel: The kernel of the test. |
| 232 | @param started_time: The start time of the test. |
| 233 | |
| 234 | @return A test instance that has partial test information. |
| 235 | |
| 236 | """ |
| 237 | tko_utils.dprint('parsing partial test %s %s' % (subdir, testname)) |
| 238 | |
| 239 | return cls(subdir, testname, 'RUNNING', reason, test_kernel, |
Dennis Jeffrey | 368c54b | 2013-07-24 11:19:03 -0700 | [diff] [blame] | 240 | job.machine, started_time, None, [], {}, [], []) |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 241 | |
| 242 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 243 | @staticmethod |
| 244 | def load_attributes(keyval_path): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 245 | """ |
| 246 | Load test attributes from a test keyval path. |
| 247 | |
| 248 | Load the test attributes into a dictionary from a test |
| 249 | keyval path. Does not assume that the path actually exists. |
| 250 | |
| 251 | @param keyval_path: The string path to a keyval file. |
| 252 | |
| 253 | @return A dictionary representing the test keyvals. |
| 254 | |
| 255 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 256 | if not os.path.exists(keyval_path): |
| 257 | return {} |
| 258 | return utils.read_keyval(keyval_path) |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 259 | |
| 260 | |
jadmanski | b591fba | 2008-09-10 16:19:22 +0000 | [diff] [blame] | 261 | @staticmethod |
Dan Shi | ba7b981 | 2016-06-30 10:55:58 -0700 | [diff] [blame] | 262 | def _parse_keyval(job_dir, sub_keyval_path): |
| 263 | """ |
| 264 | Parse a file of keyvals. |
| 265 | |
| 266 | @param job_dir: The string directory name of the associated job. |
| 267 | @param sub_keyval_path: Path to a keyval file relative to job_dir. |
| 268 | |
| 269 | @return A dictionary representing the keyvals. |
| 270 | |
| 271 | """ |
| 272 | # The "real" job dir may be higher up in the directory tree. |
| 273 | job_dir = tko_utils.find_toplevel_job_dir(job_dir) |
| 274 | if not job_dir: |
| 275 | return {} # We can't find a top-level job dir with job keyvals. |
| 276 | |
| 277 | # The keyval is <job_dir>/`sub_keyval_path` if it exists. |
| 278 | keyval_path = os.path.join(job_dir, sub_keyval_path) |
| 279 | if os.path.isfile(keyval_path): |
| 280 | return utils.read_keyval(keyval_path) |
| 281 | else: |
| 282 | return {} |
| 283 | |
| 284 | |
| 285 | @staticmethod |
jadmanski | b591fba | 2008-09-10 16:19:22 +0000 | [diff] [blame] | 286 | def parse_host_keyval(job_dir, hostname): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 287 | """ |
| 288 | Parse host keyvals. |
| 289 | |
| 290 | @param job_dir: The string directory name of the associated job. |
| 291 | @param hostname: The string hostname. |
| 292 | |
| 293 | @return A dictionary representing the host keyvals. |
| 294 | |
| 295 | """ |
Allen Li | 5977e5c | 2018-10-18 17:44:46 -0700 | [diff] [blame] | 296 | keyval_path = os.path.join('host_keyvals', hostname) |
Dan Shi | ba7b981 | 2016-06-30 10:55:58 -0700 | [diff] [blame] | 297 | # The host keyval is <job_dir>/host_keyvals/<hostname> if it exists. |
Allen Li | 5977e5c | 2018-10-18 17:44:46 -0700 | [diff] [blame] | 298 | # Otherwise we're running on Skylab which uses hostinfo. |
| 299 | if not os.path.exists(keyval_path): |
| 300 | try: |
| 301 | return _parse_hostinfo_keyval(job_dir, hostname) |
| 302 | except Exception as e: |
| 303 | # If anything goes wrong, log it and just use the old flow. |
| 304 | tko_utils.dprint("tried using hostinfo: %s" % e) |
| 305 | return test._parse_keyval(job_dir, keyval_path) |
jadmanski | b591fba | 2008-09-10 16:19:22 +0000 | [diff] [blame] | 306 | |
Dan Shi | ba7b981 | 2016-06-30 10:55:58 -0700 | [diff] [blame] | 307 | |
| 308 | @staticmethod |
| 309 | def parse_job_keyval(job_dir): |
| 310 | """ |
| 311 | Parse job keyvals. |
| 312 | |
| 313 | @param job_dir: The string directory name of the associated job. |
| 314 | |
| 315 | @return A dictionary representing the job keyvals. |
| 316 | |
| 317 | """ |
| 318 | # The job keyval is <job_dir>/keyval if it exists. |
| 319 | return test._parse_keyval(job_dir, 'keyval') |
jadmanski | b591fba | 2008-09-10 16:19:22 +0000 | [diff] [blame] | 320 | |
| 321 | |
Allen Li | 5977e5c | 2018-10-18 17:44:46 -0700 | [diff] [blame] | 322 | def _parse_hostinfo_keyval(job_dir, hostname): |
| 323 | """ |
| 324 | Parse host keyvals from hostinfo. |
| 325 | |
| 326 | @param job_dir: The string directory name of the associated job. |
| 327 | @param hostname: The string hostname. |
| 328 | |
| 329 | @return A dictionary representing the host keyvals. |
| 330 | |
| 331 | """ |
| 332 | # The hostinfo path looks like: |
| 333 | # host_info_store/dir_e499300b-8bba-4ad3-a404-8c12a9367f17/ |
| 334 | # chromeos6-row4-rack11-host6.store |
| 335 | # |
| 336 | # TODO(ayatane): We should pass hostinfo path explicitly. |
| 337 | subdir = 'host_info_store' |
Alex Zamorzaev | 6075525 | 2018-11-06 06:37:27 +0000 | [diff] [blame^] | 338 | subdir = os.path.join(subdir, os.listdir(subdir)[0]) |
| 339 | hostinfo_path = os.path.join(job_dir, subdir, hostname + '.store') |
Allen Li | 5977e5c | 2018-10-18 17:44:46 -0700 | [diff] [blame] | 340 | store = file_store.FileStore(hostinfo_path) |
| 341 | hostinfo = store.get() |
| 342 | # TODO(ayatane): Investigate if urllib.quote is better. |
| 343 | label_string = ','.join(label.replace(':', '%3A') |
| 344 | for label in hostinfo.labels) |
Allen Li | c3cdbdd | 2018-10-30 13:46:09 -0700 | [diff] [blame] | 345 | return { |
| 346 | 'host-labels': label_string, |
| 347 | 'host-platform': hostinfo.model, |
| 348 | } |
Allen Li | 5977e5c | 2018-10-18 17:44:46 -0700 | [diff] [blame] | 349 | |
| 350 | |
mbligh | e79ebb0 | 2008-04-17 15:39:22 +0000 | [diff] [blame] | 351 | class patch(object): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 352 | """Represents a patch.""" |
| 353 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 354 | def __init__(self, spec, reference, hash): |
| 355 | self.spec = spec |
| 356 | self.reference = reference |
| 357 | self.hash = hash |
mbligh | e79ebb0 | 2008-04-17 15:39:22 +0000 | [diff] [blame] | 358 | |
| 359 | |
| 360 | class iteration(object): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 361 | """Represents an iteration.""" |
| 362 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 363 | def __init__(self, index, attr_keyval, perf_keyval): |
| 364 | self.index = index |
| 365 | self.attr_keyval = attr_keyval |
| 366 | self.perf_keyval = perf_keyval |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 367 | |
| 368 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 369 | @staticmethod |
| 370 | def parse_line_into_dicts(line, attr_dict, perf_dict): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 371 | """ |
| 372 | Abstract method to parse a keyval line and insert it into a dictionary. |
| 373 | |
| 374 | @param line: The string line to parse. |
| 375 | @param attr_dict: Dictionary of generic iteration attributes. |
| 376 | @param perf_dict: Dictionary of iteration performance results. |
| 377 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 378 | """ |
| 379 | raise NotImplementedError |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 380 | |
| 381 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 382 | @classmethod |
| 383 | def load_from_keyval(cls, keyval_path): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 384 | """ |
| 385 | Load a list of iterations from an iteration keyval file. |
| 386 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 387 | Keyval data from separate iterations is separated by blank |
| 388 | lines. Makes use of the parse_line_into_dicts method to |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 389 | actually parse the individual lines. |
| 390 | |
| 391 | @param keyval_path: The string path to a keyval file. |
| 392 | |
| 393 | @return A list of iteration objects. |
| 394 | |
| 395 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 396 | if not os.path.exists(keyval_path): |
| 397 | return [] |
jadmanski | cc54917 | 2008-05-21 18:11:51 +0000 | [diff] [blame] | 398 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 399 | iterations = [] |
| 400 | index = 1 |
| 401 | attr, perf = {}, {} |
| 402 | for line in file(keyval_path): |
| 403 | line = line.strip() |
| 404 | if line: |
| 405 | cls.parse_line_into_dicts(line, attr, perf) |
| 406 | else: |
| 407 | iterations.append(cls(index, attr, perf)) |
| 408 | index += 1 |
| 409 | attr, perf = {}, {} |
| 410 | if attr or perf: |
| 411 | iterations.append(cls(index, attr, perf)) |
| 412 | return iterations |
Dennis Jeffrey | 05eb4b1 | 2013-07-17 11:11:52 -0700 | [diff] [blame] | 413 | |
| 414 | |
| 415 | class perf_value_iteration(object): |
Dennis Jeffrey | ccbc9d4 | 2013-07-23 12:16:45 -0700 | [diff] [blame] | 416 | """Represents a perf value iteration.""" |
| 417 | |
Dennis Jeffrey | 05eb4b1 | 2013-07-17 11:11:52 -0700 | [diff] [blame] | 418 | def __init__(self, index, perf_measurements): |
| 419 | """ |
| 420 | Initializes the perf values for a particular test iteration. |
| 421 | |
| 422 | @param index: The integer iteration number. |
| 423 | @param perf_measurements: A list of dictionaries, where each dictionary |
| 424 | contains the information for a measured perf metric from the |
| 425 | current iteration. |
| 426 | |
| 427 | """ |
| 428 | self.index = index |
| 429 | self.perf_measurements = perf_measurements |
| 430 | |
| 431 | |
| 432 | def add_measurement(self, measurement): |
| 433 | """ |
| 434 | Appends to the list of perf measurements for this iteration. |
| 435 | |
| 436 | @param measurement: A dictionary containing information for a measured |
| 437 | perf metric. |
| 438 | |
| 439 | """ |
| 440 | self.perf_measurements.append(measurement) |
| 441 | |
| 442 | |
| 443 | @staticmethod |
| 444 | def parse_line_into_dict(line): |
| 445 | """ |
| 446 | Abstract method to parse an individual perf measurement line. |
| 447 | |
| 448 | @param line: A string line from the perf measurement output file. |
| 449 | |
| 450 | @return A dicionary representing the information for a measured perf |
| 451 | metric from one line of the perf measurement output file, or an |
| 452 | empty dictionary if the line cannot be parsed successfully. |
| 453 | |
| 454 | """ |
| 455 | raise NotImplementedError |