Yunlian Jiang | 00cc30e | 2013-03-28 13:23:57 -0700 | [diff] [blame] | 1 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 2 | # Use of this source code is governed by a BSD-style license that can be |
| 3 | # found in the LICENSE file. |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 4 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 5 | """The experiment setting module.""" |
| 6 | |
Yunlian Jiang | 742ed2c | 2015-12-10 10:05:59 -0800 | [diff] [blame] | 7 | from __future__ import print_function |
| 8 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 9 | import os |
| 10 | import time |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 11 | |
cmtice | e5bc63b | 2015-05-27 16:59:37 -0700 | [diff] [blame] | 12 | import afe_lock_machine |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 13 | from threading import Lock |
cmtice | e5bc63b | 2015-05-27 16:59:37 -0700 | [diff] [blame] | 14 | |
Yunlian Jiang | 0d1a9f3 | 2015-12-09 10:47:11 -0800 | [diff] [blame] | 15 | from cros_utils import logger |
| 16 | from cros_utils import misc |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 17 | |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 18 | import benchmark_run |
Han Shen | 738e6de | 2015-12-07 13:22:25 -0800 | [diff] [blame] | 19 | from machine_manager import BadChecksum |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 20 | from machine_manager import MachineManager |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 21 | from machine_manager import MockMachineManager |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 22 | import test_flag |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 23 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 24 | class Experiment(object): |
| 25 | """Class representing an Experiment to be run.""" |
| 26 | |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 27 | def __init__(self, name, remote, working_directory, |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 28 | chromeos_root, cache_conditions, labels, benchmarks, |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 29 | experiment_file, email_to, acquire_timeout, log_dir, |
cmtice | 5c09fc2 | 2015-04-22 09:25:53 -0700 | [diff] [blame] | 30 | log_level, share_cache, results_directory, locks_directory): |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 31 | self.name = name |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 32 | self.working_directory = working_directory |
| 33 | self.remote = remote |
| 34 | self.chromeos_root = chromeos_root |
| 35 | self.cache_conditions = cache_conditions |
| 36 | self.experiment_file = experiment_file |
Ahmad Sharif | f395c26 | 2012-10-09 17:48:09 -0700 | [diff] [blame] | 37 | self.email_to = email_to |
Yunlian Jiang | 00cc30e | 2013-03-28 13:23:57 -0700 | [diff] [blame] | 38 | if not results_directory: |
| 39 | self.results_directory = os.path.join(self.working_directory, |
| 40 | self.name + "_results") |
| 41 | else: |
| 42 | self.results_directory = misc.CanonicalizePath(results_directory) |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 43 | self.log_dir = log_dir |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 44 | self.log_level = log_level |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 45 | self.labels = labels |
| 46 | self.benchmarks = benchmarks |
| 47 | self.num_complete = 0 |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 48 | self.num_run_complete = 0 |
cmtice | 1a22436 | 2014-10-16 15:49:56 -0700 | [diff] [blame] | 49 | self.share_cache = share_cache |
cmtice | 517dc98 | 2015-06-12 12:22:32 -0700 | [diff] [blame] | 50 | # If locks_directory (self.lock_dir) not blank, we will use the file |
| 51 | # locking mechanism; if it is blank then we will use the AFE server |
| 52 | # locking mechanism. |
| 53 | self.locks_dir = locks_directory |
cmtice | f3eb803 | 2015-07-27 13:55:52 -0700 | [diff] [blame] | 54 | self.locked_machines = [] |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 55 | |
Luis Lozano | dd41761 | 2015-12-08 12:08:44 -0800 | [diff] [blame] | 56 | if not remote: |
| 57 | raise RuntimeError("No remote hosts specified") |
| 58 | if not self.benchmarks: |
| 59 | raise RuntimeError("No benchmarks specified") |
| 60 | if not self.labels: |
| 61 | raise RuntimeError("No labels specified") |
| 62 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 63 | # We need one chromeos_root to run the benchmarks in, but it doesn't |
| 64 | # matter where it is, unless the ABIs are different. |
| 65 | if not chromeos_root: |
| 66 | for label in self.labels: |
| 67 | if label.chromeos_root: |
| 68 | chromeos_root = label.chromeos_root |
Luis Lozano | dd41761 | 2015-12-08 12:08:44 -0800 | [diff] [blame] | 69 | break |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 70 | if not chromeos_root: |
Luis Lozano | dd41761 | 2015-12-08 12:08:44 -0800 | [diff] [blame] | 71 | raise RuntimeError("No chromeos_root given and could not determine " |
| 72 | "one from the image path.") |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 73 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 74 | if test_flag.GetTestMode(): |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 75 | self.machine_manager = MockMachineManager(chromeos_root, acquire_timeout, |
cmtice | d96e457 | 2015-05-19 16:19:25 -0700 | [diff] [blame] | 76 | log_level, locks_directory) |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 77 | else: |
cmtice | 1390924 | 2014-03-11 13:38:07 -0700 | [diff] [blame] | 78 | self.machine_manager = MachineManager(chromeos_root, acquire_timeout, |
cmtice | 517dc98 | 2015-06-12 12:22:32 -0700 | [diff] [blame] | 79 | log_level, locks_directory) |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 80 | self.l = logger.GetLogger(log_dir) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 81 | |
Han Shen | f9b5035 | 2015-09-17 11:26:22 -0700 | [diff] [blame] | 82 | for machine in self.remote: |
| 83 | # machine_manager.AddMachine only adds reachable machines. |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 84 | self.machine_manager.AddMachine(machine) |
Han Shen | f9b5035 | 2015-09-17 11:26:22 -0700 | [diff] [blame] | 85 | # Now machine_manager._all_machines contains a list of reachable |
| 86 | # machines. This is a subset of self.remote. We make both lists the same. |
| 87 | self.remote = [m.name for m in self.machine_manager._all_machines] |
Caroline Tice | 51d7a9b | 2015-12-09 08:01:54 -0800 | [diff] [blame] | 88 | if not self.remote: |
| 89 | raise RuntimeError("No machine available for running experiment.") |
Han Shen | f9b5035 | 2015-09-17 11:26:22 -0700 | [diff] [blame] | 90 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 91 | for label in labels: |
Han Shen | f9b5035 | 2015-09-17 11:26:22 -0700 | [diff] [blame] | 92 | # We filter out label remotes that are not reachable (not in |
| 93 | # self.remote). So each label.remote is a sublist of experiment.remote. |
| 94 | label.remote = filter(lambda x: x in self.remote, label.remote) |
Han Shen | 738e6de | 2015-12-07 13:22:25 -0800 | [diff] [blame] | 95 | try: |
| 96 | self.machine_manager.ComputeCommonCheckSum(label) |
| 97 | except BadChecksum: |
| 98 | # Force same image on all machines, then we do checksum again. No |
| 99 | # bailout if checksums still do not match. |
| 100 | self.machine_manager.ForceSameImageToAllMachines(label) |
| 101 | self.machine_manager.ComputeCommonCheckSum(label) |
| 102 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 103 | self.machine_manager.ComputeCommonCheckSumString(label) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 104 | |
| 105 | self.start_time = None |
| 106 | self.benchmark_runs = self._GenerateBenchmarkRuns() |
| 107 | |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 108 | self._schedv2 = None |
| 109 | self._internal_counter_lock = Lock() |
| 110 | |
| 111 | def set_schedv2(self, schedv2): |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 112 | self._schedv2 = schedv2 |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 113 | |
| 114 | def schedv2(self): |
Caroline Tice | ddde505 | 2015-09-23 09:43:35 -0700 | [diff] [blame] | 115 | return self._schedv2 |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 116 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 117 | def _GenerateBenchmarkRuns(self): |
| 118 | """Generate benchmark runs from labels and benchmark defintions.""" |
| 119 | benchmark_runs = [] |
| 120 | for label in self.labels: |
| 121 | for benchmark in self.benchmarks: |
| 122 | for iteration in range(1, benchmark.iterations + 1): |
| 123 | |
| 124 | benchmark_run_name = "%s: %s (%s)" % (label.name, benchmark.name, |
| 125 | iteration) |
| 126 | full_name = "%s_%s_%s" % (label.name, benchmark.name, iteration) |
Luis Lozano | f81680c | 2013-03-15 14:44:13 -0700 | [diff] [blame] | 127 | logger_to_use = logger.Logger(self.log_dir, |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 128 | "run.%s" % (full_name), |
cmtice | 7789294 | 2014-03-18 13:47:17 -0700 | [diff] [blame] | 129 | True) |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 130 | benchmark_runs.append(benchmark_run.BenchmarkRun( |
| 131 | benchmark_run_name, |
| 132 | benchmark, |
| 133 | label, |
| 134 | iteration, |
| 135 | self.cache_conditions, |
| 136 | self.machine_manager, |
| 137 | logger_to_use, |
| 138 | self.log_level, |
| 139 | self.share_cache)) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 140 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 141 | return benchmark_runs |
| 142 | |
| 143 | def Build(self): |
| 144 | pass |
| 145 | |
| 146 | def Terminate(self): |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 147 | if self._schedv2 is not None: |
| 148 | self._schedv2.terminate() |
| 149 | else: |
| 150 | for t in self.benchmark_runs: |
| 151 | if t.isAlive(): |
| 152 | self.l.LogError("Terminating run: '%s'." % t.name) |
| 153 | t.Terminate() |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 154 | |
| 155 | def IsComplete(self): |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 156 | if self._schedv2: |
| 157 | return self._schedv2.is_complete() |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 158 | if self.active_threads: |
| 159 | for t in self.active_threads: |
| 160 | if t.isAlive(): |
| 161 | t.join(0) |
| 162 | if not t.isAlive(): |
| 163 | self.num_complete += 1 |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 164 | if not t.cache_hit: |
| 165 | self.num_run_complete += 1 |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 166 | self.active_threads.remove(t) |
| 167 | return False |
| 168 | return True |
| 169 | |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 170 | def BenchmarkRunFinished(self, br): |
Yunlian Jiang | 742ed2c | 2015-12-10 10:05:59 -0800 | [diff] [blame] | 171 | """Update internal counters after br finishes. |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 172 | |
Yunlian Jiang | 742ed2c | 2015-12-10 10:05:59 -0800 | [diff] [blame] | 173 | Note this is only used by schedv2 and is called by multiple threads. |
| 174 | Never throw any exception here. |
| 175 | """ |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 176 | |
Yunlian Jiang | 742ed2c | 2015-12-10 10:05:59 -0800 | [diff] [blame] | 177 | assert self._schedv2 is not None |
| 178 | with self._internal_counter_lock: |
| 179 | self.num_complete += 1 |
| 180 | if not br.cache_hit: |
| 181 | self.num_run_complete += 1 |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 182 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 183 | def Run(self): |
| 184 | self.start_time = time.time() |
Han Shen | ba64928 | 2015-08-05 17:19:55 -0700 | [diff] [blame] | 185 | if self._schedv2 is not None: |
| 186 | self._schedv2.run_sched() |
| 187 | else: |
| 188 | self.active_threads = [] |
| 189 | for benchmark_run in self.benchmark_runs: |
| 190 | # Set threads to daemon so program exits when ctrl-c is pressed. |
| 191 | benchmark_run.daemon = True |
| 192 | benchmark_run.start() |
| 193 | self.active_threads.append(benchmark_run) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 194 | |
| 195 | def SetCacheConditions(self, cache_conditions): |
| 196 | for benchmark_run in self.benchmark_runs: |
| 197 | benchmark_run.SetCacheConditions(cache_conditions) |
| 198 | |
| 199 | def Cleanup(self): |
cmtice | e5bc63b | 2015-05-27 16:59:37 -0700 | [diff] [blame] | 200 | """Make sure all machines are unlocked.""" |
cmtice | 517dc98 | 2015-06-12 12:22:32 -0700 | [diff] [blame] | 201 | if self.locks_dir: |
| 202 | # We are using the file locks mechanism, so call machine_manager.Cleanup |
| 203 | # to unlock everything. |
| 204 | self.machine_manager.Cleanup() |
| 205 | else: |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 206 | if test_flag.GetTestMode(): |
| 207 | return |
| 208 | |
cmtice | f3eb803 | 2015-07-27 13:55:52 -0700 | [diff] [blame] | 209 | all_machines = self.locked_machines |
| 210 | if not all_machines: |
| 211 | return |
| 212 | |
| 213 | # If we locked any machines earlier, make sure we unlock them now. |
cmtice | 517dc98 | 2015-06-12 12:22:32 -0700 | [diff] [blame] | 214 | lock_mgr = afe_lock_machine.AFELockManager(all_machines, "", |
| 215 | self.labels[0].chromeos_root, |
| 216 | None) |
| 217 | machine_states = lock_mgr.GetMachineStates("unlock") |
| 218 | for k, state in machine_states.iteritems(): |
| 219 | if state["locked"]: |
| 220 | lock_mgr.UpdateLockInAFE(False, k) |