Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | # Copyright 2015 Google Inc. All Rights Reserved. |
| 4 | |
| 5 | import mock |
| 6 | import unittest |
| 7 | import StringIO |
| 8 | |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 9 | import benchmark_run |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 10 | import machine_manager |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 11 | import schedv2 |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 12 | import test_flag |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 13 | from benchmark_run import MockBenchmarkRun |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 14 | from experiment_factory import ExperimentFactory |
| 15 | from experiment_file import ExperimentFile |
| 16 | from experiment_runner import ExperimentRunner |
| 17 | from machine_manager import MockCrosMachine |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame^] | 18 | from cros_utils import command_executer |
| 19 | from cros_utils.command_executer import CommandExecuter |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 20 | from experiment_runner_unittest import FakeLogger |
| 21 | from schedv2 import Schedv2 |
| 22 | |
| 23 | |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 24 | EXPERIMENT_FILE_1 = """\ |
| 25 | board: daisy |
| 26 | remote: chromeos-daisy1.cros chromeos-daisy2.cros |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 27 | |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 28 | benchmark: kraken { |
| 29 | suite: telemetry_Crosperf |
| 30 | iterations: 3 |
| 31 | } |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 32 | |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 33 | image1 { |
| 34 | chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin |
| 35 | remote: chromeos-daisy3.cros |
| 36 | } |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 37 | |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 38 | image2 { |
| 39 | chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin |
| 40 | remote: chromeos-daisy4.cros chromeos-daisy5.cros |
| 41 | } |
| 42 | """ |
| 43 | |
| 44 | |
| 45 | EXPERIMENT_FILE_WITH_FORMAT = """\ |
| 46 | board: daisy |
| 47 | remote: chromeos-daisy1.cros chromeos-daisy2.cros |
| 48 | |
| 49 | benchmark: kraken {{ |
| 50 | suite: telemetry_Crosperf |
| 51 | iterations: {kraken_iterations} |
| 52 | }} |
| 53 | |
| 54 | image1 {{ |
| 55 | chromeos_image: /chromeos/src/build/images/daisy/latest/cros_image1.bin |
| 56 | remote: chromeos-daisy3.cros |
| 57 | }} |
| 58 | |
| 59 | image2 {{ |
| 60 | chromeos_image: /chromeos/src/build/imaages/daisy/latest/cros_image2.bin |
| 61 | remote: chromeos-daisy4.cros chromeos-daisy5.cros |
| 62 | }} |
| 63 | """ |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 64 | |
| 65 | |
| 66 | class Schedv2Test(unittest.TestCase): |
| 67 | |
| 68 | mock_logger = FakeLogger() |
| 69 | mock_cmd_exec = mock.Mock(spec=CommandExecuter) |
| 70 | |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 71 | @mock.patch('benchmark_run.BenchmarkRun', |
| 72 | new=benchmark_run.MockBenchmarkRun) |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 73 | def _make_fake_experiment(self, expstr): |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 74 | """Create fake experiment from string. |
| 75 | |
| 76 | Note - we mock out BenchmarkRun in this step. |
| 77 | """ |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 78 | experiment_file = ExperimentFile(StringIO.StringIO(expstr)) |
| 79 | experiment = ExperimentFactory().GetExperiment( |
| 80 | experiment_file, working_directory="", log_dir="") |
| 81 | return experiment |
| 82 | |
| 83 | def test_remote(self): |
| 84 | """Test that remotes in labels are aggregated into experiment.remote.""" |
| 85 | |
| 86 | self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1) |
| 87 | self.exp.log_level = 'verbose' |
| 88 | schedv2 = Schedv2(self.exp) |
| 89 | self.assertIn('chromeos-daisy1.cros', self.exp.remote) |
| 90 | self.assertIn('chromeos-daisy2.cros', self.exp.remote) |
| 91 | self.assertIn('chromeos-daisy3.cros', self.exp.remote) |
| 92 | self.assertIn('chromeos-daisy4.cros', self.exp.remote) |
| 93 | self.assertIn('chromeos-daisy5.cros', self.exp.remote) |
| 94 | |
| 95 | def test_unreachable_remote(self): |
| 96 | """Test unreachable remotes are removed from experiment remote and |
| 97 | label.remote.""" |
| 98 | |
| 99 | def MockIsReachable(cm): |
| 100 | return (cm.name != 'chromeos-daisy3.cros' and |
| 101 | cm.name != 'chromeos-daisy5.cros') |
| 102 | |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 103 | with mock.patch('machine_manager.MockCrosMachine.IsReachable', |
| 104 | new=MockIsReachable) as f: |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 105 | self.exp = self._make_fake_experiment(EXPERIMENT_FILE_1) |
| 106 | self.assertIn('chromeos-daisy1.cros', self.exp.remote) |
| 107 | self.assertIn('chromeos-daisy2.cros', self.exp.remote) |
| 108 | self.assertNotIn('chromeos-daisy3.cros', self.exp.remote) |
| 109 | self.assertIn('chromeos-daisy4.cros', self.exp.remote) |
| 110 | self.assertNotIn('chromeos-daisy5.cros', self.exp.remote) |
| 111 | |
| 112 | for l in self.exp.labels: |
| 113 | if l.name == 'image2': |
| 114 | self.assertNotIn('chromeos-daisy5.cros', l.remote) |
| 115 | self.assertIn('chromeos-daisy4.cros', l.remote) |
| 116 | elif l.name == 'image1': |
| 117 | self.assertNotIn('chromeos-daisy3.cros', l.remote) |
Han Shen | e066297 | 2015-09-18 16:53:34 -0700 | [diff] [blame] | 118 | |
| 119 | @mock.patch('schedv2.BenchmarkRunCacheReader') |
| 120 | def test_BenchmarkRunCacheReader_1(self, reader): |
| 121 | """Test benchmarkrun set is split into 5 segments.""" |
| 122 | |
| 123 | self.exp = self._make_fake_experiment( |
| 124 | EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=9)) |
| 125 | schedv2 = Schedv2(self.exp) |
| 126 | # We have 9 * 2 == 18 brs, we use 5 threads, each reading 4, 4, 4, |
| 127 | # 4, 2 brs respectively. |
| 128 | # Assert that BenchmarkRunCacheReader() is called 5 times. |
| 129 | self.assertEquals(reader.call_count, 5) |
| 130 | # reader.call_args_list[n] - nth call. |
| 131 | # reader.call_args_list[n][0] - positioned args in nth call. |
| 132 | # reader.call_args_list[n][0][1] - the 2nd arg in nth call, |
| 133 | # that is 'br_list' in 'schedv2.BenchmarkRunCacheReader'. |
| 134 | self.assertEquals(len(reader.call_args_list[0][0][1]), 4) |
| 135 | self.assertEquals(len(reader.call_args_list[1][0][1]), 4) |
| 136 | self.assertEquals(len(reader.call_args_list[2][0][1]), 4) |
| 137 | self.assertEquals(len(reader.call_args_list[3][0][1]), 4) |
| 138 | self.assertEquals(len(reader.call_args_list[4][0][1]), 2) |
| 139 | |
| 140 | @mock.patch('schedv2.BenchmarkRunCacheReader') |
| 141 | def test_BenchmarkRunCacheReader_2(self, reader): |
| 142 | """Test benchmarkrun set is split into 4 segments.""" |
| 143 | |
| 144 | self.exp = self._make_fake_experiment( |
| 145 | EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=8)) |
| 146 | schedv2 = Schedv2(self.exp) |
| 147 | # We have 8 * 2 == 16 brs, we use 4 threads, each reading 4 brs. |
| 148 | self.assertEquals(reader.call_count, 4) |
| 149 | self.assertEquals(len(reader.call_args_list[0][0][1]), 4) |
| 150 | self.assertEquals(len(reader.call_args_list[1][0][1]), 4) |
| 151 | self.assertEquals(len(reader.call_args_list[2][0][1]), 4) |
| 152 | self.assertEquals(len(reader.call_args_list[3][0][1]), 4) |
| 153 | |
| 154 | @mock.patch('schedv2.BenchmarkRunCacheReader') |
| 155 | def test_BenchmarkRunCacheReader_3(self, reader): |
| 156 | """Test benchmarkrun set is split into 2 segments.""" |
| 157 | |
| 158 | self.exp = self._make_fake_experiment( |
| 159 | EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=3)) |
| 160 | schedv2 = Schedv2(self.exp) |
| 161 | # We have 3 * 2 == 6 brs, we use 2 threads. |
| 162 | self.assertEquals(reader.call_count, 2) |
| 163 | self.assertEquals(len(reader.call_args_list[0][0][1]), 3) |
| 164 | self.assertEquals(len(reader.call_args_list[1][0][1]), 3) |
| 165 | |
| 166 | @mock.patch('schedv2.BenchmarkRunCacheReader') |
| 167 | def test_BenchmarkRunCacheReader_4(self, reader): |
| 168 | """Test benchmarkrun set is not splitted.""" |
| 169 | |
| 170 | self.exp = self._make_fake_experiment( |
| 171 | EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=1)) |
| 172 | schedv2 = Schedv2(self.exp) |
| 173 | # We have 1 * 2 == 2 br, so only 1 instance. |
| 174 | self.assertEquals(reader.call_count, 1) |
| 175 | self.assertEquals(len(reader.call_args_list[0][0][1]), 2) |
| 176 | |
| 177 | def test_cachehit(self): |
| 178 | """Test cache-hit and none-cache-hit brs are properly organized.""" |
| 179 | |
| 180 | def MockReadCache(br): |
| 181 | br.cache_hit = (br.label.name == 'image2') |
| 182 | |
| 183 | with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache', |
| 184 | new=MockReadCache) as f: |
| 185 | # We have 2 * 30 brs, half of which are put into _cached_br_list. |
| 186 | self.exp = self._make_fake_experiment( |
| 187 | EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30)) |
| 188 | schedv2 = Schedv2(self.exp) |
| 189 | self.assertEquals(len(schedv2._cached_br_list), 30) |
| 190 | # The non-cache-hit brs are put into Schedv2._label_brl_map. |
| 191 | self.assertEquals(reduce(lambda a, x: a + len(x[1]), |
| 192 | schedv2._label_brl_map.iteritems(), 0), |
| 193 | 30) |
| 194 | |
| 195 | def test_nocachehit(self): |
| 196 | """Test no cache-hit.""" |
| 197 | |
| 198 | def MockReadCache(br): |
| 199 | br.cache_hit = False |
| 200 | |
| 201 | with mock.patch('benchmark_run.MockBenchmarkRun.ReadCache', |
| 202 | new=MockReadCache) as f: |
| 203 | # We have 2 * 30 brs, none of which are put into _cached_br_list. |
| 204 | self.exp = self._make_fake_experiment( |
| 205 | EXPERIMENT_FILE_WITH_FORMAT.format(kraken_iterations=30)) |
| 206 | schedv2 = Schedv2(self.exp) |
| 207 | self.assertEquals(len(schedv2._cached_br_list), 0) |
| 208 | # The non-cache-hit brs are put into Schedv2._label_brl_map. |
| 209 | self.assertEquals(reduce(lambda a, x: a + len(x[1]), |
| 210 | schedv2._label_brl_map.iteritems(), 0), |
| 211 | 60) |
Han Shen | 624bae7 | 2015-09-17 15:07:31 -0700 | [diff] [blame] | 212 | |
| 213 | |
| 214 | if __name__ == '__main__': |
| 215 | test_flag.SetTestMode(True) |
| 216 | unittest.main() |
| 217 | |