George Burgess IV | ed69676 | 2016-09-12 18:24:54 -0700 | [diff] [blame] | 1 | #!/usr/bin/env python2 |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 2 | |
Yunlian Jiang | 04dc5dc | 2013-04-23 15:05:05 -0700 | [diff] [blame] | 3 | # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. |
| 4 | # Use of this source code is governed by a BSD-style license that can be |
| 5 | # found in the LICENSE file. |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 6 | """Unit test for experiment_factory.py""" |
| 7 | |
| 8 | from __future__ import print_function |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 9 | |
| 10 | import StringIO |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 11 | import socket |
| 12 | import mock |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 13 | import unittest |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 14 | |
Yunlian Jiang | 0d1a9f3 | 2015-12-09 10:47:11 -0800 | [diff] [blame] | 15 | from cros_utils.file_utils import FileUtils |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 16 | |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 17 | from experiment_factory import ExperimentFactory |
| 18 | from experiment_file import ExperimentFile |
| 19 | import test_flag |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 20 | import benchmark |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 21 | import experiment_factory |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 22 | import settings_factory |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 23 | |
| 24 | EXPERIMENT_FILE_1 = """ |
| 25 | board: x86-alex |
| 26 | remote: chromeos-alex3 |
| 27 | |
| 28 | benchmark: PageCycler { |
| 29 | iterations: 3 |
| 30 | } |
| 31 | |
| 32 | image1 { |
| 33 | chromeos_image: /usr/local/google/cros_image1.bin |
| 34 | } |
| 35 | |
| 36 | image2 { |
| 37 | chromeos_image: /usr/local/google/cros_image2.bin |
| 38 | } |
| 39 | """ |
| 40 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 41 | # pylint: disable=too-many-function-args |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 42 | |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 43 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 44 | class ExperimentFactoryTest(unittest.TestCase): |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 45 | """Class for running experiment factory unittests.""" |
| 46 | |
| 47 | def setUp(self): |
| 48 | self.append_benchmark_call_args = [] |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 49 | |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 50 | def testLoadExperimentFile1(self): |
| 51 | experiment_file = ExperimentFile(StringIO.StringIO(EXPERIMENT_FILE_1)) |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 52 | exp = ExperimentFactory().GetExperiment( |
| 53 | experiment_file, working_directory='', log_dir='') |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 54 | self.assertEqual(exp.remote, ['chromeos-alex3']) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 55 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 56 | self.assertEqual(len(exp.benchmarks), 1) |
| 57 | self.assertEqual(exp.benchmarks[0].name, 'PageCycler') |
| 58 | self.assertEqual(exp.benchmarks[0].test_name, 'PageCycler') |
| 59 | self.assertEqual(exp.benchmarks[0].iterations, 3) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 60 | |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 61 | self.assertEqual(len(exp.labels), 2) |
| 62 | self.assertEqual(exp.labels[0].chromeos_image, |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 63 | '/usr/local/google/cros_image1.bin') |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 64 | self.assertEqual(exp.labels[0].board, 'x86-alex') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 65 | |
| 66 | def test_append_benchmark_set(self): |
| 67 | ef = ExperimentFactory() |
| 68 | |
| 69 | bench_list = [] |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 70 | ef.AppendBenchmarkSet(bench_list, experiment_factory.telemetry_perfv2_tests, |
| 71 | '', 1, False, '', 'telemetry_Crosperf', False, 0, |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 72 | False) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 73 | self.assertEqual( |
| 74 | len(bench_list), len(experiment_factory.telemetry_perfv2_tests)) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 75 | self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) |
| 76 | |
| 77 | bench_list = [] |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 78 | ef.AppendBenchmarkSet(bench_list, |
| 79 | experiment_factory.telemetry_pagecycler_tests, '', 1, |
| 80 | False, '', 'telemetry_Crosperf', False, 0, False) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 81 | self.assertEqual( |
| 82 | len(bench_list), len(experiment_factory.telemetry_pagecycler_tests)) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 83 | self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) |
| 84 | |
| 85 | bench_list = [] |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 86 | ef.AppendBenchmarkSet(bench_list, |
| 87 | experiment_factory.telemetry_toolchain_perf_tests, '', |
| 88 | 1, False, '', 'telemetry_Crosperf', False, 0, False) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 89 | self.assertEqual( |
| 90 | len(bench_list), len(experiment_factory.telemetry_toolchain_perf_tests)) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 91 | self.assertTrue(type(bench_list[0]) is benchmark.Benchmark) |
| 92 | |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 93 | @mock.patch.object(socket, 'gethostname') |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 94 | def test_get_experiment(self, mock_socket): |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 95 | |
| 96 | test_flag.SetTestMode(False) |
| 97 | self.append_benchmark_call_args = [] |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 98 | |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 99 | def FakeAppendBenchmarkSet(bench_list, set_list, args, iters, rm_ch, |
| 100 | perf_args, suite, show_all): |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 101 | 'Helper function for test_get_experiment' |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 102 | arg_list = [ |
| 103 | bench_list, set_list, args, iters, rm_ch, perf_args, suite, show_all |
| 104 | ] |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 105 | self.append_benchmark_call_args.append(arg_list) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 106 | |
| 107 | def FakeGetDefaultRemotes(board): |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 108 | if not board: |
| 109 | return [] |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 110 | return ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros'] |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 111 | |
Manoj Gupta | 4b13396 | 2016-11-02 10:03:17 -0700 | [diff] [blame^] | 112 | def FakeGetXbuddyPath(build, autotest_dir, board, chroot, log_level): |
| 113 | autotest_path = autotest_dir |
| 114 | if not autotest_path: |
| 115 | autotest_path = 'fake_autotest_path' |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 116 | if not build or not board or not chroot or not log_level: |
Manoj Gupta | 4b13396 | 2016-11-02 10:03:17 -0700 | [diff] [blame^] | 117 | return '', autotest_path |
| 118 | return 'fake_image_path', autotest_path |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 119 | |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 120 | ef = ExperimentFactory() |
Caroline Tice | 88272d4 | 2016-01-13 09:48:29 -0800 | [diff] [blame] | 121 | ef.AppendBenchmarkSet = FakeAppendBenchmarkSet |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 122 | ef.GetDefaultRemotes = FakeGetDefaultRemotes |
| 123 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 124 | label_settings = settings_factory.LabelSettings('image_label') |
| 125 | benchmark_settings = settings_factory.BenchmarkSettings('bench_test') |
| 126 | global_settings = settings_factory.GlobalSettings('test_name') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 127 | |
| 128 | label_settings.GetXbuddyPath = FakeGetXbuddyPath |
| 129 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 130 | mock_experiment_file = ExperimentFile(StringIO.StringIO('')) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 131 | mock_experiment_file.all_settings = [] |
| 132 | |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame] | 133 | test_flag.SetTestMode(True) |
| 134 | # Basic test. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 135 | global_settings.SetField('name', 'unittest_test') |
| 136 | global_settings.SetField('board', 'lumpy') |
| 137 | global_settings.SetField('remote', '123.45.67.89 123.45.76.80') |
| 138 | benchmark_settings.SetField('test_name', 'kraken') |
| 139 | benchmark_settings.SetField('suite', 'telemetry_Crosperf') |
| 140 | benchmark_settings.SetField('iterations', 1) |
| 141 | label_settings.SetField( |
| 142 | 'chromeos_image', |
| 143 | 'chromeos/src/build/images/lumpy/latest/chromiumos_test_image.bin') |
| 144 | label_settings.SetField('chrome_src', '/usr/local/google/home/chrome-top') |
Manoj Gupta | 4b13396 | 2016-11-02 10:03:17 -0700 | [diff] [blame^] | 145 | label_settings.SetField('autotest_path', '/tmp/autotest') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 146 | |
| 147 | mock_experiment_file.global_settings = global_settings |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 148 | mock_experiment_file.all_settings.append(label_settings) |
| 149 | mock_experiment_file.all_settings.append(benchmark_settings) |
| 150 | mock_experiment_file.all_settings.append(global_settings) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 151 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 152 | mock_socket.return_value = '' |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 153 | |
| 154 | # First test. General test. |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 155 | exp = ef.GetExperiment(mock_experiment_file, '', '') |
| 156 | self.assertEqual(exp.remote, ['123.45.67.89', '123.45.76.80']) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 157 | self.assertEqual(exp.cache_conditions, [0, 2, 1]) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 158 | self.assertEqual(exp.log_level, 'average') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 159 | |
| 160 | self.assertEqual(len(exp.benchmarks), 1) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 161 | self.assertEqual(exp.benchmarks[0].name, 'kraken') |
| 162 | self.assertEqual(exp.benchmarks[0].test_name, 'kraken') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 163 | self.assertEqual(exp.benchmarks[0].iterations, 1) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 164 | self.assertEqual(exp.benchmarks[0].suite, 'telemetry_Crosperf') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 165 | self.assertFalse(exp.benchmarks[0].show_all_results) |
| 166 | |
| 167 | self.assertEqual(len(exp.labels), 1) |
| 168 | self.assertEqual(exp.labels[0].chromeos_image, |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 169 | 'chromeos/src/build/images/lumpy/latest/' |
| 170 | 'chromiumos_test_image.bin') |
Manoj Gupta | 4b13396 | 2016-11-02 10:03:17 -0700 | [diff] [blame^] | 171 | self.assertEqual(exp.labels[0].autotest_path, '/tmp/autotest') |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 172 | self.assertEqual(exp.labels[0].board, 'lumpy') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 173 | |
| 174 | # Second test: Remotes listed in labels. |
Caroline Tice | 7057cf6 | 2015-12-10 12:09:40 -0800 | [diff] [blame] | 175 | test_flag.SetTestMode(True) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 176 | label_settings.SetField('remote', 'chromeos1.cros chromeos2.cros') |
| 177 | exp = ef.GetExperiment(mock_experiment_file, '', '') |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 178 | self.assertEqual( |
| 179 | exp.remote, |
| 180 | ['chromeos1.cros', 'chromeos2.cros', '123.45.67.89', '123.45.76.80']) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 181 | |
| 182 | # Third test: Automatic fixing of bad logging_level param: |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 183 | global_settings.SetField('logging_level', 'really loud!') |
| 184 | exp = ef.GetExperiment(mock_experiment_file, '', '') |
| 185 | self.assertEqual(exp.log_level, 'verbose') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 186 | |
| 187 | # Fourth test: Setting cache conditions; only 1 remote with "same_machine" |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 188 | global_settings.SetField('rerun_if_failed', 'true') |
| 189 | global_settings.SetField('rerun', 'true') |
| 190 | global_settings.SetField('same_machine', 'true') |
| 191 | global_settings.SetField('same_specs', 'true') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 192 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 193 | self.assertRaises(Exception, ef.GetExperiment, mock_experiment_file, '', '') |
| 194 | label_settings.SetField('remote', '') |
| 195 | global_settings.SetField('remote', '123.45.67.89') |
| 196 | exp = ef.GetExperiment(mock_experiment_file, '', '') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 197 | self.assertEqual(exp.cache_conditions, [0, 2, 3, 4, 6, 1]) |
| 198 | |
| 199 | # Fifth Test: Adding a second label; calling GetXbuddyPath; omitting all |
| 200 | # remotes (Call GetDefaultRemotes). |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 201 | mock_socket.return_value = 'test.corp.google.com' |
| 202 | global_settings.SetField('remote', '') |
| 203 | global_settings.SetField('same_machine', 'false') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 204 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 205 | label_settings_2 = settings_factory.LabelSettings('official_image_label') |
| 206 | label_settings_2.SetField('chromeos_root', 'chromeos') |
| 207 | label_settings_2.SetField('build', 'official-dev') |
Manoj Gupta | 4b13396 | 2016-11-02 10:03:17 -0700 | [diff] [blame^] | 208 | label_settings_2.SetField('autotest_path', '') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 209 | label_settings_2.GetXbuddyPath = FakeGetXbuddyPath |
| 210 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 211 | mock_experiment_file.all_settings.append(label_settings_2) |
| 212 | exp = ef.GetExperiment(mock_experiment_file, '', '') |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 213 | self.assertEqual(len(exp.labels), 2) |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 214 | self.assertEqual(exp.labels[1].chromeos_image, 'fake_image_path') |
Manoj Gupta | 4b13396 | 2016-11-02 10:03:17 -0700 | [diff] [blame^] | 215 | self.assertEqual(exp.labels[1].autotest_path, 'fake_autotest_path') |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 216 | self.assertEqual( |
| 217 | exp.remote, |
| 218 | ['fake_chromeos_machine1.cros', 'fake_chromeos_machine2.cros']) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 219 | |
| 220 | def test_get_default_remotes(self): |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 221 | board_list = [ |
| 222 | 'x86-alex', 'lumpy', 'elm', 'parrot', 'daisy', 'peach_pit', 'peppy', |
| 223 | 'squawks' |
| 224 | ] |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 225 | |
| 226 | ef = ExperimentFactory() |
| 227 | self.assertRaises(Exception, ef.GetDefaultRemotes, 'bad-board') |
| 228 | |
| 229 | # Verify that we have entries for every board, and that we get three |
| 230 | # machines back for each board. |
| 231 | for b in board_list: |
| 232 | remotes = ef.GetDefaultRemotes(b) |
Caroline Tice | d768dc1 | 2016-10-28 14:35:21 -0700 | [diff] [blame] | 233 | if b == 'elm': |
| 234 | self.assertEqual(len(remotes), 2) |
| 235 | else: |
| 236 | self.assertEqual(len(remotes), 3) |
cmtice | 2250df9 | 2014-05-09 13:58:58 -0700 | [diff] [blame] | 237 | |
Luis Lozano | f2a3ef4 | 2015-12-15 13:49:30 -0800 | [diff] [blame] | 238 | |
| 239 | if __name__ == '__main__': |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 240 | FileUtils.Configure(True) |
Ahmad Sharif | 4467f00 | 2012-12-20 12:09:49 -0800 | [diff] [blame] | 241 | test_flag.SetTestMode(True) |
Ahmad Sharif | 0dcbc4b | 2012-02-02 16:37:18 -0800 | [diff] [blame] | 242 | unittest.main() |