showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
| 3 | import logging, unittest |
| 4 | import common |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 5 | from autotest_lib.client.common_lib import enum, global_config |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 6 | from autotest_lib.database import database_connection |
| 7 | from autotest_lib.frontend import setup_django_environment |
| 8 | from autotest_lib.frontend.afe import frontend_test_utils |
| 9 | from autotest_lib.scheduler import drone_manager, email_manager, monitor_db |
| 10 | |
| 11 | # translations necessary for scheduler queries to work with SQLite |
| 12 | _re_translator = database_connection.TranslatingDatabase.make_regexp_translator |
| 13 | _DB_TRANSLATORS = ( |
| 14 | _re_translator(r'NOW\(\)', 'time("now")'), |
| 15 | # older SQLite doesn't support group_concat, so just don't bother until |
| 16 | # it arises in an important query |
| 17 | _re_translator(r'GROUP_CONCAT\((.*?)\)', r'\1'), |
| 18 | ) |
| 19 | |
| 20 | class NullMethodObject(object): |
| 21 | _NULL_METHODS = () |
| 22 | |
| 23 | def __init__(self): |
| 24 | def null_method(*args, **kwargs): |
| 25 | pass |
| 26 | |
| 27 | for method_name in self._NULL_METHODS: |
| 28 | setattr(self, method_name, null_method) |
| 29 | |
| 30 | class MockGlobalConfig(object): |
| 31 | def __init__(self): |
| 32 | self._config_info = {} |
| 33 | |
| 34 | |
| 35 | def set_config_value(self, section, key, value): |
| 36 | self._config_info[(section, key)] = value |
| 37 | |
| 38 | |
| 39 | def get_config_value(self, section, key, type=str, |
| 40 | default=None, allow_blank=False): |
| 41 | identifier = (section, key) |
| 42 | if identifier not in self._config_info: |
| 43 | raise RuntimeError('Unset global config value: %s' % (identifier,)) |
| 44 | return self._config_info[identifier] |
| 45 | |
| 46 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 47 | # the SpecialTask names here must match the suffixes used on the SpecialTask |
| 48 | # results directories |
| 49 | _PidfileType = enum.Enum('verify', 'cleanup', 'repair', 'job', 'gather', |
| 50 | 'parse') |
| 51 | |
| 52 | |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 53 | class MockDroneManager(NullMethodObject): |
| 54 | _NULL_METHODS = ('refresh', 'reinitialize_drones', |
| 55 | 'copy_to_results_repository') |
| 56 | |
| 57 | def __init__(self): |
| 58 | super(MockDroneManager, self).__init__() |
| 59 | # maps result_dir to set of tuples (file_path, file_contents) |
| 60 | self._attached_files = {} |
| 61 | # maps pidfile IDs to PidfileContents |
| 62 | self._pidfiles = {} |
| 63 | # pidfile IDs that haven't been created yet |
| 64 | self._future_pidfiles = [] |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 65 | # maps _PidfileType to the most recently created pidfile ID of that type |
| 66 | self._last_pidfile_id = {} |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 67 | # maps (working_directory, pidfile_name) to pidfile IDs |
| 68 | self._pidfile_index = {} |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 69 | # maps process to pidfile IDs |
| 70 | self._process_index = {} |
| 71 | # tracks pidfiles of processes that have been killed |
| 72 | self._killed_pidfiles = set() |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 73 | |
| 74 | |
| 75 | # utility APIs for use by the test |
| 76 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 77 | def finish_process(self, pidfile_type, exit_status=0): |
| 78 | pidfile_id = self._last_pidfile_id[pidfile_type] |
| 79 | self._set_pidfile_exit_status(pidfile_id, exit_status) |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 80 | |
| 81 | |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 82 | def _set_pidfile_exit_status(self, pidfile_id, exit_status): |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 83 | assert pidfile_id is not None |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 84 | contents = self._pidfiles[pidfile_id] |
| 85 | contents.exit_status = exit_status |
| 86 | contents.num_tests_failed = 0 |
| 87 | |
| 88 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 89 | def was_last_process_killed(self, pidfile_type): |
| 90 | pidfile_id = self._last_pidfile_id[pidfile_type] |
| 91 | return pidfile_id in self._killed_pidfiles |
| 92 | |
| 93 | |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 94 | # DroneManager emulation APIs for use by monitor_db |
| 95 | |
| 96 | def get_orphaned_autoserv_processes(self): |
| 97 | return set() |
| 98 | |
| 99 | |
| 100 | def total_running_processes(self): |
| 101 | return 0 |
| 102 | |
| 103 | |
| 104 | def max_runnable_processes(self): |
| 105 | return 100 |
| 106 | |
| 107 | |
| 108 | def execute_actions(self): |
| 109 | # executing an "execute_command" causes a pidfile to be created |
| 110 | for pidfile_id in self._future_pidfiles: |
| 111 | # Process objects are opaque to monitor_db |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 112 | process = object() |
| 113 | self._pidfiles[pidfile_id].process = process |
| 114 | self._process_index[process] = pidfile_id |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 115 | self._future_pidfiles = [] |
| 116 | |
| 117 | |
| 118 | def attach_file_to_execution(self, result_dir, file_contents, |
| 119 | file_path=None): |
| 120 | self._attached_files.setdefault(result_dir, set()).add((file_path, |
| 121 | file_contents)) |
| 122 | return 'attach_path' |
| 123 | |
| 124 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 125 | def _initialize_pidfile(self, pidfile_id): |
| 126 | if pidfile_id not in self._pidfiles: |
| 127 | self._pidfiles[pidfile_id] = drone_manager.PidfileContents() |
| 128 | |
| 129 | |
| 130 | _pidfile_type_map = { |
| 131 | monitor_db._AUTOSERV_PID_FILE: _PidfileType.JOB, |
| 132 | monitor_db._CRASHINFO_PID_FILE: _PidfileType.GATHER, |
| 133 | monitor_db._PARSER_PID_FILE: _PidfileType.PARSE, |
| 134 | } |
| 135 | |
| 136 | |
| 137 | def _set_last_pidfile(self, pidfile_id, working_directory, pidfile_name): |
| 138 | if working_directory.startswith('hosts/'): |
| 139 | # such paths look like hosts/host1/1-verify, we'll grab the end |
| 140 | type_string = working_directory.rsplit('-', 1)[1] |
| 141 | pidfile_type = _PidfileType.get_value(type_string) |
| 142 | else: |
| 143 | pidfile_type = self._pidfile_type_map[pidfile_name] |
| 144 | self._last_pidfile_id[pidfile_type] = pidfile_id |
| 145 | |
| 146 | |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 147 | def execute_command(self, command, working_directory, pidfile_name, |
| 148 | log_file=None, paired_with_pidfile=None): |
| 149 | # TODO: record this |
| 150 | pidfile_id = object() # PidfileIds are opaque to monitor_db |
| 151 | self._future_pidfiles.append(pidfile_id) |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 152 | self._initialize_pidfile(pidfile_id) |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 153 | self._pidfile_index[(working_directory, pidfile_name)] = pidfile_id |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 154 | self._set_last_pidfile(pidfile_id, working_directory, pidfile_name) |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 155 | return pidfile_id |
| 156 | |
| 157 | |
| 158 | def get_pidfile_contents(self, pidfile_id, use_second_read=False): |
| 159 | return self._pidfiles.get(pidfile_id, |
| 160 | drone_manager.PidfileContents()) |
| 161 | |
| 162 | |
| 163 | def is_process_running(self, process): |
| 164 | return True |
| 165 | |
| 166 | |
| 167 | def register_pidfile(self, pidfile_id): |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 168 | self._initialize_pidfile(pidfile_id) |
| 169 | |
| 170 | |
| 171 | def unregister_pidfile(self, pidfile_id): |
| 172 | # intentionally handle non-registered pidfiles silently |
| 173 | self._pidfiles.pop(pidfile_id, None) |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 174 | |
| 175 | |
| 176 | def absolute_path(self, path): |
| 177 | return 'absolute/' + path |
| 178 | |
| 179 | |
| 180 | def write_lines_to_file(self, file_path, lines, paired_with_process=None): |
| 181 | # TODO: record this |
| 182 | pass |
| 183 | |
| 184 | |
| 185 | def get_pidfile_id_from(self, execution_tag, pidfile_name): |
| 186 | return self._pidfile_index.get((execution_tag, pidfile_name), object()) |
| 187 | |
| 188 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 189 | def kill_process(self, process): |
| 190 | pidfile_id = self._process_index[process] |
| 191 | self._killed_pidfiles.add(pidfile_id) |
| 192 | self._set_pidfile_exit_status(pidfile_id, 271) |
| 193 | |
| 194 | |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 195 | class MockEmailManager(NullMethodObject): |
| 196 | _NULL_METHODS = ('send_queued_emails', 'send_email') |
| 197 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 198 | def enqueue_notify_email(self, subject, message): |
| 199 | logging.warn('enqueue_notify_email: %s', subject) |
| 200 | logging.warn(message) |
| 201 | |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 202 | |
| 203 | class SchedulerFunctionalTest(unittest.TestCase, |
| 204 | frontend_test_utils.FrontendTestMixin): |
| 205 | # some number of ticks after which the scheduler is presumed to have |
| 206 | # stabilized, given no external changes |
| 207 | _A_LOT_OF_TICKS = 10 |
| 208 | |
| 209 | def setUp(self): |
| 210 | self._frontend_common_setup() |
| 211 | self._set_stubs() |
| 212 | self._set_global_config_values() |
| 213 | self.dispatcher = monitor_db.Dispatcher() |
| 214 | |
| 215 | logging.basicConfig(level=logging.DEBUG) |
| 216 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 217 | self._initialize_test() |
| 218 | |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 219 | |
| 220 | def tearDown(self): |
| 221 | self._frontend_common_teardown() |
| 222 | |
| 223 | |
| 224 | def _set_stubs(self): |
| 225 | self.mock_config = MockGlobalConfig() |
| 226 | self.god.stub_with(global_config, 'global_config', self.mock_config) |
| 227 | |
| 228 | self.mock_drone_manager = MockDroneManager() |
| 229 | self.god.stub_with(monitor_db, '_drone_manager', |
| 230 | self.mock_drone_manager) |
| 231 | |
| 232 | self.mock_email_manager = MockEmailManager() |
| 233 | self.god.stub_with(email_manager, 'manager', self.mock_email_manager) |
| 234 | |
| 235 | self._database = ( |
| 236 | database_connection.TranslatingDatabase.get_test_database( |
| 237 | file_path=self._test_db_file, |
| 238 | translators=_DB_TRANSLATORS)) |
| 239 | self._database.connect(db_type='django') |
| 240 | self.god.stub_with(monitor_db, '_db', self._database) |
| 241 | |
| 242 | |
| 243 | def _set_global_config_values(self): |
| 244 | self.mock_config.set_config_value('SCHEDULER', 'pidfile_timeout_mins', |
| 245 | 1) |
| 246 | |
| 247 | |
| 248 | def _initialize_test(self): |
| 249 | self.dispatcher.initialize() |
| 250 | |
| 251 | |
| 252 | def _run_dispatcher(self): |
| 253 | for _ in xrange(self._A_LOT_OF_TICKS): |
| 254 | self.dispatcher.tick() |
| 255 | |
| 256 | |
| 257 | def test_idle(self): |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 258 | self._run_dispatcher() |
| 259 | |
| 260 | |
| 261 | def test_simple_job(self): |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 262 | self._create_job(hosts=[1]) |
| 263 | self._run_dispatcher() # launches verify |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 264 | self.mock_drone_manager.finish_process(_PidfileType.VERIFY) |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 265 | self._run_dispatcher() # launches job |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 266 | self.mock_drone_manager.finish_process(_PidfileType.JOB) |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 267 | self._run_dispatcher() # launches parsing + cleanup |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 268 | self._finish_parsing_and_cleanup() |
| 269 | |
| 270 | |
| 271 | def _finish_parsing_and_cleanup(self): |
| 272 | self.mock_drone_manager.finish_process(_PidfileType.CLEANUP) |
| 273 | self.mock_drone_manager.finish_process(_PidfileType.PARSE) |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 274 | self._run_dispatcher() |
| 275 | |
| 276 | |
showard | f85a0b7 | 2009-10-07 20:48:45 +0000 | [diff] [blame^] | 277 | def test_job_abort_in_verify(self): |
| 278 | job = self._create_job(hosts=[1]) |
| 279 | self._run_dispatcher() # launches verify |
| 280 | job.hostqueueentry_set.update(aborted=True) |
| 281 | self._run_dispatcher() # kills verify, launches cleanup |
| 282 | self.assert_(self.mock_drone_manager.was_last_process_killed( |
| 283 | _PidfileType.VERIFY)) |
| 284 | self.mock_drone_manager.finish_process(_PidfileType.CLEANUP) |
| 285 | self._run_dispatcher() |
| 286 | |
| 287 | |
| 288 | def test_job_abort(self): |
| 289 | job = self._create_job(hosts=[1]) |
| 290 | job.run_verify = False |
| 291 | job.save() |
| 292 | |
| 293 | self._run_dispatcher() # launches job |
| 294 | job.hostqueueentry_set.update(aborted=True) |
| 295 | self._run_dispatcher() # kills job, launches gathering |
| 296 | self.assert_(self.mock_drone_manager.was_last_process_killed( |
| 297 | _PidfileType.JOB)) |
| 298 | self.mock_drone_manager.finish_process(_PidfileType.GATHER) |
| 299 | self._run_dispatcher() # launches parsing + cleanup |
| 300 | self._finish_parsing_and_cleanup() |
| 301 | |
| 302 | |
| 303 | def test_no_pidfile_leaking(self): |
| 304 | self.test_simple_job() |
| 305 | self.assertEquals(self.mock_drone_manager._pidfiles, {}) |
| 306 | |
| 307 | self.test_job_abort_in_verify() |
| 308 | self.assertEquals(self.mock_drone_manager._pidfiles, {}) |
| 309 | |
| 310 | self.test_job_abort() |
| 311 | self.assertEquals(self.mock_drone_manager._pidfiles, {}) |
| 312 | |
| 313 | |
showard | 34ab099 | 2009-10-05 22:47:57 +0000 | [diff] [blame] | 314 | if __name__ == '__main__': |
| 315 | unittest.main() |