showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 3 | import unittest, time, subprocess, os, StringIO, tempfile, datetime, shutil |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 4 | import MySQLdb |
| 5 | import common |
showard | 364fe86 | 2008-10-17 02:01:16 +0000 | [diff] [blame] | 6 | from autotest_lib.frontend import setup_django_environment |
| 7 | from autotest_lib.frontend import setup_test_environment |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 8 | from autotest_lib.client.common_lib import global_config, host_protections |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 9 | from autotest_lib.client.common_lib.test_utils import mock |
showard | 442e71e | 2008-10-06 10:05:20 +0000 | [diff] [blame] | 10 | from autotest_lib.database import database_connection, migrate |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 11 | from autotest_lib.frontend import thread_local |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 12 | from autotest_lib.frontend.afe import models |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 13 | from autotest_lib.scheduler import monitor_db, drone_manager, email_manager |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 14 | |
| 15 | _DEBUG = False |
| 16 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 17 | class DummyAgent(object): |
| 18 | _is_running = False |
| 19 | _is_done = False |
| 20 | num_processes = 1 |
| 21 | host_ids = [] |
| 22 | queue_entry_ids = [] |
| 23 | |
| 24 | def is_running(self): |
| 25 | return self._is_running |
| 26 | |
| 27 | |
| 28 | def tick(self): |
| 29 | self._is_running = True |
| 30 | |
| 31 | |
| 32 | def is_done(self): |
| 33 | return self._is_done |
| 34 | |
| 35 | |
| 36 | def set_done(self, done): |
| 37 | self._is_done = done |
| 38 | self._is_running = not done |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 39 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 40 | |
| 41 | class IsRow(mock.argument_comparator): |
| 42 | def __init__(self, row_id): |
| 43 | self.row_id = row_id |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 44 | |
| 45 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 46 | def is_satisfied_by(self, parameter): |
| 47 | return list(parameter)[0] == self.row_id |
| 48 | |
| 49 | |
| 50 | def __str__(self): |
| 51 | return 'row with id %s' % self.row_id |
| 52 | |
| 53 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 54 | class BaseSchedulerTest(unittest.TestCase): |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 55 | _config_section = 'AUTOTEST_WEB' |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 56 | _test_db_initialized = False |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 57 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 58 | def _do_query(self, sql): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 59 | self._database.execute(sql) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 60 | |
| 61 | |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 62 | @classmethod |
| 63 | def _initialize_test_db(cls): |
| 64 | if cls._test_db_initialized: |
| 65 | return |
| 66 | temp_fd, cls._test_db_file = tempfile.mkstemp(suffix='.monitor_test') |
| 67 | os.close(temp_fd) |
showard | 364fe86 | 2008-10-17 02:01:16 +0000 | [diff] [blame] | 68 | setup_test_environment.set_test_database(cls._test_db_file) |
| 69 | setup_test_environment.run_syncdb() |
| 70 | cls._test_db_backup = setup_test_environment.backup_test_database() |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 71 | cls._test_db_initialized = True |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 72 | |
| 73 | |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 74 | def _open_test_db(self): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 75 | self._initialize_test_db() |
showard | 364fe86 | 2008-10-17 02:01:16 +0000 | [diff] [blame] | 76 | setup_test_environment.restore_test_database(self._test_db_backup) |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 77 | self._database = ( |
| 78 | database_connection.DatabaseConnection.get_test_database( |
| 79 | self._test_db_file)) |
| 80 | self._database.connect() |
| 81 | self._database.debug = _DEBUG |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 82 | |
| 83 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 84 | def _close_test_db(self): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 85 | self._database.disconnect() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 86 | |
| 87 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 88 | def _set_monitor_stubs(self): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 89 | monitor_db._db = self._database |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 90 | monitor_db._drone_manager._results_dir = '/test/path' |
| 91 | monitor_db._drone_manager._temporary_directory = '/test/path/tmp' |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 92 | |
| 93 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 94 | def _fill_in_test_data(self): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 95 | user = models.User.objects.create(login='my_user') |
| 96 | acl_group = models.AclGroup.objects.create(name='my_acl') |
| 97 | acl_group.users.add(user) |
| 98 | |
| 99 | hosts = [models.Host.objects.create(hostname=hostname) for hostname in |
| 100 | ('host1', 'host2', 'host3', 'host4')] |
| 101 | acl_group.hosts = hosts |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 102 | models.AclGroup.smart_get('Everyone').hosts = [] |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 103 | |
| 104 | labels = [models.Label.objects.create(name=name) for name in |
| 105 | ('label1', 'label2', 'label3')] |
| 106 | labels[2].only_if_needed = True |
| 107 | labels[2].save() |
| 108 | hosts[0].labels.add(labels[0]) |
| 109 | hosts[1].labels.add(labels[1]) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 110 | |
| 111 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 112 | def _setup_dummy_user(self): |
| 113 | user = models.User.objects.create(login='dummy', access_level=100) |
| 114 | thread_local.set_user(user) |
| 115 | |
| 116 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 117 | def setUp(self): |
| 118 | self.god = mock.mock_god() |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 119 | self._open_test_db() |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 120 | self._fill_in_test_data() |
| 121 | self._set_monitor_stubs() |
| 122 | self._dispatcher = monitor_db.Dispatcher() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 123 | self._setup_dummy_user() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 124 | |
| 125 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 126 | def tearDown(self): |
| 127 | self._close_test_db() |
| 128 | self.god.unstub_all() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 129 | |
| 130 | |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 131 | def _create_job(self, hosts=[], metahosts=[], priority=0, active=False, |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 132 | synchronous=False): |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 133 | synch_count = synchronous and 2 or 1 |
showard | 12bc8a8 | 2008-10-09 16:49:53 +0000 | [diff] [blame] | 134 | created_on = datetime.datetime(2008, 1, 1) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 135 | status = models.HostQueueEntry.Status.QUEUED |
| 136 | if active: |
| 137 | status = models.HostQueueEntry.Status.RUNNING |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 138 | job = models.Job.objects.create( |
| 139 | name='test', owner='my_user', priority=priority, |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 140 | synch_count=synch_count, created_on=created_on, |
showard | 0fc3830 | 2008-10-23 00:44:07 +0000 | [diff] [blame] | 141 | reboot_before=models.RebootBefore.NEVER) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 142 | for host_id in hosts: |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 143 | models.HostQueueEntry.objects.create(job=job, priority=priority, |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 144 | host_id=host_id, status=status) |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 145 | models.IneligibleHostQueue.objects.create(job=job, host_id=host_id) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 146 | for label_id in metahosts: |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 147 | models.HostQueueEntry.objects.create(job=job, priority=priority, |
| 148 | meta_host_id=label_id, |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 149 | status=status) |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 150 | return job |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 151 | |
| 152 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 153 | def _create_job_simple(self, hosts, use_metahost=False, |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 154 | priority=0, active=False): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 155 | 'An alternative interface to _create_job' |
| 156 | args = {'hosts' : [], 'metahosts' : []} |
| 157 | if use_metahost: |
| 158 | args['metahosts'] = hosts |
| 159 | else: |
| 160 | args['hosts'] = hosts |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 161 | return self._create_job(priority=priority, active=active, **args) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 162 | |
| 163 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 164 | def _update_hqe(self, set, where=''): |
| 165 | query = 'UPDATE host_queue_entries SET ' + set |
| 166 | if where: |
| 167 | query += ' WHERE ' + where |
| 168 | self._do_query(query) |
| 169 | |
| 170 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 171 | class DispatcherSchedulingTest(BaseSchedulerTest): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 172 | _jobs_scheduled = [] |
| 173 | |
| 174 | def _set_monitor_stubs(self): |
| 175 | super(DispatcherSchedulingTest, self)._set_monitor_stubs() |
| 176 | def run_stub(hqe_self, assigned_host=None): |
| 177 | hqe_self.set_status('Starting') |
| 178 | if hqe_self.meta_host: |
| 179 | host = assigned_host |
| 180 | else: |
| 181 | host = hqe_self.host |
| 182 | self._record_job_scheduled(hqe_self.job.id, host.id) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 183 | return DummyAgent() |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 184 | monitor_db.HostQueueEntry.run = run_stub |
| 185 | |
| 186 | |
| 187 | def _record_job_scheduled(self, job_id, host_id): |
| 188 | record = (job_id, host_id) |
| 189 | self.assert_(record not in self._jobs_scheduled, |
| 190 | 'Job %d scheduled on host %d twice' % |
| 191 | (job_id, host_id)) |
| 192 | self._jobs_scheduled.append(record) |
| 193 | |
| 194 | |
| 195 | def _assert_job_scheduled_on(self, job_id, host_id): |
| 196 | record = (job_id, host_id) |
| 197 | self.assert_(record in self._jobs_scheduled, |
| 198 | 'Job %d not scheduled on host %d as expected\n' |
| 199 | 'Jobs scheduled: %s' % |
| 200 | (job_id, host_id, self._jobs_scheduled)) |
| 201 | self._jobs_scheduled.remove(record) |
| 202 | |
| 203 | |
| 204 | def _check_for_extra_schedulings(self): |
| 205 | if len(self._jobs_scheduled) != 0: |
| 206 | self.fail('Extra jobs scheduled: ' + |
| 207 | str(self._jobs_scheduled)) |
| 208 | |
| 209 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 210 | def _convert_jobs_to_metahosts(self, *job_ids): |
| 211 | sql_tuple = '(' + ','.join(str(i) for i in job_ids) + ')' |
| 212 | self._do_query('UPDATE host_queue_entries SET ' |
| 213 | 'meta_host=host_id, host_id=NULL ' |
| 214 | 'WHERE job_id IN ' + sql_tuple) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 215 | |
| 216 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 217 | def _lock_host(self, host_id): |
| 218 | self._do_query('UPDATE hosts SET locked=1 WHERE id=' + |
| 219 | str(host_id)) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 220 | |
| 221 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 222 | def setUp(self): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 223 | super(DispatcherSchedulingTest, self).setUp() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 224 | self._jobs_scheduled = [] |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 225 | |
| 226 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 227 | def _test_basic_scheduling_helper(self, use_metahosts): |
| 228 | 'Basic nonmetahost scheduling' |
| 229 | self._create_job_simple([1], use_metahosts) |
| 230 | self._create_job_simple([2], use_metahosts) |
| 231 | self._dispatcher._schedule_new_jobs() |
| 232 | self._assert_job_scheduled_on(1, 1) |
| 233 | self._assert_job_scheduled_on(2, 2) |
| 234 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 235 | |
| 236 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 237 | def _test_priorities_helper(self, use_metahosts): |
| 238 | 'Test prioritization ordering' |
| 239 | self._create_job_simple([1], use_metahosts) |
| 240 | self._create_job_simple([2], use_metahosts) |
| 241 | self._create_job_simple([1,2], use_metahosts) |
| 242 | self._create_job_simple([1], use_metahosts, priority=1) |
| 243 | self._dispatcher._schedule_new_jobs() |
| 244 | self._assert_job_scheduled_on(4, 1) # higher priority |
| 245 | self._assert_job_scheduled_on(2, 2) # earlier job over later |
| 246 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 247 | |
| 248 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 249 | def _test_hosts_ready_helper(self, use_metahosts): |
| 250 | """ |
| 251 | Only hosts that are status=Ready, unlocked and not invalid get |
| 252 | scheduled. |
| 253 | """ |
| 254 | self._create_job_simple([1], use_metahosts) |
| 255 | self._do_query('UPDATE hosts SET status="Running" WHERE id=1') |
| 256 | self._dispatcher._schedule_new_jobs() |
| 257 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 258 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 259 | self._do_query('UPDATE hosts SET status="Ready", locked=1 ' |
| 260 | 'WHERE id=1') |
| 261 | self._dispatcher._schedule_new_jobs() |
| 262 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 263 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 264 | self._do_query('UPDATE hosts SET locked=0, invalid=1 ' |
| 265 | 'WHERE id=1') |
| 266 | self._dispatcher._schedule_new_jobs() |
showard | 5df2b19 | 2008-07-03 19:51:57 +0000 | [diff] [blame] | 267 | if not use_metahosts: |
| 268 | self._assert_job_scheduled_on(1, 1) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 269 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 270 | |
| 271 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 272 | def _test_hosts_idle_helper(self, use_metahosts): |
| 273 | 'Only idle hosts get scheduled' |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 274 | self._create_job(hosts=[1], active=True) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 275 | self._create_job_simple([1], use_metahosts) |
| 276 | self._dispatcher._schedule_new_jobs() |
| 277 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 278 | |
| 279 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 280 | def _test_obey_ACLs_helper(self, use_metahosts): |
| 281 | self._do_query('DELETE FROM acl_groups_hosts WHERE host_id=1') |
| 282 | self._create_job_simple([1], use_metahosts) |
| 283 | self._dispatcher._schedule_new_jobs() |
| 284 | self._check_for_extra_schedulings() |
| 285 | |
| 286 | |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 287 | def _test_only_if_needed_labels_helper(self, use_metahosts): |
| 288 | # apply only_if_needed label3 to host1 |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 289 | label3 = models.Label.smart_get('label3') |
| 290 | models.Host.smart_get('host1').labels.add(label3) |
| 291 | |
| 292 | job = self._create_job_simple([1], use_metahosts) |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 293 | # if the job doesn't depend on label3, there should be no scheduling |
| 294 | self._dispatcher._schedule_new_jobs() |
| 295 | self._check_for_extra_schedulings() |
| 296 | |
| 297 | # now make the job depend on label3 |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 298 | job.dependency_labels.add(label3) |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 299 | self._dispatcher._schedule_new_jobs() |
| 300 | self._assert_job_scheduled_on(1, 1) |
| 301 | self._check_for_extra_schedulings() |
| 302 | |
| 303 | if use_metahosts: |
| 304 | # should also work if the metahost is the only_if_needed label |
| 305 | self._do_query('DELETE FROM jobs_dependency_labels') |
| 306 | self._create_job(metahosts=[3]) |
| 307 | self._dispatcher._schedule_new_jobs() |
| 308 | self._assert_job_scheduled_on(2, 1) |
| 309 | self._check_for_extra_schedulings() |
| 310 | |
| 311 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 312 | def test_basic_scheduling(self): |
| 313 | self._test_basic_scheduling_helper(False) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 314 | |
| 315 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 316 | def test_priorities(self): |
| 317 | self._test_priorities_helper(False) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 318 | |
| 319 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 320 | def test_hosts_ready(self): |
| 321 | self._test_hosts_ready_helper(False) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 322 | |
| 323 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 324 | def test_hosts_idle(self): |
| 325 | self._test_hosts_idle_helper(False) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 326 | |
| 327 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 328 | def test_obey_ACLs(self): |
| 329 | self._test_obey_ACLs_helper(False) |
| 330 | |
| 331 | |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 332 | def test_only_if_needed_labels(self): |
| 333 | self._test_only_if_needed_labels_helper(False) |
| 334 | |
| 335 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 336 | def test_non_metahost_on_invalid_host(self): |
| 337 | """ |
| 338 | Non-metahost entries can get scheduled on invalid hosts (this is how |
| 339 | one-time hosts work). |
| 340 | """ |
| 341 | self._do_query('UPDATE hosts SET invalid=1') |
| 342 | self._test_basic_scheduling_helper(False) |
| 343 | |
| 344 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 345 | def test_metahost_scheduling(self): |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 346 | """ |
| 347 | Basic metahost scheduling |
| 348 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 349 | self._test_basic_scheduling_helper(True) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 350 | |
| 351 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 352 | def test_metahost_priorities(self): |
| 353 | self._test_priorities_helper(True) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 354 | |
| 355 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 356 | def test_metahost_hosts_ready(self): |
| 357 | self._test_hosts_ready_helper(True) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 358 | |
| 359 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 360 | def test_metahost_hosts_idle(self): |
| 361 | self._test_hosts_idle_helper(True) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 362 | |
| 363 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 364 | def test_metahost_obey_ACLs(self): |
| 365 | self._test_obey_ACLs_helper(True) |
| 366 | |
| 367 | |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 368 | def test_metahost_only_if_needed_labels(self): |
| 369 | self._test_only_if_needed_labels_helper(True) |
| 370 | |
| 371 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 372 | def test_nonmetahost_over_metahost(self): |
| 373 | """ |
| 374 | Non-metahost entries should take priority over metahost entries |
| 375 | for the same host |
| 376 | """ |
| 377 | self._create_job(metahosts=[1]) |
| 378 | self._create_job(hosts=[1]) |
| 379 | self._dispatcher._schedule_new_jobs() |
| 380 | self._assert_job_scheduled_on(2, 1) |
| 381 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 382 | |
| 383 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 384 | def test_metahosts_obey_blocks(self): |
| 385 | """ |
| 386 | Metahosts can't get scheduled on hosts already scheduled for |
| 387 | that job. |
| 388 | """ |
| 389 | self._create_job(metahosts=[1], hosts=[1]) |
| 390 | # make the nonmetahost entry complete, so the metahost can try |
| 391 | # to get scheduled |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 392 | self._update_hqe(set='complete = 1', where='host_id=1') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 393 | self._dispatcher._schedule_new_jobs() |
| 394 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 395 | |
| 396 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 397 | def test_only_schedule_queued_entries(self): |
| 398 | self._create_job(metahosts=[1]) |
| 399 | self._update_hqe(set='active=1, host_id=2') |
| 400 | self._dispatcher._schedule_new_jobs() |
| 401 | self._check_for_extra_schedulings() |
| 402 | |
| 403 | |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 404 | def test_no_ready_hosts(self): |
| 405 | self._create_job(hosts=[1]) |
| 406 | self._do_query('UPDATE hosts SET status="Repair Failed"') |
| 407 | self._dispatcher._schedule_new_jobs() |
| 408 | self._check_for_extra_schedulings() |
| 409 | |
| 410 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 411 | class DispatcherThrottlingTest(BaseSchedulerTest): |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 412 | """ |
| 413 | Test that the dispatcher throttles: |
| 414 | * total number of running processes |
| 415 | * number of processes started per cycle |
| 416 | """ |
| 417 | _MAX_RUNNING = 3 |
| 418 | _MAX_STARTED = 2 |
| 419 | |
| 420 | def setUp(self): |
| 421 | super(DispatcherThrottlingTest, self).setUp() |
| 422 | self._dispatcher.max_running_processes = self._MAX_RUNNING |
| 423 | self._dispatcher.max_processes_started_per_cycle = self._MAX_STARTED |
| 424 | |
| 425 | |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 426 | def _setup_some_agents(self, num_agents): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 427 | self._agents = [DummyAgent() for i in xrange(num_agents)] |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 428 | self._dispatcher._agents = list(self._agents) |
| 429 | |
| 430 | |
| 431 | def _run_a_few_cycles(self): |
| 432 | for i in xrange(4): |
| 433 | self._dispatcher._handle_agents() |
| 434 | |
| 435 | |
| 436 | def _assert_agents_started(self, indexes, is_started=True): |
| 437 | for i in indexes: |
| 438 | self.assert_(self._agents[i].is_running() == is_started, |
| 439 | 'Agent %d %sstarted' % |
| 440 | (i, is_started and 'not ' or '')) |
| 441 | |
| 442 | |
| 443 | def _assert_agents_not_started(self, indexes): |
| 444 | self._assert_agents_started(indexes, False) |
| 445 | |
| 446 | |
| 447 | def test_throttle_total(self): |
| 448 | self._setup_some_agents(4) |
| 449 | self._run_a_few_cycles() |
| 450 | self._assert_agents_started([0, 1, 2]) |
| 451 | self._assert_agents_not_started([3]) |
| 452 | |
| 453 | |
| 454 | def test_throttle_per_cycle(self): |
| 455 | self._setup_some_agents(3) |
| 456 | self._dispatcher._handle_agents() |
| 457 | self._assert_agents_started([0, 1]) |
| 458 | self._assert_agents_not_started([2]) |
| 459 | |
| 460 | |
| 461 | def test_throttle_with_synchronous(self): |
| 462 | self._setup_some_agents(2) |
| 463 | self._agents[0].num_processes = 3 |
| 464 | self._run_a_few_cycles() |
| 465 | self._assert_agents_started([0]) |
| 466 | self._assert_agents_not_started([1]) |
| 467 | |
| 468 | |
| 469 | def test_large_agent_starvation(self): |
| 470 | """ |
| 471 | Ensure large agents don't get starved by lower-priority agents. |
| 472 | """ |
| 473 | self._setup_some_agents(3) |
| 474 | self._agents[1].num_processes = 3 |
| 475 | self._run_a_few_cycles() |
| 476 | self._assert_agents_started([0]) |
| 477 | self._assert_agents_not_started([1, 2]) |
| 478 | |
| 479 | self._agents[0].set_done(True) |
| 480 | self._run_a_few_cycles() |
| 481 | self._assert_agents_started([1]) |
| 482 | self._assert_agents_not_started([2]) |
| 483 | |
| 484 | |
| 485 | def test_zero_process_agent(self): |
| 486 | self._setup_some_agents(5) |
| 487 | self._agents[4].num_processes = 0 |
| 488 | self._run_a_few_cycles() |
| 489 | self._assert_agents_started([0, 1, 2, 4]) |
| 490 | self._assert_agents_not_started([3]) |
| 491 | |
| 492 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 493 | class FindAbortTest(BaseSchedulerTest): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 494 | """ |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 495 | Test the dispatcher abort functionality. |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 496 | """ |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 497 | def _check_abort_agent(self, agent, entry_id): |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 498 | self.assert_(isinstance(agent, monitor_db.Agent)) |
| 499 | tasks = list(agent.queue.queue) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 500 | self.assertEquals(len(tasks), 1) |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 501 | abort = tasks[0] |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 502 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 503 | self.assert_(isinstance(abort, monitor_db.AbortTask)) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 504 | self.assertEquals(abort.queue_entry.id, entry_id) |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 505 | |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 506 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 507 | def _check_host_agent(self, agent, host_id): |
| 508 | self.assert_(isinstance(agent, monitor_db.Agent)) |
| 509 | tasks = list(agent.queue.queue) |
| 510 | self.assertEquals(len(tasks), 2) |
| 511 | cleanup, verify = tasks |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 512 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 513 | self.assert_(isinstance(cleanup, monitor_db.CleanupTask)) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 514 | self.assertEquals(cleanup.host.id, host_id) |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 515 | |
| 516 | self.assert_(isinstance(verify, monitor_db.VerifyTask)) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 517 | self.assertEquals(verify.host.id, host_id) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 518 | |
| 519 | |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 520 | def _check_agents(self, agents, include_host_tasks): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 521 | agents = list(agents) |
| 522 | if include_host_tasks: |
| 523 | self.assertEquals(len(agents), 4) |
| 524 | self._check_host_agent(agents.pop(0), 1) |
| 525 | self._check_host_agent(agents.pop(1), 2) |
| 526 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 527 | self.assertEquals(len(agents), 2) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 528 | self._check_abort_agent(agents[0], 1) |
| 529 | self._check_abort_agent(agents[1], 2) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 530 | |
| 531 | |
| 532 | def test_find_aborting_inactive(self): |
| 533 | self._create_job(hosts=[1, 2]) |
| 534 | self._update_hqe(set='status="Abort"') |
| 535 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 536 | self._dispatcher._find_aborting() |
| 537 | |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 538 | self._check_agents(self._dispatcher._agents, include_host_tasks=False) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 539 | self.god.check_playback() |
| 540 | |
| 541 | |
| 542 | def test_find_aborting_active(self): |
| 543 | self._create_job(hosts=[1, 2]) |
| 544 | self._update_hqe(set='status="Abort", active=1') |
| 545 | # have to make an Agent for the active HQEs |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 546 | agent = self.god.create_mock_class(monitor_db.Agent, 'old_agent') |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 547 | agent.host_ids = agent.queue_entry_ids = [1, 2] |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 548 | self._dispatcher.add_agent(agent) |
| 549 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 550 | self._dispatcher._find_aborting() |
| 551 | |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 552 | self._check_agents(self._dispatcher._agents, include_host_tasks=True) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 553 | self.god.check_playback() |
| 554 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 555 | # ensure agent gets aborted |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 556 | abort1 = self._dispatcher._agents[1].queue.queue[0] |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 557 | self.assertEquals(abort1.agents_to_abort, [agent]) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 558 | abort2 = self._dispatcher._agents[3].queue.queue[0] |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 559 | self.assertEquals(abort2.agents_to_abort, []) |
| 560 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 561 | |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 562 | class JobTimeoutTest(BaseSchedulerTest): |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 563 | def _test_synch_start_timeout_helper(self, expect_abort, |
| 564 | set_created_on=True, set_active=True, |
| 565 | set_acl=True): |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 566 | self._dispatcher.synch_job_start_timeout_minutes = 60 |
| 567 | job = self._create_job(hosts=[1, 2]) |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 568 | if set_active: |
| 569 | hqe = job.hostqueueentry_set.filter(host__id=1)[0] |
| 570 | hqe.status = 'Pending' |
| 571 | hqe.active = 1 |
| 572 | hqe.save() |
| 573 | |
| 574 | everyone_acl = models.AclGroup.smart_get('Everyone') |
| 575 | host1 = models.Host.smart_get(1) |
| 576 | if set_acl: |
| 577 | everyone_acl.hosts.add(host1) |
| 578 | else: |
| 579 | everyone_acl.hosts.remove(host1) |
| 580 | |
| 581 | job.created_on = datetime.datetime.now() |
| 582 | if set_created_on: |
| 583 | job.created_on -= datetime.timedelta(minutes=100) |
| 584 | job.save() |
| 585 | |
| 586 | self._dispatcher._abort_jobs_past_synch_start_timeout() |
| 587 | |
| 588 | for hqe in job.hostqueueentry_set.all(): |
| 589 | if expect_abort: |
| 590 | self.assert_(hqe.status in ('Abort', 'Aborted'), hqe.status) |
| 591 | else: |
| 592 | self.assert_(hqe.status not in ('Abort', 'Aborted'), hqe.status) |
| 593 | |
| 594 | |
| 595 | def test_synch_start_timeout_helper(self): |
| 596 | # no abort if any of the condition aren't met |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 597 | self._test_synch_start_timeout_helper(False, set_created_on=False) |
| 598 | self._test_synch_start_timeout_helper(False, set_active=False) |
| 599 | self._test_synch_start_timeout_helper(False, set_acl=False) |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 600 | # abort if all conditions are met |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 601 | self._test_synch_start_timeout_helper(True) |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 602 | |
| 603 | |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 604 | class PidfileRunMonitorTest(unittest.TestCase): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 605 | execution_tag = 'test_tag' |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 606 | pid = 12345 |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 607 | process = drone_manager.Process('myhost', pid) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 608 | num_tests_failed = 1 |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 609 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 610 | def setUp(self): |
| 611 | self.god = mock.mock_god() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 612 | self.mock_drone_manager = self.god.create_mock_class( |
| 613 | drone_manager.DroneManager, 'drone_manager') |
| 614 | self.god.stub_with(monitor_db, '_drone_manager', |
| 615 | self.mock_drone_manager) |
| 616 | self.god.stub_function(email_manager.manager, 'enqueue_notify_email') |
| 617 | |
| 618 | self.pidfile_id = object() |
| 619 | |
| 620 | self.mock_drone_manager.get_pidfile_id_from.expect_call( |
| 621 | self.execution_tag).and_return(self.pidfile_id) |
| 622 | self.mock_drone_manager.register_pidfile.expect_call(self.pidfile_id) |
| 623 | |
| 624 | self.monitor = monitor_db.PidfileRunMonitor() |
| 625 | self.monitor.attach_to_existing_process(self.execution_tag) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 626 | |
| 627 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 628 | def tearDown(self): |
| 629 | self.god.unstub_all() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 630 | |
| 631 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 632 | def setup_pidfile(self, pid=None, exit_code=None, tests_failed=None, |
| 633 | use_second_read=False): |
| 634 | contents = drone_manager.PidfileContents() |
| 635 | if pid is not None: |
| 636 | contents.process = drone_manager.Process('myhost', pid) |
| 637 | contents.exit_status = exit_code |
| 638 | contents.num_tests_failed = tests_failed |
| 639 | self.mock_drone_manager.get_pidfile_contents.expect_call( |
| 640 | self.pidfile_id, use_second_read=use_second_read).and_return( |
| 641 | contents) |
| 642 | |
| 643 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 644 | def set_not_yet_run(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 645 | self.setup_pidfile() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 646 | |
| 647 | |
showard | 3dd6b88 | 2008-10-27 19:21:39 +0000 | [diff] [blame] | 648 | def set_empty_pidfile(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 649 | self.setup_pidfile() |
showard | 3dd6b88 | 2008-10-27 19:21:39 +0000 | [diff] [blame] | 650 | |
| 651 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 652 | def set_running(self, use_second_read=False): |
| 653 | self.setup_pidfile(self.pid, use_second_read=use_second_read) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 654 | |
| 655 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 656 | def set_complete(self, error_code, use_second_read=False): |
| 657 | self.setup_pidfile(self.pid, error_code, self.num_tests_failed, |
| 658 | use_second_read=use_second_read) |
| 659 | |
| 660 | |
| 661 | def _check_monitor(self, expected_pid, expected_exit_status, |
| 662 | expected_num_tests_failed): |
| 663 | if expected_pid is None: |
| 664 | self.assertEquals(self.monitor._state.process, None) |
| 665 | else: |
| 666 | self.assertEquals(self.monitor._state.process.pid, expected_pid) |
| 667 | self.assertEquals(self.monitor._state.exit_status, expected_exit_status) |
| 668 | self.assertEquals(self.monitor._state.num_tests_failed, |
| 669 | expected_num_tests_failed) |
| 670 | |
| 671 | |
| 672 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 673 | |
| 674 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 675 | def _test_read_pidfile_helper(self, expected_pid, expected_exit_status, |
| 676 | expected_num_tests_failed): |
| 677 | self.monitor._read_pidfile() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 678 | self._check_monitor(expected_pid, expected_exit_status, |
| 679 | expected_num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 680 | |
| 681 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 682 | def _get_expected_tests_failed(self, expected_exit_status): |
| 683 | if expected_exit_status is None: |
| 684 | expected_tests_failed = None |
| 685 | else: |
| 686 | expected_tests_failed = self.num_tests_failed |
| 687 | return expected_tests_failed |
| 688 | |
| 689 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 690 | def test_read_pidfile(self): |
| 691 | self.set_not_yet_run() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 692 | self._test_read_pidfile_helper(None, None, None) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 693 | |
showard | 3dd6b88 | 2008-10-27 19:21:39 +0000 | [diff] [blame] | 694 | self.set_empty_pidfile() |
| 695 | self._test_read_pidfile_helper(None, None, None) |
| 696 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 697 | self.set_running() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 698 | self._test_read_pidfile_helper(self.pid, None, None) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 699 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 700 | self.set_complete(123) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 701 | self._test_read_pidfile_helper(self.pid, 123, self.num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 702 | |
| 703 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 704 | def test_read_pidfile_error(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 705 | self.mock_drone_manager.get_pidfile_contents.expect_call( |
| 706 | self.pidfile_id, use_second_read=False).and_return( |
| 707 | drone_manager.InvalidPidfile('error')) |
| 708 | self.assertRaises(monitor_db.PidfileRunMonitor._PidfileException, |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 709 | self.monitor._read_pidfile) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 710 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 711 | |
| 712 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 713 | def setup_is_running(self, is_running): |
| 714 | self.mock_drone_manager.is_process_running.expect_call( |
| 715 | self.process).and_return(is_running) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 716 | |
| 717 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 718 | def _test_get_pidfile_info_helper(self, expected_pid, expected_exit_status, |
| 719 | expected_num_tests_failed): |
| 720 | self.monitor._get_pidfile_info() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 721 | self._check_monitor(expected_pid, expected_exit_status, |
| 722 | expected_num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 723 | |
| 724 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 725 | def test_get_pidfile_info(self): |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 726 | """ |
| 727 | normal cases for get_pidfile_info |
| 728 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 729 | # running |
| 730 | self.set_running() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 731 | self.setup_is_running(True) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 732 | self._test_get_pidfile_info_helper(self.pid, None, None) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 733 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 734 | # exited during check |
| 735 | self.set_running() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 736 | self.setup_is_running(False) |
| 737 | self.set_complete(123, use_second_read=True) # pidfile gets read again |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 738 | self._test_get_pidfile_info_helper(self.pid, 123, self.num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 739 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 740 | # completed |
| 741 | self.set_complete(123) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 742 | self._test_get_pidfile_info_helper(self.pid, 123, self.num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 743 | |
| 744 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 745 | def test_get_pidfile_info_running_no_proc(self): |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 746 | """ |
| 747 | pidfile shows process running, but no proc exists |
| 748 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 749 | # running but no proc |
| 750 | self.set_running() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 751 | self.setup_is_running(False) |
| 752 | self.set_running(use_second_read=True) |
| 753 | email_manager.manager.enqueue_notify_email.expect_call( |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 754 | mock.is_string_comparator(), mock.is_string_comparator()) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 755 | self._test_get_pidfile_info_helper(self.pid, 1, 0) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 756 | self.assertTrue(self.monitor.lost_process) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 757 | |
| 758 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 759 | def test_get_pidfile_info_not_yet_run(self): |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 760 | """ |
| 761 | pidfile hasn't been written yet |
| 762 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 763 | self.set_not_yet_run() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 764 | self._test_get_pidfile_info_helper(None, None, None) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 765 | |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 766 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 767 | def test_process_failed_to_write_pidfile(self): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 768 | self.set_not_yet_run() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 769 | email_manager.manager.enqueue_notify_email.expect_call( |
| 770 | mock.is_string_comparator(), mock.is_string_comparator()) |
| 771 | dummy_process = drone_manager.Process('dummy', 12345) |
| 772 | self.mock_drone_manager.get_dummy_process.expect_call().and_return( |
| 773 | dummy_process) |
| 774 | self.monitor._start_time = time.time() - monitor_db.PIDFILE_TIMEOUT - 1 |
| 775 | self._test_get_pidfile_info_helper(12345, 1, 0) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 776 | |
| 777 | |
| 778 | class AgentTest(unittest.TestCase): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 779 | def setUp(self): |
| 780 | self.god = mock.mock_god() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 781 | |
| 782 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 783 | def tearDown(self): |
| 784 | self.god.unstub_all() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 785 | |
| 786 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 787 | def _create_mock_task(self, name): |
| 788 | task = self.god.create_mock_class(monitor_db.AgentTask, name) |
| 789 | task.host_ids = task.queue_entry_ids = [] |
| 790 | return task |
| 791 | |
| 792 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 793 | def test_agent(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 794 | task1 = self._create_mock_task('task1') |
| 795 | task2 = self._create_mock_task('task2') |
| 796 | task3 = self._create_mock_task('task3') |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 797 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 798 | task1.start.expect_call() |
| 799 | task1.is_done.expect_call().and_return(False) |
| 800 | task1.poll.expect_call() |
| 801 | task1.is_done.expect_call().and_return(True) |
| 802 | task1.is_done.expect_call().and_return(True) |
| 803 | task1.success = True |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 804 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 805 | task2.start.expect_call() |
| 806 | task2.is_done.expect_call().and_return(True) |
| 807 | task2.is_done.expect_call().and_return(True) |
| 808 | task2.success = False |
| 809 | task2.failure_tasks = [task3] |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 810 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 811 | task3.start.expect_call() |
| 812 | task3.is_done.expect_call().and_return(True) |
| 813 | task3.is_done.expect_call().and_return(True) |
| 814 | task3.success = True |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 815 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 816 | agent = monitor_db.Agent([task1, task2]) |
| 817 | agent.dispatcher = object() |
| 818 | agent.start() |
| 819 | while not agent.is_done(): |
| 820 | agent.tick() |
| 821 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 822 | |
| 823 | |
| 824 | class AgentTasksTest(unittest.TestCase): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 825 | TEMP_DIR = '/abspath/tempdir' |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 826 | RESULTS_DIR = '/results/dir' |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 827 | HOSTNAME = 'myhost' |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 828 | DUMMY_PROCESS = object() |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 829 | HOST_PROTECTION = host_protections.default |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 830 | PIDFILE_ID = object() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 831 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 832 | def setUp(self): |
| 833 | self.god = mock.mock_god() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 834 | self.god.stub_with(drone_manager.DroneManager, 'get_temporary_path', |
| 835 | mock.mock_function('get_temporary_path', |
| 836 | default_return_val='tempdir')) |
| 837 | self.god.stub_function(drone_manager.DroneManager, |
| 838 | 'copy_to_results_repository') |
| 839 | self.god.stub_function(drone_manager.DroneManager, |
| 840 | 'get_pidfile_id_from') |
| 841 | |
| 842 | def dummy_absolute_path(self, path): |
| 843 | return '/abspath/' + path |
| 844 | self.god.stub_with(drone_manager.DroneManager, 'absolute_path', |
| 845 | dummy_absolute_path) |
| 846 | |
| 847 | self.god.stub_class_method(monitor_db.PidfileRunMonitor, 'run') |
| 848 | self.god.stub_class_method(monitor_db.PidfileRunMonitor, 'exit_code') |
| 849 | self.god.stub_class_method(monitor_db.PidfileRunMonitor, 'get_process') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 850 | self.host = self.god.create_mock_class(monitor_db.Host, 'host') |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 851 | self.host.id = 1 |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 852 | self.host.hostname = self.HOSTNAME |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 853 | self.host.protection = self.HOST_PROTECTION |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 854 | self.queue_entry = self.god.create_mock_class( |
| 855 | monitor_db.HostQueueEntry, 'queue_entry') |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 856 | self.job = self.god.create_mock_class(monitor_db.Job, 'job') |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 857 | self.queue_entry.id = 1 |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 858 | self.queue_entry.job = self.job |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 859 | self.queue_entry.host = self.host |
| 860 | self.queue_entry.meta_host = None |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 861 | |
| 862 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 863 | def tearDown(self): |
| 864 | self.god.unstub_all() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 865 | |
| 866 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 867 | def run_task(self, task, success): |
| 868 | """ |
| 869 | Do essentially what an Agent would do, but protect againt |
| 870 | infinite looping from test errors. |
| 871 | """ |
| 872 | if not getattr(task, 'agent', None): |
| 873 | task.agent = object() |
| 874 | task.start() |
| 875 | count = 0 |
| 876 | while not task.is_done(): |
| 877 | count += 1 |
| 878 | if count > 10: |
| 879 | print 'Task failed to finish' |
| 880 | # in case the playback has clues to why it |
| 881 | # failed |
| 882 | self.god.check_playback() |
| 883 | self.fail() |
| 884 | task.poll() |
| 885 | self.assertEquals(task.success, success) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 886 | |
| 887 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 888 | def setup_run_monitor(self, exit_status, copy_log_file=True): |
| 889 | monitor_db.PidfileRunMonitor.run.expect_call( |
| 890 | mock.is_instance_comparator(list), |
| 891 | 'tempdir', |
| 892 | nice_level=monitor_db.AUTOSERV_NICE_LEVEL, |
| 893 | log_file=mock.anything_comparator()) |
| 894 | monitor_db.PidfileRunMonitor.exit_code.expect_call() |
| 895 | monitor_db.PidfileRunMonitor.exit_code.expect_call().and_return( |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 896 | exit_status) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 897 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 898 | if copy_log_file: |
| 899 | self._setup_move_logfile() |
| 900 | |
| 901 | |
| 902 | def _setup_move_logfile(self, include_destination=False): |
| 903 | monitor_db.PidfileRunMonitor.get_process.expect_call().and_return( |
| 904 | self.DUMMY_PROCESS) |
| 905 | if include_destination: |
| 906 | drone_manager.DroneManager.copy_to_results_repository.expect_call( |
| 907 | self.DUMMY_PROCESS, mock.is_string_comparator(), |
| 908 | destination_path=mock.is_string_comparator()) |
| 909 | else: |
| 910 | drone_manager.DroneManager.copy_to_results_repository.expect_call( |
| 911 | self.DUMMY_PROCESS, mock.is_string_comparator()) |
| 912 | |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 913 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 914 | def _test_repair_task_helper(self, success): |
| 915 | self.host.set_status.expect_call('Repairing') |
| 916 | if success: |
| 917 | self.setup_run_monitor(0) |
| 918 | self.host.set_status.expect_call('Ready') |
| 919 | else: |
| 920 | self.setup_run_monitor(1) |
| 921 | self.host.set_status.expect_call('Repair Failed') |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 922 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 923 | task = monitor_db.RepairTask(self.host) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 924 | self.assertEquals(task.failure_tasks, []) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 925 | self.run_task(task, success) |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 926 | |
| 927 | expected_protection = host_protections.Protection.get_string( |
| 928 | host_protections.default) |
mbligh | 3e0f7e0 | 2008-07-28 19:42:01 +0000 | [diff] [blame] | 929 | expected_protection = host_protections.Protection.get_attr_name( |
| 930 | expected_protection) |
| 931 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 932 | self.assertTrue(set(task.cmd) >= |
| 933 | set([monitor_db._autoserv_path, '-p', '-R', '-m', |
| 934 | self.HOSTNAME, '-r', self.TEMP_DIR, |
| 935 | '--host-protection', expected_protection])) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 936 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 937 | |
| 938 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 939 | def test_repair_task(self): |
| 940 | self._test_repair_task_helper(True) |
| 941 | self._test_repair_task_helper(False) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 942 | |
| 943 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 944 | def test_repair_task_with_queue_entry(self): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 945 | self.host.set_status.expect_call('Repairing') |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 946 | self.queue_entry.requeue.expect_call() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 947 | self.setup_run_monitor(1) |
| 948 | self.host.set_status.expect_call('Repair Failed') |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 949 | self.queue_entry.handle_host_failure.expect_call() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 950 | |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 951 | task = monitor_db.RepairTask(self.host, self.queue_entry) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 952 | self.run_task(task, False) |
| 953 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 954 | |
| 955 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 956 | def setup_verify_expects(self, success, use_queue_entry): |
| 957 | if use_queue_entry: |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 958 | self.queue_entry.set_status.expect_call('Verifying') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 959 | self.host.set_status.expect_call('Verifying') |
| 960 | if success: |
| 961 | self.setup_run_monitor(0) |
| 962 | self.host.set_status.expect_call('Ready') |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 963 | if use_queue_entry: |
| 964 | self.queue_entry.on_pending.expect_call() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 965 | else: |
| 966 | self.setup_run_monitor(1) |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 967 | if use_queue_entry and not self.queue_entry.meta_host: |
| 968 | self.queue_entry.set_execution_subdir.expect_call() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 969 | self.queue_entry.execution_tag.expect_call().and_return('tag') |
| 970 | self._setup_move_logfile(include_destination=True) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 971 | |
| 972 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 973 | def _check_verify_failure_tasks(self, verify_task): |
| 974 | self.assertEquals(len(verify_task.failure_tasks), 1) |
| 975 | repair_task = verify_task.failure_tasks[0] |
| 976 | self.assert_(isinstance(repair_task, monitor_db.RepairTask)) |
| 977 | self.assertEquals(verify_task.host, repair_task.host) |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 978 | if verify_task.queue_entry: |
| 979 | self.assertEquals(repair_task.queue_entry, verify_task.queue_entry) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 980 | else: |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 981 | self.assertEquals(repair_task.queue_entry, None) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 982 | |
| 983 | |
| 984 | def _test_verify_task_helper(self, success, use_queue_entry=False, |
| 985 | use_meta_host=False): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 986 | self.setup_verify_expects(success, use_queue_entry) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 987 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 988 | if use_queue_entry: |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 989 | task = monitor_db.VerifyTask(queue_entry=self.queue_entry) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 990 | else: |
| 991 | task = monitor_db.VerifyTask(host=self.host) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 992 | self._check_verify_failure_tasks(task) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 993 | self.run_task(task, success) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 994 | self.assertTrue(set(task.cmd) >= |
| 995 | set([monitor_db._autoserv_path, '-p', '-v', '-m', |
| 996 | self.HOSTNAME, '-r', self.TEMP_DIR])) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 997 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 998 | |
| 999 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1000 | def test_verify_task_with_host(self): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 1001 | self._test_verify_task_helper(True) |
| 1002 | self._test_verify_task_helper(False) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 1003 | |
| 1004 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1005 | def test_verify_task_with_queue_entry(self): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 1006 | self._test_verify_task_helper(True, use_queue_entry=True) |
| 1007 | self._test_verify_task_helper(False, use_queue_entry=True) |
| 1008 | |
| 1009 | |
| 1010 | def test_verify_task_with_metahost(self): |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 1011 | self.queue_entry.meta_host = 1 |
| 1012 | self.test_verify_task_with_queue_entry() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 1013 | |
| 1014 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 1015 | def test_abort_task(self): |
| 1016 | queue_entry = self.god.create_mock_class(monitor_db.HostQueueEntry, |
| 1017 | 'queue_entry') |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1018 | queue_entry.id = 1 |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 1019 | queue_entry.host_id, queue_entry.job_id = 1, 2 |
| 1020 | task = self.god.create_mock_class(monitor_db.AgentTask, 'task') |
| 1021 | agent = self.god.create_mock_class(monitor_db.Agent, 'agent') |
| 1022 | agent.active_task = task |
| 1023 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 1024 | task.abort.expect_call() |
| 1025 | queue_entry.set_status.expect_call('Aborted') |
| 1026 | |
| 1027 | abort_task = monitor_db.AbortTask(queue_entry, [agent]) |
| 1028 | self.run_task(abort_task, True) |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1029 | self.god.check_playback() |
| 1030 | |
| 1031 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1032 | def _setup_pre_parse_expects(self, autoserv_success): |
| 1033 | self.queue_entry.execution_tag.expect_call().and_return('tag') |
| 1034 | self.pidfile_monitor = monitor_db.PidfileRunMonitor.expect_new() |
| 1035 | self.pidfile_monitor.pidfile_id = self.PIDFILE_ID |
| 1036 | self.pidfile_monitor.attach_to_existing_process.expect_call('tag') |
| 1037 | if autoserv_success: |
| 1038 | code = 0 |
| 1039 | else: |
| 1040 | code = 1 |
| 1041 | self.pidfile_monitor.exit_code.expect_call().and_return(code) |
| 1042 | |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1043 | self.queue_entry.set_status.expect_call('Parsing') |
| 1044 | |
| 1045 | |
| 1046 | def _setup_post_parse_expects(self, autoserv_success): |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1047 | if autoserv_success: |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1048 | status = 'Completed' |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1049 | else: |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1050 | status = 'Failed' |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1051 | self.queue_entry.set_status.expect_call(status) |
| 1052 | |
| 1053 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1054 | def setup_reparse_run_monitor(self): |
| 1055 | autoserv_pidfile_id = object() |
| 1056 | monitor = monitor_db.PidfileRunMonitor.expect_new() |
| 1057 | monitor.run.expect_call( |
| 1058 | mock.is_instance_comparator(list), |
| 1059 | 'tag', |
| 1060 | log_file=mock.anything_comparator(), |
| 1061 | pidfile_name='.parser_execute', |
| 1062 | paired_with_pidfile=self.PIDFILE_ID) |
| 1063 | monitor.exit_code.expect_call() |
| 1064 | monitor.exit_code.expect_call().and_return(0) |
| 1065 | monitor.get_process.expect_call().and_return(self.DUMMY_PROCESS) |
| 1066 | drone_manager.DroneManager.copy_to_results_repository.expect_call( |
| 1067 | self.DUMMY_PROCESS, mock.is_string_comparator()) |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1068 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1069 | |
| 1070 | def _test_final_reparse_task_helper(self, autoserv_success=True): |
| 1071 | self._setup_pre_parse_expects(autoserv_success) |
| 1072 | self.setup_reparse_run_monitor() |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1073 | self._setup_post_parse_expects(autoserv_success) |
| 1074 | |
| 1075 | task = monitor_db.FinalReparseTask([self.queue_entry]) |
| 1076 | self.run_task(task, True) |
| 1077 | |
| 1078 | self.god.check_playback() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1079 | cmd = [monitor_db._parser_path, '--write-pidfile', '-l', '2', '-r', |
| 1080 | '-o', '/abspath/tag'] |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1081 | self.assertEquals(task.cmd, cmd) |
| 1082 | |
| 1083 | |
| 1084 | def test_final_reparse_task(self): |
| 1085 | self.god.stub_class(monitor_db, 'PidfileRunMonitor') |
| 1086 | self._test_final_reparse_task_helper() |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1087 | self._test_final_reparse_task_helper(autoserv_success=False) |
| 1088 | |
| 1089 | |
| 1090 | def test_final_reparse_throttling(self): |
| 1091 | self.god.stub_class(monitor_db, 'PidfileRunMonitor') |
| 1092 | self.god.stub_function(monitor_db.FinalReparseTask, |
| 1093 | '_can_run_new_parse') |
| 1094 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1095 | self._setup_pre_parse_expects(True) |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1096 | monitor_db.FinalReparseTask._can_run_new_parse.expect_call().and_return( |
| 1097 | False) |
| 1098 | monitor_db.FinalReparseTask._can_run_new_parse.expect_call().and_return( |
| 1099 | True) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1100 | self.setup_reparse_run_monitor() |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1101 | self._setup_post_parse_expects(True) |
| 1102 | |
| 1103 | task = monitor_db.FinalReparseTask([self.queue_entry]) |
| 1104 | self.run_task(task, True) |
| 1105 | self.god.check_playback() |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 1106 | |
| 1107 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1108 | def _test_cleanup_task_helper(self, success, use_queue_entry=False): |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1109 | if use_queue_entry: |
| 1110 | self.queue_entry.get_host.expect_call().and_return(self.host) |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1111 | self.host.set_status.expect_call('Cleaning') |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1112 | if success: |
| 1113 | self.setup_run_monitor(0) |
| 1114 | self.host.set_status.expect_call('Ready') |
| 1115 | self.host.update_field.expect_call('dirty', 0) |
| 1116 | else: |
| 1117 | self.setup_run_monitor(1) |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 1118 | if use_queue_entry and not self.queue_entry.meta_host: |
| 1119 | self.queue_entry.set_execution_subdir.expect_call() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1120 | self.queue_entry.execution_tag.expect_call().and_return('tag') |
| 1121 | self._setup_move_logfile(include_destination=True) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1122 | |
| 1123 | if use_queue_entry: |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1124 | task = monitor_db.CleanupTask(queue_entry=self.queue_entry) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1125 | else: |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1126 | task = monitor_db.CleanupTask(host=self.host) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1127 | self.assertEquals(len(task.failure_tasks), 1) |
| 1128 | repair_task = task.failure_tasks[0] |
| 1129 | self.assert_(isinstance(repair_task, monitor_db.RepairTask)) |
| 1130 | if use_queue_entry: |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 1131 | self.assertEquals(repair_task.queue_entry, self.queue_entry) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1132 | |
| 1133 | self.run_task(task, success) |
| 1134 | |
| 1135 | self.god.check_playback() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1136 | self.assert_(set(task.cmd) >= |
| 1137 | set([monitor_db._autoserv_path, '-p', '--cleanup', '-m', |
| 1138 | self.HOSTNAME, '-r', self.TEMP_DIR])) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1139 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1140 | def test_cleanup_task(self): |
| 1141 | self._test_cleanup_task_helper(True) |
| 1142 | self._test_cleanup_task_helper(False) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1143 | |
| 1144 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1145 | def test_cleanup_task_with_queue_entry(self): |
| 1146 | self._test_cleanup_task_helper(False, True) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1147 | |
| 1148 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1149 | class JobTest(BaseSchedulerTest): |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1150 | def setUp(self): |
| 1151 | super(JobTest, self).setUp() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame^] | 1152 | self.god.stub_with( |
| 1153 | drone_manager.DroneManager, 'attach_file_to_execution', |
| 1154 | mock.mock_function('attach_file_to_execution', |
| 1155 | default_return_val='/test/path/tmp/foo')) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1156 | |
| 1157 | |
| 1158 | def _setup_directory_expects(self, execution_subdir): |
| 1159 | job_path = os.path.join('.', '1-my_user') |
| 1160 | results_dir = os.path.join(job_path, execution_subdir) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1161 | |
| 1162 | |
showard | e58e3f8 | 2008-11-20 19:04:59 +0000 | [diff] [blame] | 1163 | def _test_run_helper(self, expect_agent=True, expect_starting=False, |
| 1164 | expect_pending=False): |
| 1165 | if expect_starting: |
| 1166 | expected_status = models.HostQueueEntry.Status.STARTING |
| 1167 | elif expect_pending: |
| 1168 | expected_status = models.HostQueueEntry.Status.PENDING |
| 1169 | else: |
| 1170 | expected_status = models.HostQueueEntry.Status.VERIFYING |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1171 | job = monitor_db.Job.fetch('id = 1').next() |
| 1172 | queue_entry = monitor_db.HostQueueEntry.fetch('id = 1').next() |
| 1173 | agent = job.run(queue_entry) |
| 1174 | |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1175 | self.god.check_playback() |
showard | e77ac67 | 2008-11-14 22:42:33 +0000 | [diff] [blame] | 1176 | self.assertEquals(models.HostQueueEntry.smart_get(1).status, |
| 1177 | expected_status) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1178 | |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1179 | if not expect_agent: |
| 1180 | self.assertEquals(agent, None) |
| 1181 | return |
| 1182 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1183 | self.assert_(isinstance(agent, monitor_db.Agent)) |
| 1184 | tasks = list(agent.queue.queue) |
| 1185 | return tasks |
| 1186 | |
| 1187 | |
| 1188 | def test_run_asynchronous(self): |
| 1189 | self._create_job(hosts=[1, 2]) |
| 1190 | |
| 1191 | tasks = self._test_run_helper() |
| 1192 | |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1193 | self.assertEquals(len(tasks), 1) |
| 1194 | verify_task = tasks[0] |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1195 | |
| 1196 | self.assert_(isinstance(verify_task, monitor_db.VerifyTask)) |
| 1197 | self.assertEquals(verify_task.queue_entry.id, 1) |
| 1198 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1199 | |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1200 | def test_run_asynchronous_skip_verify(self): |
| 1201 | job = self._create_job(hosts=[1, 2]) |
| 1202 | job.run_verify = False |
| 1203 | job.save() |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1204 | self._setup_directory_expects('host1') |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1205 | |
showard | e58e3f8 | 2008-11-20 19:04:59 +0000 | [diff] [blame] | 1206 | tasks = self._test_run_helper(expect_starting=True) |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1207 | |
| 1208 | self.assertEquals(len(tasks), 1) |
| 1209 | queue_task = tasks[0] |
| 1210 | |
| 1211 | self.assert_(isinstance(queue_task, monitor_db.QueueTask)) |
| 1212 | self.assertEquals(queue_task.job.id, 1) |
| 1213 | |
| 1214 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1215 | def test_run_synchronous_verify(self): |
| 1216 | self._create_job(hosts=[1, 2], synchronous=True) |
| 1217 | |
| 1218 | tasks = self._test_run_helper() |
| 1219 | self.assertEquals(len(tasks), 1) |
| 1220 | verify_task = tasks[0] |
| 1221 | |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1222 | self.assert_(isinstance(verify_task, monitor_db.VerifyTask)) |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1223 | self.assertEquals(verify_task.queue_entry.id, 1) |
| 1224 | |
| 1225 | |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1226 | def test_run_synchronous_skip_verify(self): |
| 1227 | job = self._create_job(hosts=[1, 2], synchronous=True) |
| 1228 | job.run_verify = False |
| 1229 | job.save() |
| 1230 | |
showard | e58e3f8 | 2008-11-20 19:04:59 +0000 | [diff] [blame] | 1231 | self._test_run_helper(expect_agent=False, expect_pending=True) |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1232 | |
| 1233 | queue_entry = models.HostQueueEntry.smart_get(1) |
| 1234 | self.assertEquals(queue_entry.status, 'Pending') |
| 1235 | |
| 1236 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1237 | def test_run_synchronous_ready(self): |
| 1238 | self._create_job(hosts=[1, 2], synchronous=True) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1239 | self._update_hqe("status='Pending', execution_subdir='") |
| 1240 | self._setup_directory_expects('group0') |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1241 | |
showard | e58e3f8 | 2008-11-20 19:04:59 +0000 | [diff] [blame] | 1242 | tasks = self._test_run_helper(expect_starting=True) |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1243 | self.assertEquals(len(tasks), 1) |
| 1244 | queue_task = tasks[0] |
| 1245 | |
| 1246 | self.assert_(isinstance(queue_task, monitor_db.QueueTask)) |
| 1247 | self.assertEquals(queue_task.job.id, 1) |
| 1248 | hqe_ids = [hqe.id for hqe in queue_task.queue_entries] |
| 1249 | self.assertEquals(hqe_ids, [1, 2]) |
| 1250 | |
| 1251 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1252 | def test_reboot_before_always(self): |
| 1253 | job = self._create_job(hosts=[1]) |
showard | 0fc3830 | 2008-10-23 00:44:07 +0000 | [diff] [blame] | 1254 | job.reboot_before = models.RebootBefore.ALWAYS |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1255 | job.save() |
| 1256 | |
| 1257 | tasks = self._test_run_helper() |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1258 | self.assertEquals(len(tasks), 2) |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1259 | cleanup_task = tasks[0] |
| 1260 | self.assert_(isinstance(cleanup_task, monitor_db.CleanupTask)) |
| 1261 | self.assertEquals(cleanup_task.host.id, 1) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1262 | |
| 1263 | |
| 1264 | def _test_reboot_before_if_dirty_helper(self, expect_reboot): |
| 1265 | job = self._create_job(hosts=[1]) |
showard | 0fc3830 | 2008-10-23 00:44:07 +0000 | [diff] [blame] | 1266 | job.reboot_before = models.RebootBefore.IF_DIRTY |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1267 | job.save() |
| 1268 | |
| 1269 | tasks = self._test_run_helper() |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1270 | self.assertEquals(len(tasks), expect_reboot and 2 or 1) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1271 | if expect_reboot: |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1272 | cleanup_task = tasks[0] |
| 1273 | self.assert_(isinstance(cleanup_task, monitor_db.CleanupTask)) |
| 1274 | self.assertEquals(cleanup_task.host.id, 1) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1275 | |
| 1276 | def test_reboot_before_if_dirty(self): |
| 1277 | models.Host.smart_get(1).update_object(dirty=True) |
| 1278 | self._test_reboot_before_if_dirty_helper(True) |
| 1279 | |
| 1280 | |
| 1281 | def test_reboot_before_not_dirty(self): |
| 1282 | models.Host.smart_get(1).update_object(dirty=False) |
| 1283 | self._test_reboot_before_if_dirty_helper(False) |
| 1284 | |
| 1285 | |
| 1286 | |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 1287 | if __name__ == '__main__': |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1288 | unittest.main() |