showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 1 | #!/usr/bin/python |
| 2 | |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 3 | import unittest, time, subprocess, os, StringIO, tempfile, datetime, shutil |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 4 | import MySQLdb |
| 5 | import common |
showard | 364fe86 | 2008-10-17 02:01:16 +0000 | [diff] [blame] | 6 | from autotest_lib.frontend import setup_django_environment |
| 7 | from autotest_lib.frontend import setup_test_environment |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 8 | from autotest_lib.client.common_lib import global_config, host_protections |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 9 | from autotest_lib.client.common_lib.test_utils import mock |
showard | 442e71e | 2008-10-06 10:05:20 +0000 | [diff] [blame] | 10 | from autotest_lib.database import database_connection, migrate |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 11 | from autotest_lib.frontend import thread_local |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 12 | from autotest_lib.frontend.afe import models |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 13 | from autotest_lib.scheduler import monitor_db, drone_manager, email_manager |
showard | d1ee1dd | 2009-01-07 21:33:08 +0000 | [diff] [blame] | 14 | from autotest_lib.scheduler import scheduler_config |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 15 | |
| 16 | _DEBUG = False |
| 17 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 18 | class DummyAgent(object): |
| 19 | _is_running = False |
| 20 | _is_done = False |
| 21 | num_processes = 1 |
| 22 | host_ids = [] |
| 23 | queue_entry_ids = [] |
| 24 | |
| 25 | def is_running(self): |
| 26 | return self._is_running |
| 27 | |
| 28 | |
| 29 | def tick(self): |
| 30 | self._is_running = True |
| 31 | |
| 32 | |
| 33 | def is_done(self): |
| 34 | return self._is_done |
| 35 | |
| 36 | |
| 37 | def set_done(self, done): |
| 38 | self._is_done = done |
| 39 | self._is_running = not done |
showard | 04c82c5 | 2008-05-29 19:38:12 +0000 | [diff] [blame] | 40 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 41 | |
| 42 | class IsRow(mock.argument_comparator): |
| 43 | def __init__(self, row_id): |
| 44 | self.row_id = row_id |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 45 | |
| 46 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 47 | def is_satisfied_by(self, parameter): |
| 48 | return list(parameter)[0] == self.row_id |
| 49 | |
| 50 | |
| 51 | def __str__(self): |
| 52 | return 'row with id %s' % self.row_id |
| 53 | |
| 54 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 55 | class BaseSchedulerTest(unittest.TestCase): |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 56 | _config_section = 'AUTOTEST_WEB' |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 57 | _test_db_initialized = False |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 58 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 59 | def _do_query(self, sql): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 60 | self._database.execute(sql) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 61 | |
| 62 | |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 63 | @classmethod |
| 64 | def _initialize_test_db(cls): |
| 65 | if cls._test_db_initialized: |
| 66 | return |
| 67 | temp_fd, cls._test_db_file = tempfile.mkstemp(suffix='.monitor_test') |
| 68 | os.close(temp_fd) |
showard | 364fe86 | 2008-10-17 02:01:16 +0000 | [diff] [blame] | 69 | setup_test_environment.set_test_database(cls._test_db_file) |
| 70 | setup_test_environment.run_syncdb() |
| 71 | cls._test_db_backup = setup_test_environment.backup_test_database() |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 72 | cls._test_db_initialized = True |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 73 | |
| 74 | |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 75 | def _open_test_db(self): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 76 | self._initialize_test_db() |
showard | 364fe86 | 2008-10-17 02:01:16 +0000 | [diff] [blame] | 77 | setup_test_environment.restore_test_database(self._test_db_backup) |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 78 | self._database = ( |
| 79 | database_connection.DatabaseConnection.get_test_database( |
| 80 | self._test_db_file)) |
| 81 | self._database.connect() |
| 82 | self._database.debug = _DEBUG |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 83 | |
| 84 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 85 | def _close_test_db(self): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 86 | self._database.disconnect() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 87 | |
| 88 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 89 | def _set_monitor_stubs(self): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 90 | monitor_db._db = self._database |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 91 | monitor_db._drone_manager._results_dir = '/test/path' |
| 92 | monitor_db._drone_manager._temporary_directory = '/test/path/tmp' |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 93 | |
| 94 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 95 | def _fill_in_test_data(self): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 96 | user = models.User.objects.create(login='my_user') |
| 97 | acl_group = models.AclGroup.objects.create(name='my_acl') |
| 98 | acl_group.users.add(user) |
| 99 | |
| 100 | hosts = [models.Host.objects.create(hostname=hostname) for hostname in |
| 101 | ('host1', 'host2', 'host3', 'host4')] |
| 102 | acl_group.hosts = hosts |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 103 | models.AclGroup.smart_get('Everyone').hosts = [] |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 104 | |
| 105 | labels = [models.Label.objects.create(name=name) for name in |
| 106 | ('label1', 'label2', 'label3')] |
| 107 | labels[2].only_if_needed = True |
| 108 | labels[2].save() |
| 109 | hosts[0].labels.add(labels[0]) |
| 110 | hosts[1].labels.add(labels[1]) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 111 | |
| 112 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 113 | def _setup_dummy_user(self): |
| 114 | user = models.User.objects.create(login='dummy', access_level=100) |
| 115 | thread_local.set_user(user) |
| 116 | |
| 117 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 118 | def setUp(self): |
| 119 | self.god = mock.mock_god() |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 120 | self._open_test_db() |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 121 | self._fill_in_test_data() |
| 122 | self._set_monitor_stubs() |
| 123 | self._dispatcher = monitor_db.Dispatcher() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 124 | self._setup_dummy_user() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 125 | |
| 126 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 127 | def tearDown(self): |
| 128 | self._close_test_db() |
| 129 | self.god.unstub_all() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 130 | |
| 131 | |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 132 | def _create_job(self, hosts=[], metahosts=[], priority=0, active=False, |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 133 | synchronous=False): |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 134 | synch_count = synchronous and 2 or 1 |
showard | 12bc8a8 | 2008-10-09 16:49:53 +0000 | [diff] [blame] | 135 | created_on = datetime.datetime(2008, 1, 1) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 136 | status = models.HostQueueEntry.Status.QUEUED |
| 137 | if active: |
| 138 | status = models.HostQueueEntry.Status.RUNNING |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 139 | job = models.Job.objects.create( |
| 140 | name='test', owner='my_user', priority=priority, |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 141 | synch_count=synch_count, created_on=created_on, |
showard | 0fc3830 | 2008-10-23 00:44:07 +0000 | [diff] [blame] | 142 | reboot_before=models.RebootBefore.NEVER) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 143 | for host_id in hosts: |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 144 | models.HostQueueEntry.objects.create(job=job, priority=priority, |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 145 | host_id=host_id, status=status) |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 146 | models.IneligibleHostQueue.objects.create(job=job, host_id=host_id) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 147 | for label_id in metahosts: |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 148 | models.HostQueueEntry.objects.create(job=job, priority=priority, |
| 149 | meta_host_id=label_id, |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 150 | status=status) |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 151 | return job |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 152 | |
| 153 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 154 | def _create_job_simple(self, hosts, use_metahost=False, |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 155 | priority=0, active=False): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 156 | 'An alternative interface to _create_job' |
| 157 | args = {'hosts' : [], 'metahosts' : []} |
| 158 | if use_metahost: |
| 159 | args['metahosts'] = hosts |
| 160 | else: |
| 161 | args['hosts'] = hosts |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame] | 162 | return self._create_job(priority=priority, active=active, **args) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 163 | |
| 164 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 165 | def _update_hqe(self, set, where=''): |
| 166 | query = 'UPDATE host_queue_entries SET ' + set |
| 167 | if where: |
| 168 | query += ' WHERE ' + where |
| 169 | self._do_query(query) |
| 170 | |
| 171 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 172 | class DispatcherSchedulingTest(BaseSchedulerTest): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 173 | _jobs_scheduled = [] |
| 174 | |
| 175 | def _set_monitor_stubs(self): |
| 176 | super(DispatcherSchedulingTest, self)._set_monitor_stubs() |
| 177 | def run_stub(hqe_self, assigned_host=None): |
| 178 | hqe_self.set_status('Starting') |
| 179 | if hqe_self.meta_host: |
| 180 | host = assigned_host |
| 181 | else: |
| 182 | host = hqe_self.host |
| 183 | self._record_job_scheduled(hqe_self.job.id, host.id) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 184 | return DummyAgent() |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 185 | monitor_db.HostQueueEntry.run = run_stub |
| 186 | |
| 187 | |
| 188 | def _record_job_scheduled(self, job_id, host_id): |
| 189 | record = (job_id, host_id) |
| 190 | self.assert_(record not in self._jobs_scheduled, |
| 191 | 'Job %d scheduled on host %d twice' % |
| 192 | (job_id, host_id)) |
| 193 | self._jobs_scheduled.append(record) |
| 194 | |
| 195 | |
| 196 | def _assert_job_scheduled_on(self, job_id, host_id): |
| 197 | record = (job_id, host_id) |
| 198 | self.assert_(record in self._jobs_scheduled, |
| 199 | 'Job %d not scheduled on host %d as expected\n' |
| 200 | 'Jobs scheduled: %s' % |
| 201 | (job_id, host_id, self._jobs_scheduled)) |
| 202 | self._jobs_scheduled.remove(record) |
| 203 | |
| 204 | |
| 205 | def _check_for_extra_schedulings(self): |
| 206 | if len(self._jobs_scheduled) != 0: |
| 207 | self.fail('Extra jobs scheduled: ' + |
| 208 | str(self._jobs_scheduled)) |
| 209 | |
| 210 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 211 | def _convert_jobs_to_metahosts(self, *job_ids): |
| 212 | sql_tuple = '(' + ','.join(str(i) for i in job_ids) + ')' |
| 213 | self._do_query('UPDATE host_queue_entries SET ' |
| 214 | 'meta_host=host_id, host_id=NULL ' |
| 215 | 'WHERE job_id IN ' + sql_tuple) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 216 | |
| 217 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 218 | def _lock_host(self, host_id): |
| 219 | self._do_query('UPDATE hosts SET locked=1 WHERE id=' + |
| 220 | str(host_id)) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 221 | |
| 222 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 223 | def setUp(self): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 224 | super(DispatcherSchedulingTest, self).setUp() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 225 | self._jobs_scheduled = [] |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 226 | |
| 227 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 228 | def _test_basic_scheduling_helper(self, use_metahosts): |
| 229 | 'Basic nonmetahost scheduling' |
| 230 | self._create_job_simple([1], use_metahosts) |
| 231 | self._create_job_simple([2], use_metahosts) |
| 232 | self._dispatcher._schedule_new_jobs() |
| 233 | self._assert_job_scheduled_on(1, 1) |
| 234 | self._assert_job_scheduled_on(2, 2) |
| 235 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 236 | |
| 237 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 238 | def _test_priorities_helper(self, use_metahosts): |
| 239 | 'Test prioritization ordering' |
| 240 | self._create_job_simple([1], use_metahosts) |
| 241 | self._create_job_simple([2], use_metahosts) |
| 242 | self._create_job_simple([1,2], use_metahosts) |
| 243 | self._create_job_simple([1], use_metahosts, priority=1) |
| 244 | self._dispatcher._schedule_new_jobs() |
| 245 | self._assert_job_scheduled_on(4, 1) # higher priority |
| 246 | self._assert_job_scheduled_on(2, 2) # earlier job over later |
| 247 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 248 | |
| 249 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 250 | def _test_hosts_ready_helper(self, use_metahosts): |
| 251 | """ |
| 252 | Only hosts that are status=Ready, unlocked and not invalid get |
| 253 | scheduled. |
| 254 | """ |
| 255 | self._create_job_simple([1], use_metahosts) |
| 256 | self._do_query('UPDATE hosts SET status="Running" WHERE id=1') |
| 257 | self._dispatcher._schedule_new_jobs() |
| 258 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 259 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 260 | self._do_query('UPDATE hosts SET status="Ready", locked=1 ' |
| 261 | 'WHERE id=1') |
| 262 | self._dispatcher._schedule_new_jobs() |
| 263 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 264 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 265 | self._do_query('UPDATE hosts SET locked=0, invalid=1 ' |
| 266 | 'WHERE id=1') |
| 267 | self._dispatcher._schedule_new_jobs() |
showard | 5df2b19 | 2008-07-03 19:51:57 +0000 | [diff] [blame] | 268 | if not use_metahosts: |
| 269 | self._assert_job_scheduled_on(1, 1) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 270 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 271 | |
| 272 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 273 | def _test_hosts_idle_helper(self, use_metahosts): |
| 274 | 'Only idle hosts get scheduled' |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 275 | self._create_job(hosts=[1], active=True) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 276 | self._create_job_simple([1], use_metahosts) |
| 277 | self._dispatcher._schedule_new_jobs() |
| 278 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 279 | |
| 280 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 281 | def _test_obey_ACLs_helper(self, use_metahosts): |
| 282 | self._do_query('DELETE FROM acl_groups_hosts WHERE host_id=1') |
| 283 | self._create_job_simple([1], use_metahosts) |
| 284 | self._dispatcher._schedule_new_jobs() |
| 285 | self._check_for_extra_schedulings() |
| 286 | |
| 287 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 288 | def test_basic_scheduling(self): |
| 289 | self._test_basic_scheduling_helper(False) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 290 | |
| 291 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 292 | def test_priorities(self): |
| 293 | self._test_priorities_helper(False) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 294 | |
| 295 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 296 | def test_hosts_ready(self): |
| 297 | self._test_hosts_ready_helper(False) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 298 | |
| 299 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 300 | def test_hosts_idle(self): |
| 301 | self._test_hosts_idle_helper(False) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 302 | |
| 303 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 304 | def test_obey_ACLs(self): |
| 305 | self._test_obey_ACLs_helper(False) |
| 306 | |
| 307 | |
| 308 | def test_non_metahost_on_invalid_host(self): |
| 309 | """ |
| 310 | Non-metahost entries can get scheduled on invalid hosts (this is how |
| 311 | one-time hosts work). |
| 312 | """ |
| 313 | self._do_query('UPDATE hosts SET invalid=1') |
| 314 | self._test_basic_scheduling_helper(False) |
| 315 | |
| 316 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 317 | def test_metahost_scheduling(self): |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 318 | """ |
| 319 | Basic metahost scheduling |
| 320 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 321 | self._test_basic_scheduling_helper(True) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 322 | |
| 323 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 324 | def test_metahost_priorities(self): |
| 325 | self._test_priorities_helper(True) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 326 | |
| 327 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 328 | def test_metahost_hosts_ready(self): |
| 329 | self._test_hosts_ready_helper(True) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 330 | |
| 331 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 332 | def test_metahost_hosts_idle(self): |
| 333 | self._test_hosts_idle_helper(True) |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 334 | |
| 335 | |
showard | 63a3477 | 2008-08-18 19:32:50 +0000 | [diff] [blame] | 336 | def test_metahost_obey_ACLs(self): |
| 337 | self._test_obey_ACLs_helper(True) |
| 338 | |
| 339 | |
showard | ade14e2 | 2009-01-26 22:38:32 +0000 | [diff] [blame^] | 340 | def test_only_if_needed_labels(self): |
| 341 | # apply only_if_needed label3 to host1 |
| 342 | label3 = models.Label.smart_get('label3') |
| 343 | models.Host.smart_get('host1').labels.add(label3) |
| 344 | |
| 345 | job = self._create_job_simple([1], use_metahost=True) |
| 346 | # if the job doesn't depend on label3, there should be no scheduling |
| 347 | self._dispatcher._schedule_new_jobs() |
| 348 | self._check_for_extra_schedulings() |
| 349 | |
| 350 | # now make the job depend on label3 |
| 351 | job.dependency_labels.add(label3) |
| 352 | self._dispatcher._schedule_new_jobs() |
| 353 | self._assert_job_scheduled_on(1, 1) |
| 354 | self._check_for_extra_schedulings() |
| 355 | |
| 356 | # should also work if the metahost is the only_if_needed label |
| 357 | self._do_query('DELETE FROM jobs_dependency_labels') |
| 358 | self._create_job(metahosts=[3]) |
| 359 | self._dispatcher._schedule_new_jobs() |
| 360 | self._assert_job_scheduled_on(2, 1) |
| 361 | self._check_for_extra_schedulings() |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 362 | |
| 363 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 364 | def test_nonmetahost_over_metahost(self): |
| 365 | """ |
| 366 | Non-metahost entries should take priority over metahost entries |
| 367 | for the same host |
| 368 | """ |
| 369 | self._create_job(metahosts=[1]) |
| 370 | self._create_job(hosts=[1]) |
| 371 | self._dispatcher._schedule_new_jobs() |
| 372 | self._assert_job_scheduled_on(2, 1) |
| 373 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 374 | |
| 375 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 376 | def test_metahosts_obey_blocks(self): |
| 377 | """ |
| 378 | Metahosts can't get scheduled on hosts already scheduled for |
| 379 | that job. |
| 380 | """ |
| 381 | self._create_job(metahosts=[1], hosts=[1]) |
| 382 | # make the nonmetahost entry complete, so the metahost can try |
| 383 | # to get scheduled |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 384 | self._update_hqe(set='complete = 1', where='host_id=1') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 385 | self._dispatcher._schedule_new_jobs() |
| 386 | self._check_for_extra_schedulings() |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 387 | |
| 388 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 389 | def test_only_schedule_queued_entries(self): |
| 390 | self._create_job(metahosts=[1]) |
| 391 | self._update_hqe(set='active=1, host_id=2') |
| 392 | self._dispatcher._schedule_new_jobs() |
| 393 | self._check_for_extra_schedulings() |
| 394 | |
| 395 | |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 396 | def test_no_ready_hosts(self): |
| 397 | self._create_job(hosts=[1]) |
| 398 | self._do_query('UPDATE hosts SET status="Repair Failed"') |
| 399 | self._dispatcher._schedule_new_jobs() |
| 400 | self._check_for_extra_schedulings() |
| 401 | |
| 402 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 403 | class DispatcherThrottlingTest(BaseSchedulerTest): |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 404 | """ |
| 405 | Test that the dispatcher throttles: |
| 406 | * total number of running processes |
| 407 | * number of processes started per cycle |
| 408 | """ |
| 409 | _MAX_RUNNING = 3 |
| 410 | _MAX_STARTED = 2 |
| 411 | |
| 412 | def setUp(self): |
| 413 | super(DispatcherThrottlingTest, self).setUp() |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 414 | scheduler_config.config.max_processes_per_drone = self._MAX_RUNNING |
showard | d1ee1dd | 2009-01-07 21:33:08 +0000 | [diff] [blame] | 415 | scheduler_config.config.max_processes_started_per_cycle = ( |
| 416 | self._MAX_STARTED) |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 417 | |
showard | 324bf81 | 2009-01-20 23:23:38 +0000 | [diff] [blame] | 418 | def fake_max_runnable_processes(fake_self): |
| 419 | running = sum(agent.num_processes |
| 420 | for agent in self._agents |
| 421 | if agent.is_running()) |
| 422 | return self._MAX_RUNNING - running |
| 423 | self.god.stub_with(drone_manager.DroneManager, 'max_runnable_processes', |
| 424 | fake_max_runnable_processes) |
showard | 2fa5169 | 2009-01-13 23:48:08 +0000 | [diff] [blame] | 425 | |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 426 | |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 427 | def _setup_some_agents(self, num_agents): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 428 | self._agents = [DummyAgent() for i in xrange(num_agents)] |
showard | 4c5374f | 2008-09-04 17:02:56 +0000 | [diff] [blame] | 429 | self._dispatcher._agents = list(self._agents) |
| 430 | |
| 431 | |
| 432 | def _run_a_few_cycles(self): |
| 433 | for i in xrange(4): |
| 434 | self._dispatcher._handle_agents() |
| 435 | |
| 436 | |
| 437 | def _assert_agents_started(self, indexes, is_started=True): |
| 438 | for i in indexes: |
| 439 | self.assert_(self._agents[i].is_running() == is_started, |
| 440 | 'Agent %d %sstarted' % |
| 441 | (i, is_started and 'not ' or '')) |
| 442 | |
| 443 | |
| 444 | def _assert_agents_not_started(self, indexes): |
| 445 | self._assert_agents_started(indexes, False) |
| 446 | |
| 447 | |
| 448 | def test_throttle_total(self): |
| 449 | self._setup_some_agents(4) |
| 450 | self._run_a_few_cycles() |
| 451 | self._assert_agents_started([0, 1, 2]) |
| 452 | self._assert_agents_not_started([3]) |
| 453 | |
| 454 | |
| 455 | def test_throttle_per_cycle(self): |
| 456 | self._setup_some_agents(3) |
| 457 | self._dispatcher._handle_agents() |
| 458 | self._assert_agents_started([0, 1]) |
| 459 | self._assert_agents_not_started([2]) |
| 460 | |
| 461 | |
| 462 | def test_throttle_with_synchronous(self): |
| 463 | self._setup_some_agents(2) |
| 464 | self._agents[0].num_processes = 3 |
| 465 | self._run_a_few_cycles() |
| 466 | self._assert_agents_started([0]) |
| 467 | self._assert_agents_not_started([1]) |
| 468 | |
| 469 | |
| 470 | def test_large_agent_starvation(self): |
| 471 | """ |
| 472 | Ensure large agents don't get starved by lower-priority agents. |
| 473 | """ |
| 474 | self._setup_some_agents(3) |
| 475 | self._agents[1].num_processes = 3 |
| 476 | self._run_a_few_cycles() |
| 477 | self._assert_agents_started([0]) |
| 478 | self._assert_agents_not_started([1, 2]) |
| 479 | |
| 480 | self._agents[0].set_done(True) |
| 481 | self._run_a_few_cycles() |
| 482 | self._assert_agents_started([1]) |
| 483 | self._assert_agents_not_started([2]) |
| 484 | |
| 485 | |
| 486 | def test_zero_process_agent(self): |
| 487 | self._setup_some_agents(5) |
| 488 | self._agents[4].num_processes = 0 |
| 489 | self._run_a_few_cycles() |
| 490 | self._assert_agents_started([0, 1, 2, 4]) |
| 491 | self._assert_agents_not_started([3]) |
| 492 | |
| 493 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 494 | class FindAbortTest(BaseSchedulerTest): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 495 | """ |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 496 | Test the dispatcher abort functionality. |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 497 | """ |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 498 | def _check_abort_agent(self, agent, entry_id): |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 499 | self.assert_(isinstance(agent, monitor_db.Agent)) |
| 500 | tasks = list(agent.queue.queue) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 501 | self.assertEquals(len(tasks), 1) |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 502 | abort = tasks[0] |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 503 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 504 | self.assert_(isinstance(abort, monitor_db.AbortTask)) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 505 | self.assertEquals(abort.queue_entry.id, entry_id) |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 506 | |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 507 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 508 | def _check_host_agent(self, agent, host_id): |
| 509 | self.assert_(isinstance(agent, monitor_db.Agent)) |
| 510 | tasks = list(agent.queue.queue) |
| 511 | self.assertEquals(len(tasks), 2) |
| 512 | cleanup, verify = tasks |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 513 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 514 | self.assert_(isinstance(cleanup, monitor_db.CleanupTask)) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 515 | self.assertEquals(cleanup.host.id, host_id) |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 516 | |
| 517 | self.assert_(isinstance(verify, monitor_db.VerifyTask)) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 518 | self.assertEquals(verify.host.id, host_id) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 519 | |
| 520 | |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 521 | def _check_agents(self, agents, include_host_tasks): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 522 | agents = list(agents) |
| 523 | if include_host_tasks: |
| 524 | self.assertEquals(len(agents), 4) |
| 525 | self._check_host_agent(agents.pop(0), 1) |
| 526 | self._check_host_agent(agents.pop(1), 2) |
| 527 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 528 | self.assertEquals(len(agents), 2) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 529 | self._check_abort_agent(agents[0], 1) |
| 530 | self._check_abort_agent(agents[1], 2) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 531 | |
| 532 | |
| 533 | def test_find_aborting_inactive(self): |
| 534 | self._create_job(hosts=[1, 2]) |
| 535 | self._update_hqe(set='status="Abort"') |
| 536 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 537 | self._dispatcher._find_aborting() |
| 538 | |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 539 | self._check_agents(self._dispatcher._agents, include_host_tasks=False) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 540 | self.god.check_playback() |
| 541 | |
| 542 | |
| 543 | def test_find_aborting_active(self): |
| 544 | self._create_job(hosts=[1, 2]) |
| 545 | self._update_hqe(set='status="Abort", active=1') |
| 546 | # have to make an Agent for the active HQEs |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 547 | agent = self.god.create_mock_class(monitor_db.Agent, 'old_agent') |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 548 | agent.host_ids = agent.queue_entry_ids = [1, 2] |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 549 | self._dispatcher.add_agent(agent) |
| 550 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 551 | self._dispatcher._find_aborting() |
| 552 | |
showard | 9d9ffd5 | 2008-11-09 23:14:35 +0000 | [diff] [blame] | 553 | self._check_agents(self._dispatcher._agents, include_host_tasks=True) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 554 | self.god.check_playback() |
| 555 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 556 | # ensure agent gets aborted |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 557 | abort1 = self._dispatcher._agents[1].queue.queue[0] |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 558 | self.assertEquals(abort1.agents_to_abort, [agent]) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 559 | abort2 = self._dispatcher._agents[3].queue.queue[0] |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 560 | self.assertEquals(abort2.agents_to_abort, []) |
| 561 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 562 | |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 563 | class JobTimeoutTest(BaseSchedulerTest): |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 564 | def _test_synch_start_timeout_helper(self, expect_abort, |
| 565 | set_created_on=True, set_active=True, |
| 566 | set_acl=True): |
showard | d1ee1dd | 2009-01-07 21:33:08 +0000 | [diff] [blame] | 567 | scheduler_config.config.synch_job_start_timeout_minutes = 60 |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 568 | job = self._create_job(hosts=[1, 2]) |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 569 | if set_active: |
| 570 | hqe = job.hostqueueentry_set.filter(host__id=1)[0] |
| 571 | hqe.status = 'Pending' |
| 572 | hqe.active = 1 |
| 573 | hqe.save() |
| 574 | |
| 575 | everyone_acl = models.AclGroup.smart_get('Everyone') |
| 576 | host1 = models.Host.smart_get(1) |
| 577 | if set_acl: |
| 578 | everyone_acl.hosts.add(host1) |
| 579 | else: |
| 580 | everyone_acl.hosts.remove(host1) |
| 581 | |
| 582 | job.created_on = datetime.datetime.now() |
| 583 | if set_created_on: |
| 584 | job.created_on -= datetime.timedelta(minutes=100) |
| 585 | job.save() |
| 586 | |
| 587 | self._dispatcher._abort_jobs_past_synch_start_timeout() |
| 588 | |
| 589 | for hqe in job.hostqueueentry_set.all(): |
| 590 | if expect_abort: |
| 591 | self.assert_(hqe.status in ('Abort', 'Aborted'), hqe.status) |
| 592 | else: |
| 593 | self.assert_(hqe.status not in ('Abort', 'Aborted'), hqe.status) |
| 594 | |
| 595 | |
| 596 | def test_synch_start_timeout_helper(self): |
| 597 | # no abort if any of the condition aren't met |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 598 | self._test_synch_start_timeout_helper(False, set_created_on=False) |
| 599 | self._test_synch_start_timeout_helper(False, set_active=False) |
| 600 | self._test_synch_start_timeout_helper(False, set_acl=False) |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 601 | # abort if all conditions are met |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 602 | self._test_synch_start_timeout_helper(True) |
showard | 9886397 | 2008-10-29 21:14:56 +0000 | [diff] [blame] | 603 | |
| 604 | |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 605 | class PidfileRunMonitorTest(unittest.TestCase): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 606 | execution_tag = 'test_tag' |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 607 | pid = 12345 |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 608 | process = drone_manager.Process('myhost', pid) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 609 | num_tests_failed = 1 |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 610 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 611 | def setUp(self): |
| 612 | self.god = mock.mock_god() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 613 | self.mock_drone_manager = self.god.create_mock_class( |
| 614 | drone_manager.DroneManager, 'drone_manager') |
| 615 | self.god.stub_with(monitor_db, '_drone_manager', |
| 616 | self.mock_drone_manager) |
| 617 | self.god.stub_function(email_manager.manager, 'enqueue_notify_email') |
| 618 | |
| 619 | self.pidfile_id = object() |
| 620 | |
| 621 | self.mock_drone_manager.get_pidfile_id_from.expect_call( |
| 622 | self.execution_tag).and_return(self.pidfile_id) |
| 623 | self.mock_drone_manager.register_pidfile.expect_call(self.pidfile_id) |
| 624 | |
| 625 | self.monitor = monitor_db.PidfileRunMonitor() |
| 626 | self.monitor.attach_to_existing_process(self.execution_tag) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 627 | |
| 628 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 629 | def tearDown(self): |
| 630 | self.god.unstub_all() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 631 | |
| 632 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 633 | def setup_pidfile(self, pid=None, exit_code=None, tests_failed=None, |
| 634 | use_second_read=False): |
| 635 | contents = drone_manager.PidfileContents() |
| 636 | if pid is not None: |
| 637 | contents.process = drone_manager.Process('myhost', pid) |
| 638 | contents.exit_status = exit_code |
| 639 | contents.num_tests_failed = tests_failed |
| 640 | self.mock_drone_manager.get_pidfile_contents.expect_call( |
| 641 | self.pidfile_id, use_second_read=use_second_read).and_return( |
| 642 | contents) |
| 643 | |
| 644 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 645 | def set_not_yet_run(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 646 | self.setup_pidfile() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 647 | |
| 648 | |
showard | 3dd6b88 | 2008-10-27 19:21:39 +0000 | [diff] [blame] | 649 | def set_empty_pidfile(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 650 | self.setup_pidfile() |
showard | 3dd6b88 | 2008-10-27 19:21:39 +0000 | [diff] [blame] | 651 | |
| 652 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 653 | def set_running(self, use_second_read=False): |
| 654 | self.setup_pidfile(self.pid, use_second_read=use_second_read) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 655 | |
| 656 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 657 | def set_complete(self, error_code, use_second_read=False): |
| 658 | self.setup_pidfile(self.pid, error_code, self.num_tests_failed, |
| 659 | use_second_read=use_second_read) |
| 660 | |
| 661 | |
| 662 | def _check_monitor(self, expected_pid, expected_exit_status, |
| 663 | expected_num_tests_failed): |
| 664 | if expected_pid is None: |
| 665 | self.assertEquals(self.monitor._state.process, None) |
| 666 | else: |
| 667 | self.assertEquals(self.monitor._state.process.pid, expected_pid) |
| 668 | self.assertEquals(self.monitor._state.exit_status, expected_exit_status) |
| 669 | self.assertEquals(self.monitor._state.num_tests_failed, |
| 670 | expected_num_tests_failed) |
| 671 | |
| 672 | |
| 673 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 674 | |
| 675 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 676 | def _test_read_pidfile_helper(self, expected_pid, expected_exit_status, |
| 677 | expected_num_tests_failed): |
| 678 | self.monitor._read_pidfile() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 679 | self._check_monitor(expected_pid, expected_exit_status, |
| 680 | expected_num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 681 | |
| 682 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 683 | def _get_expected_tests_failed(self, expected_exit_status): |
| 684 | if expected_exit_status is None: |
| 685 | expected_tests_failed = None |
| 686 | else: |
| 687 | expected_tests_failed = self.num_tests_failed |
| 688 | return expected_tests_failed |
| 689 | |
| 690 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 691 | def test_read_pidfile(self): |
| 692 | self.set_not_yet_run() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 693 | self._test_read_pidfile_helper(None, None, None) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 694 | |
showard | 3dd6b88 | 2008-10-27 19:21:39 +0000 | [diff] [blame] | 695 | self.set_empty_pidfile() |
| 696 | self._test_read_pidfile_helper(None, None, None) |
| 697 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 698 | self.set_running() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 699 | self._test_read_pidfile_helper(self.pid, None, None) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 700 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 701 | self.set_complete(123) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 702 | self._test_read_pidfile_helper(self.pid, 123, self.num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 703 | |
| 704 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 705 | def test_read_pidfile_error(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 706 | self.mock_drone_manager.get_pidfile_contents.expect_call( |
| 707 | self.pidfile_id, use_second_read=False).and_return( |
| 708 | drone_manager.InvalidPidfile('error')) |
| 709 | self.assertRaises(monitor_db.PidfileRunMonitor._PidfileException, |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 710 | self.monitor._read_pidfile) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 711 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 712 | |
| 713 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 714 | def setup_is_running(self, is_running): |
| 715 | self.mock_drone_manager.is_process_running.expect_call( |
| 716 | self.process).and_return(is_running) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 717 | |
| 718 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 719 | def _test_get_pidfile_info_helper(self, expected_pid, expected_exit_status, |
| 720 | expected_num_tests_failed): |
| 721 | self.monitor._get_pidfile_info() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 722 | self._check_monitor(expected_pid, expected_exit_status, |
| 723 | expected_num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 724 | |
| 725 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 726 | def test_get_pidfile_info(self): |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 727 | """ |
| 728 | normal cases for get_pidfile_info |
| 729 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 730 | # running |
| 731 | self.set_running() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 732 | self.setup_is_running(True) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 733 | self._test_get_pidfile_info_helper(self.pid, None, None) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 734 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 735 | # exited during check |
| 736 | self.set_running() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 737 | self.setup_is_running(False) |
| 738 | self.set_complete(123, use_second_read=True) # pidfile gets read again |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 739 | self._test_get_pidfile_info_helper(self.pid, 123, self.num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 740 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 741 | # completed |
| 742 | self.set_complete(123) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 743 | self._test_get_pidfile_info_helper(self.pid, 123, self.num_tests_failed) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 744 | |
| 745 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 746 | def test_get_pidfile_info_running_no_proc(self): |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 747 | """ |
| 748 | pidfile shows process running, but no proc exists |
| 749 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 750 | # running but no proc |
| 751 | self.set_running() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 752 | self.setup_is_running(False) |
| 753 | self.set_running(use_second_read=True) |
| 754 | email_manager.manager.enqueue_notify_email.expect_call( |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 755 | mock.is_string_comparator(), mock.is_string_comparator()) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 756 | self._test_get_pidfile_info_helper(self.pid, 1, 0) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 757 | self.assertTrue(self.monitor.lost_process) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 758 | |
| 759 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 760 | def test_get_pidfile_info_not_yet_run(self): |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 761 | """ |
| 762 | pidfile hasn't been written yet |
| 763 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 764 | self.set_not_yet_run() |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 765 | self._test_get_pidfile_info_helper(None, None, None) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 766 | |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 767 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 768 | def test_process_failed_to_write_pidfile(self): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 769 | self.set_not_yet_run() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 770 | email_manager.manager.enqueue_notify_email.expect_call( |
| 771 | mock.is_string_comparator(), mock.is_string_comparator()) |
| 772 | dummy_process = drone_manager.Process('dummy', 12345) |
| 773 | self.mock_drone_manager.get_dummy_process.expect_call().and_return( |
| 774 | dummy_process) |
| 775 | self.monitor._start_time = time.time() - monitor_db.PIDFILE_TIMEOUT - 1 |
| 776 | self._test_get_pidfile_info_helper(12345, 1, 0) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 777 | |
| 778 | |
| 779 | class AgentTest(unittest.TestCase): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 780 | def setUp(self): |
| 781 | self.god = mock.mock_god() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 782 | |
| 783 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 784 | def tearDown(self): |
| 785 | self.god.unstub_all() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 786 | |
| 787 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 788 | def _create_mock_task(self, name): |
| 789 | task = self.god.create_mock_class(monitor_db.AgentTask, name) |
| 790 | task.host_ids = task.queue_entry_ids = [] |
| 791 | return task |
| 792 | |
| 793 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 794 | def test_agent(self): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 795 | task1 = self._create_mock_task('task1') |
| 796 | task2 = self._create_mock_task('task2') |
| 797 | task3 = self._create_mock_task('task3') |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 798 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 799 | task1.start.expect_call() |
| 800 | task1.is_done.expect_call().and_return(False) |
| 801 | task1.poll.expect_call() |
| 802 | task1.is_done.expect_call().and_return(True) |
| 803 | task1.is_done.expect_call().and_return(True) |
| 804 | task1.success = True |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 805 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 806 | task2.start.expect_call() |
| 807 | task2.is_done.expect_call().and_return(True) |
| 808 | task2.is_done.expect_call().and_return(True) |
| 809 | task2.success = False |
| 810 | task2.failure_tasks = [task3] |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 811 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 812 | task3.start.expect_call() |
| 813 | task3.is_done.expect_call().and_return(True) |
| 814 | task3.is_done.expect_call().and_return(True) |
| 815 | task3.success = True |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 816 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 817 | agent = monitor_db.Agent([task1, task2]) |
| 818 | agent.dispatcher = object() |
| 819 | agent.start() |
| 820 | while not agent.is_done(): |
| 821 | agent.tick() |
| 822 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 823 | |
| 824 | |
| 825 | class AgentTasksTest(unittest.TestCase): |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 826 | TEMP_DIR = '/abspath/tempdir' |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 827 | RESULTS_DIR = '/results/dir' |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 828 | HOSTNAME = 'myhost' |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 829 | DUMMY_PROCESS = object() |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 830 | HOST_PROTECTION = host_protections.default |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 831 | PIDFILE_ID = object() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 832 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 833 | def setUp(self): |
| 834 | self.god = mock.mock_god() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 835 | self.god.stub_with(drone_manager.DroneManager, 'get_temporary_path', |
| 836 | mock.mock_function('get_temporary_path', |
| 837 | default_return_val='tempdir')) |
| 838 | self.god.stub_function(drone_manager.DroneManager, |
| 839 | 'copy_to_results_repository') |
| 840 | self.god.stub_function(drone_manager.DroneManager, |
| 841 | 'get_pidfile_id_from') |
| 842 | |
| 843 | def dummy_absolute_path(self, path): |
| 844 | return '/abspath/' + path |
| 845 | self.god.stub_with(drone_manager.DroneManager, 'absolute_path', |
| 846 | dummy_absolute_path) |
| 847 | |
| 848 | self.god.stub_class_method(monitor_db.PidfileRunMonitor, 'run') |
| 849 | self.god.stub_class_method(monitor_db.PidfileRunMonitor, 'exit_code') |
| 850 | self.god.stub_class_method(monitor_db.PidfileRunMonitor, 'get_process') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 851 | self.host = self.god.create_mock_class(monitor_db.Host, 'host') |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 852 | self.host.id = 1 |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 853 | self.host.hostname = self.HOSTNAME |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 854 | self.host.protection = self.HOST_PROTECTION |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 855 | self.queue_entry = self.god.create_mock_class( |
| 856 | monitor_db.HostQueueEntry, 'queue_entry') |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 857 | self.job = self.god.create_mock_class(monitor_db.Job, 'job') |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 858 | self.queue_entry.id = 1 |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 859 | self.queue_entry.job = self.job |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 860 | self.queue_entry.host = self.host |
| 861 | self.queue_entry.meta_host = None |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 862 | |
| 863 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 864 | def tearDown(self): |
| 865 | self.god.unstub_all() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 866 | |
| 867 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 868 | def run_task(self, task, success): |
| 869 | """ |
| 870 | Do essentially what an Agent would do, but protect againt |
| 871 | infinite looping from test errors. |
| 872 | """ |
| 873 | if not getattr(task, 'agent', None): |
| 874 | task.agent = object() |
| 875 | task.start() |
| 876 | count = 0 |
| 877 | while not task.is_done(): |
| 878 | count += 1 |
| 879 | if count > 10: |
| 880 | print 'Task failed to finish' |
| 881 | # in case the playback has clues to why it |
| 882 | # failed |
| 883 | self.god.check_playback() |
| 884 | self.fail() |
| 885 | task.poll() |
| 886 | self.assertEquals(task.success, success) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 887 | |
| 888 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 889 | def setup_run_monitor(self, exit_status, copy_log_file=True): |
| 890 | monitor_db.PidfileRunMonitor.run.expect_call( |
| 891 | mock.is_instance_comparator(list), |
| 892 | 'tempdir', |
| 893 | nice_level=monitor_db.AUTOSERV_NICE_LEVEL, |
| 894 | log_file=mock.anything_comparator()) |
| 895 | monitor_db.PidfileRunMonitor.exit_code.expect_call() |
| 896 | monitor_db.PidfileRunMonitor.exit_code.expect_call().and_return( |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 897 | exit_status) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 898 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 899 | if copy_log_file: |
| 900 | self._setup_move_logfile() |
| 901 | |
| 902 | |
| 903 | def _setup_move_logfile(self, include_destination=False): |
| 904 | monitor_db.PidfileRunMonitor.get_process.expect_call().and_return( |
| 905 | self.DUMMY_PROCESS) |
| 906 | if include_destination: |
| 907 | drone_manager.DroneManager.copy_to_results_repository.expect_call( |
| 908 | self.DUMMY_PROCESS, mock.is_string_comparator(), |
| 909 | destination_path=mock.is_string_comparator()) |
| 910 | else: |
| 911 | drone_manager.DroneManager.copy_to_results_repository.expect_call( |
| 912 | self.DUMMY_PROCESS, mock.is_string_comparator()) |
| 913 | |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 914 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 915 | def _test_repair_task_helper(self, success): |
| 916 | self.host.set_status.expect_call('Repairing') |
| 917 | if success: |
| 918 | self.setup_run_monitor(0) |
| 919 | self.host.set_status.expect_call('Ready') |
| 920 | else: |
| 921 | self.setup_run_monitor(1) |
| 922 | self.host.set_status.expect_call('Repair Failed') |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 923 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 924 | task = monitor_db.RepairTask(self.host) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 925 | self.assertEquals(task.failure_tasks, []) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 926 | self.run_task(task, success) |
jadmanski | fb7cfb1 | 2008-07-09 14:13:21 +0000 | [diff] [blame] | 927 | |
| 928 | expected_protection = host_protections.Protection.get_string( |
| 929 | host_protections.default) |
mbligh | 3e0f7e0 | 2008-07-28 19:42:01 +0000 | [diff] [blame] | 930 | expected_protection = host_protections.Protection.get_attr_name( |
| 931 | expected_protection) |
| 932 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 933 | self.assertTrue(set(task.cmd) >= |
| 934 | set([monitor_db._autoserv_path, '-p', '-R', '-m', |
| 935 | self.HOSTNAME, '-r', self.TEMP_DIR, |
| 936 | '--host-protection', expected_protection])) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 937 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 938 | |
| 939 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 940 | def test_repair_task(self): |
| 941 | self._test_repair_task_helper(True) |
| 942 | self._test_repair_task_helper(False) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 943 | |
| 944 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 945 | def test_repair_task_with_queue_entry(self): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 946 | self.host.set_status.expect_call('Repairing') |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 947 | self.queue_entry.requeue.expect_call() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 948 | self.setup_run_monitor(1) |
| 949 | self.host.set_status.expect_call('Repair Failed') |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 950 | self.queue_entry.handle_host_failure.expect_call() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 951 | |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 952 | task = monitor_db.RepairTask(self.host, self.queue_entry) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 953 | self.run_task(task, False) |
| 954 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 955 | |
| 956 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 957 | def setup_verify_expects(self, success, use_queue_entry): |
| 958 | if use_queue_entry: |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 959 | self.queue_entry.set_status.expect_call('Verifying') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 960 | self.host.set_status.expect_call('Verifying') |
| 961 | if success: |
| 962 | self.setup_run_monitor(0) |
| 963 | self.host.set_status.expect_call('Ready') |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 964 | if use_queue_entry: |
| 965 | self.queue_entry.on_pending.expect_call() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 966 | else: |
| 967 | self.setup_run_monitor(1) |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 968 | if use_queue_entry and not self.queue_entry.meta_host: |
| 969 | self.queue_entry.set_execution_subdir.expect_call() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 970 | self.queue_entry.execution_tag.expect_call().and_return('tag') |
| 971 | self._setup_move_logfile(include_destination=True) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 972 | |
| 973 | |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 974 | def _check_verify_failure_tasks(self, verify_task): |
| 975 | self.assertEquals(len(verify_task.failure_tasks), 1) |
| 976 | repair_task = verify_task.failure_tasks[0] |
| 977 | self.assert_(isinstance(repair_task, monitor_db.RepairTask)) |
| 978 | self.assertEquals(verify_task.host, repair_task.host) |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 979 | if verify_task.queue_entry: |
| 980 | self.assertEquals(repair_task.queue_entry, verify_task.queue_entry) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 981 | else: |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 982 | self.assertEquals(repair_task.queue_entry, None) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 983 | |
| 984 | |
| 985 | def _test_verify_task_helper(self, success, use_queue_entry=False, |
| 986 | use_meta_host=False): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 987 | self.setup_verify_expects(success, use_queue_entry) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 988 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 989 | if use_queue_entry: |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 990 | task = monitor_db.VerifyTask(queue_entry=self.queue_entry) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 991 | else: |
| 992 | task = monitor_db.VerifyTask(host=self.host) |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 993 | self._check_verify_failure_tasks(task) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 994 | self.run_task(task, success) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 995 | self.assertTrue(set(task.cmd) >= |
| 996 | set([monitor_db._autoserv_path, '-p', '-v', '-m', |
| 997 | self.HOSTNAME, '-r', self.TEMP_DIR])) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 998 | self.god.check_playback() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 999 | |
| 1000 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1001 | def test_verify_task_with_host(self): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 1002 | self._test_verify_task_helper(True) |
| 1003 | self._test_verify_task_helper(False) |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 1004 | |
| 1005 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1006 | def test_verify_task_with_queue_entry(self): |
showard | 56193bb | 2008-08-13 20:07:41 +0000 | [diff] [blame] | 1007 | self._test_verify_task_helper(True, use_queue_entry=True) |
| 1008 | self._test_verify_task_helper(False, use_queue_entry=True) |
| 1009 | |
| 1010 | |
| 1011 | def test_verify_task_with_metahost(self): |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 1012 | self.queue_entry.meta_host = 1 |
| 1013 | self.test_verify_task_with_queue_entry() |
jadmanski | 3d161b0 | 2008-06-06 15:43:36 +0000 | [diff] [blame] | 1014 | |
| 1015 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 1016 | def test_abort_task(self): |
| 1017 | queue_entry = self.god.create_mock_class(monitor_db.HostQueueEntry, |
| 1018 | 'queue_entry') |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1019 | queue_entry.id = 1 |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 1020 | queue_entry.host_id, queue_entry.job_id = 1, 2 |
| 1021 | task = self.god.create_mock_class(monitor_db.AgentTask, 'task') |
| 1022 | agent = self.god.create_mock_class(monitor_db.Agent, 'agent') |
| 1023 | agent.active_task = task |
| 1024 | |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 1025 | task.abort.expect_call() |
| 1026 | queue_entry.set_status.expect_call('Aborted') |
| 1027 | |
| 1028 | abort_task = monitor_db.AbortTask(queue_entry, [agent]) |
| 1029 | self.run_task(abort_task, True) |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1030 | self.god.check_playback() |
| 1031 | |
| 1032 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1033 | def _setup_pre_parse_expects(self, autoserv_success): |
| 1034 | self.queue_entry.execution_tag.expect_call().and_return('tag') |
| 1035 | self.pidfile_monitor = monitor_db.PidfileRunMonitor.expect_new() |
| 1036 | self.pidfile_monitor.pidfile_id = self.PIDFILE_ID |
| 1037 | self.pidfile_monitor.attach_to_existing_process.expect_call('tag') |
| 1038 | if autoserv_success: |
| 1039 | code = 0 |
| 1040 | else: |
| 1041 | code = 1 |
| 1042 | self.pidfile_monitor.exit_code.expect_call().and_return(code) |
| 1043 | |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1044 | self.queue_entry.set_status.expect_call('Parsing') |
| 1045 | |
| 1046 | |
| 1047 | def _setup_post_parse_expects(self, autoserv_success): |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1048 | if autoserv_success: |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1049 | status = 'Completed' |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1050 | else: |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1051 | status = 'Failed' |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1052 | self.queue_entry.set_status.expect_call(status) |
| 1053 | |
| 1054 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1055 | def setup_reparse_run_monitor(self): |
| 1056 | autoserv_pidfile_id = object() |
| 1057 | monitor = monitor_db.PidfileRunMonitor.expect_new() |
| 1058 | monitor.run.expect_call( |
| 1059 | mock.is_instance_comparator(list), |
| 1060 | 'tag', |
| 1061 | log_file=mock.anything_comparator(), |
| 1062 | pidfile_name='.parser_execute', |
| 1063 | paired_with_pidfile=self.PIDFILE_ID) |
| 1064 | monitor.exit_code.expect_call() |
| 1065 | monitor.exit_code.expect_call().and_return(0) |
| 1066 | monitor.get_process.expect_call().and_return(self.DUMMY_PROCESS) |
| 1067 | drone_manager.DroneManager.copy_to_results_repository.expect_call( |
| 1068 | self.DUMMY_PROCESS, mock.is_string_comparator()) |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1069 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1070 | |
| 1071 | def _test_final_reparse_task_helper(self, autoserv_success=True): |
| 1072 | self._setup_pre_parse_expects(autoserv_success) |
| 1073 | self.setup_reparse_run_monitor() |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1074 | self._setup_post_parse_expects(autoserv_success) |
| 1075 | |
| 1076 | task = monitor_db.FinalReparseTask([self.queue_entry]) |
| 1077 | self.run_task(task, True) |
| 1078 | |
| 1079 | self.god.check_playback() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1080 | cmd = [monitor_db._parser_path, '--write-pidfile', '-l', '2', '-r', |
| 1081 | '-o', '/abspath/tag'] |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1082 | self.assertEquals(task.cmd, cmd) |
| 1083 | |
| 1084 | |
| 1085 | def test_final_reparse_task(self): |
| 1086 | self.god.stub_class(monitor_db, 'PidfileRunMonitor') |
| 1087 | self._test_final_reparse_task_helper() |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1088 | self._test_final_reparse_task_helper(autoserv_success=False) |
| 1089 | |
| 1090 | |
| 1091 | def test_final_reparse_throttling(self): |
| 1092 | self.god.stub_class(monitor_db, 'PidfileRunMonitor') |
| 1093 | self.god.stub_function(monitor_db.FinalReparseTask, |
| 1094 | '_can_run_new_parse') |
| 1095 | |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1096 | self._setup_pre_parse_expects(True) |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1097 | monitor_db.FinalReparseTask._can_run_new_parse.expect_call().and_return( |
| 1098 | False) |
| 1099 | monitor_db.FinalReparseTask._can_run_new_parse.expect_call().and_return( |
| 1100 | True) |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1101 | self.setup_reparse_run_monitor() |
showard | 97aed50 | 2008-11-04 02:01:24 +0000 | [diff] [blame] | 1102 | self._setup_post_parse_expects(True) |
| 1103 | |
| 1104 | task = monitor_db.FinalReparseTask([self.queue_entry]) |
| 1105 | self.run_task(task, True) |
| 1106 | self.god.check_playback() |
showard | 1be9743 | 2008-10-17 15:30:45 +0000 | [diff] [blame] | 1107 | |
| 1108 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1109 | def _test_cleanup_task_helper(self, success, use_queue_entry=False): |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1110 | if use_queue_entry: |
| 1111 | self.queue_entry.get_host.expect_call().and_return(self.host) |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1112 | self.host.set_status.expect_call('Cleaning') |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1113 | if success: |
| 1114 | self.setup_run_monitor(0) |
| 1115 | self.host.set_status.expect_call('Ready') |
| 1116 | self.host.update_field.expect_call('dirty', 0) |
| 1117 | else: |
| 1118 | self.setup_run_monitor(1) |
showard | 8fe93b5 | 2008-11-18 17:53:22 +0000 | [diff] [blame] | 1119 | if use_queue_entry and not self.queue_entry.meta_host: |
| 1120 | self.queue_entry.set_execution_subdir.expect_call() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1121 | self.queue_entry.execution_tag.expect_call().and_return('tag') |
| 1122 | self._setup_move_logfile(include_destination=True) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1123 | |
| 1124 | if use_queue_entry: |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1125 | task = monitor_db.CleanupTask(queue_entry=self.queue_entry) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1126 | else: |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1127 | task = monitor_db.CleanupTask(host=self.host) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1128 | self.assertEquals(len(task.failure_tasks), 1) |
| 1129 | repair_task = task.failure_tasks[0] |
| 1130 | self.assert_(isinstance(repair_task, monitor_db.RepairTask)) |
| 1131 | if use_queue_entry: |
showard | e788ea6 | 2008-11-17 21:02:47 +0000 | [diff] [blame] | 1132 | self.assertEquals(repair_task.queue_entry, self.queue_entry) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1133 | |
| 1134 | self.run_task(task, success) |
| 1135 | |
| 1136 | self.god.check_playback() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1137 | self.assert_(set(task.cmd) >= |
| 1138 | set([monitor_db._autoserv_path, '-p', '--cleanup', '-m', |
| 1139 | self.HOSTNAME, '-r', self.TEMP_DIR])) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1140 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1141 | def test_cleanup_task(self): |
| 1142 | self._test_cleanup_task_helper(True) |
| 1143 | self._test_cleanup_task_helper(False) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1144 | |
| 1145 | |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1146 | def test_cleanup_task_with_queue_entry(self): |
| 1147 | self._test_cleanup_task_helper(False, True) |
showard | fa8629c | 2008-11-04 16:51:23 +0000 | [diff] [blame] | 1148 | |
| 1149 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1150 | class JobTest(BaseSchedulerTest): |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1151 | def setUp(self): |
| 1152 | super(JobTest, self).setUp() |
showard | 170873e | 2009-01-07 00:22:26 +0000 | [diff] [blame] | 1153 | self.god.stub_with( |
| 1154 | drone_manager.DroneManager, 'attach_file_to_execution', |
| 1155 | mock.mock_function('attach_file_to_execution', |
| 1156 | default_return_val='/test/path/tmp/foo')) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1157 | |
| 1158 | |
| 1159 | def _setup_directory_expects(self, execution_subdir): |
| 1160 | job_path = os.path.join('.', '1-my_user') |
| 1161 | results_dir = os.path.join(job_path, execution_subdir) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1162 | |
| 1163 | |
showard | e58e3f8 | 2008-11-20 19:04:59 +0000 | [diff] [blame] | 1164 | def _test_run_helper(self, expect_agent=True, expect_starting=False, |
| 1165 | expect_pending=False): |
| 1166 | if expect_starting: |
| 1167 | expected_status = models.HostQueueEntry.Status.STARTING |
| 1168 | elif expect_pending: |
| 1169 | expected_status = models.HostQueueEntry.Status.PENDING |
| 1170 | else: |
| 1171 | expected_status = models.HostQueueEntry.Status.VERIFYING |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1172 | job = monitor_db.Job.fetch('id = 1').next() |
| 1173 | queue_entry = monitor_db.HostQueueEntry.fetch('id = 1').next() |
| 1174 | agent = job.run(queue_entry) |
| 1175 | |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1176 | self.god.check_playback() |
showard | e77ac67 | 2008-11-14 22:42:33 +0000 | [diff] [blame] | 1177 | self.assertEquals(models.HostQueueEntry.smart_get(1).status, |
| 1178 | expected_status) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1179 | |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1180 | if not expect_agent: |
| 1181 | self.assertEquals(agent, None) |
| 1182 | return |
| 1183 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1184 | self.assert_(isinstance(agent, monitor_db.Agent)) |
| 1185 | tasks = list(agent.queue.queue) |
| 1186 | return tasks |
| 1187 | |
| 1188 | |
| 1189 | def test_run_asynchronous(self): |
| 1190 | self._create_job(hosts=[1, 2]) |
| 1191 | |
| 1192 | tasks = self._test_run_helper() |
| 1193 | |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1194 | self.assertEquals(len(tasks), 1) |
| 1195 | verify_task = tasks[0] |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1196 | |
| 1197 | self.assert_(isinstance(verify_task, monitor_db.VerifyTask)) |
| 1198 | self.assertEquals(verify_task.queue_entry.id, 1) |
| 1199 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1200 | |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1201 | def test_run_asynchronous_skip_verify(self): |
| 1202 | job = self._create_job(hosts=[1, 2]) |
| 1203 | job.run_verify = False |
| 1204 | job.save() |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1205 | self._setup_directory_expects('host1') |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1206 | |
showard | e58e3f8 | 2008-11-20 19:04:59 +0000 | [diff] [blame] | 1207 | tasks = self._test_run_helper(expect_starting=True) |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1208 | |
| 1209 | self.assertEquals(len(tasks), 1) |
| 1210 | queue_task = tasks[0] |
| 1211 | |
| 1212 | self.assert_(isinstance(queue_task, monitor_db.QueueTask)) |
| 1213 | self.assertEquals(queue_task.job.id, 1) |
| 1214 | |
| 1215 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1216 | def test_run_synchronous_verify(self): |
| 1217 | self._create_job(hosts=[1, 2], synchronous=True) |
| 1218 | |
| 1219 | tasks = self._test_run_helper() |
| 1220 | self.assertEquals(len(tasks), 1) |
| 1221 | verify_task = tasks[0] |
| 1222 | |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1223 | self.assert_(isinstance(verify_task, monitor_db.VerifyTask)) |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1224 | self.assertEquals(verify_task.queue_entry.id, 1) |
| 1225 | |
| 1226 | |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1227 | def test_run_synchronous_skip_verify(self): |
| 1228 | job = self._create_job(hosts=[1, 2], synchronous=True) |
| 1229 | job.run_verify = False |
| 1230 | job.save() |
| 1231 | |
showard | e58e3f8 | 2008-11-20 19:04:59 +0000 | [diff] [blame] | 1232 | self._test_run_helper(expect_agent=False, expect_pending=True) |
showard | 9976ce9 | 2008-10-15 20:28:13 +0000 | [diff] [blame] | 1233 | |
| 1234 | queue_entry = models.HostQueueEntry.smart_get(1) |
| 1235 | self.assertEquals(queue_entry.status, 'Pending') |
| 1236 | |
| 1237 | |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1238 | def test_run_synchronous_ready(self): |
| 1239 | self._create_job(hosts=[1, 2], synchronous=True) |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1240 | self._update_hqe("status='Pending', execution_subdir='") |
| 1241 | self._setup_directory_expects('group0') |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1242 | |
showard | e58e3f8 | 2008-11-20 19:04:59 +0000 | [diff] [blame] | 1243 | tasks = self._test_run_helper(expect_starting=True) |
showard | b2e2c32 | 2008-10-14 17:33:55 +0000 | [diff] [blame] | 1244 | self.assertEquals(len(tasks), 1) |
| 1245 | queue_task = tasks[0] |
| 1246 | |
| 1247 | self.assert_(isinstance(queue_task, monitor_db.QueueTask)) |
| 1248 | self.assertEquals(queue_task.job.id, 1) |
| 1249 | hqe_ids = [hqe.id for hqe in queue_task.queue_entries] |
| 1250 | self.assertEquals(hqe_ids, [1, 2]) |
| 1251 | |
| 1252 | |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1253 | def test_reboot_before_always(self): |
| 1254 | job = self._create_job(hosts=[1]) |
showard | 0fc3830 | 2008-10-23 00:44:07 +0000 | [diff] [blame] | 1255 | job.reboot_before = models.RebootBefore.ALWAYS |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1256 | job.save() |
| 1257 | |
| 1258 | tasks = self._test_run_helper() |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1259 | self.assertEquals(len(tasks), 2) |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1260 | cleanup_task = tasks[0] |
| 1261 | self.assert_(isinstance(cleanup_task, monitor_db.CleanupTask)) |
| 1262 | self.assertEquals(cleanup_task.host.id, 1) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1263 | |
| 1264 | |
| 1265 | def _test_reboot_before_if_dirty_helper(self, expect_reboot): |
| 1266 | job = self._create_job(hosts=[1]) |
showard | 0fc3830 | 2008-10-23 00:44:07 +0000 | [diff] [blame] | 1267 | job.reboot_before = models.RebootBefore.IF_DIRTY |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1268 | job.save() |
| 1269 | |
| 1270 | tasks = self._test_run_helper() |
showard | 2bab8f4 | 2008-11-12 18:15:22 +0000 | [diff] [blame] | 1271 | self.assertEquals(len(tasks), expect_reboot and 2 or 1) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1272 | if expect_reboot: |
showard | 45ae819 | 2008-11-05 19:32:53 +0000 | [diff] [blame] | 1273 | cleanup_task = tasks[0] |
| 1274 | self.assert_(isinstance(cleanup_task, monitor_db.CleanupTask)) |
| 1275 | self.assertEquals(cleanup_task.host.id, 1) |
showard | 21baa45 | 2008-10-21 00:08:39 +0000 | [diff] [blame] | 1276 | |
| 1277 | def test_reboot_before_if_dirty(self): |
| 1278 | models.Host.smart_get(1).update_object(dirty=True) |
| 1279 | self._test_reboot_before_if_dirty_helper(True) |
| 1280 | |
| 1281 | |
| 1282 | def test_reboot_before_not_dirty(self): |
| 1283 | models.Host.smart_get(1).update_object(dirty=False) |
| 1284 | self._test_reboot_before_if_dirty_helper(False) |
| 1285 | |
| 1286 | |
| 1287 | |
showard | ce38e0c | 2008-05-29 19:36:16 +0000 | [diff] [blame] | 1288 | if __name__ == '__main__': |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 1289 | unittest.main() |