mbligh | fa29a2a | 2008-05-16 22:48:09 +0000 | [diff] [blame] | 1 | #!/usr/bin/python -u |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 2 | |
| 3 | import os, sys, re, subprocess, tempfile |
| 4 | import MySQLdb, MySQLdb.constants.ER |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 5 | from optparse import OptionParser |
mbligh | 9b907d6 | 2008-05-13 17:56:24 +0000 | [diff] [blame] | 6 | import common |
| 7 | from autotest_lib.client.common_lib import global_config |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 8 | from autotest_lib.database import database_connection |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 9 | |
| 10 | MIGRATE_TABLE = 'migrate_info' |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 11 | |
| 12 | _AUTODIR = os.path.join(os.path.dirname(__file__), '..') |
| 13 | _MIGRATIONS_DIRS = { |
| 14 | 'AUTOTEST_WEB' : os.path.join(_AUTODIR, 'frontend', 'migrations'), |
| 15 | 'TKO' : os.path.join(_AUTODIR, 'tko', 'migrations'), |
| 16 | } |
| 17 | _DEFAULT_MIGRATIONS_DIR = 'migrations' # use CWD |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 18 | |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 19 | class Migration(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 20 | def __init__(self, filename): |
| 21 | self.version = int(filename[:3]) |
| 22 | self.name = filename[:-3] |
| 23 | self.module = __import__(self.name, globals(), locals(), []) |
| 24 | assert hasattr(self.module, 'migrate_up') |
| 25 | assert hasattr(self.module, 'migrate_down') |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 26 | |
| 27 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 28 | def migrate_up(self, manager): |
| 29 | self.module.migrate_up(manager) |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 30 | |
| 31 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 32 | def migrate_down(self, manager): |
| 33 | self.module.migrate_down(manager) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 34 | |
| 35 | |
| 36 | class MigrationManager(object): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 37 | connection = None |
| 38 | cursor = None |
| 39 | migrations_dir = None |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 40 | |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 41 | def __init__(self, database_connection, migrations_dir=None, force=False): |
| 42 | self._database = database_connection |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 43 | self.force = force |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 44 | self._set_migrations_dir(migrations_dir) |
| 45 | |
| 46 | |
| 47 | def _set_migrations_dir(self, migrations_dir=None): |
| 48 | config_section = self._database.global_config_section |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 49 | if migrations_dir is None: |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 50 | migrations_dir = os.path.abspath( |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 51 | _MIGRATIONS_DIRS.get(config_section, _DEFAULT_MIGRATIONS_DIR)) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 52 | self.migrations_dir = migrations_dir |
| 53 | sys.path.append(migrations_dir) |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 54 | assert os.path.exists(migrations_dir), migrations_dir + " doesn't exist" |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 55 | |
| 56 | |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 57 | def _get_db_name(self): |
| 58 | return self._database.get_database_info()['db_name'] |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 59 | |
| 60 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 61 | def execute(self, query, *parameters): |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 62 | return self._database.execute(query, parameters) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 63 | |
| 64 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 65 | def execute_script(self, script): |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame^] | 66 | sql_statements = [statement.strip() |
| 67 | for statement in script.split(';') |
| 68 | if statement.strip()] |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 69 | for statement in sql_statements: |
showard | b1e5187 | 2008-10-07 11:08:18 +0000 | [diff] [blame^] | 70 | self.execute(statement) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 71 | |
| 72 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 73 | def check_migrate_table_exists(self): |
| 74 | try: |
| 75 | self.execute("SELECT * FROM %s" % MIGRATE_TABLE) |
| 76 | return True |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 77 | except self._database.DatabaseError, exc: |
| 78 | # we can't check for more specifics due to differences between DB |
| 79 | # backends (we can't even check for a subclass of DatabaseError) |
| 80 | return False |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 81 | |
| 82 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 83 | def create_migrate_table(self): |
| 84 | if not self.check_migrate_table_exists(): |
| 85 | self.execute("CREATE TABLE %s (`version` integer)" % |
| 86 | MIGRATE_TABLE) |
| 87 | else: |
| 88 | self.execute("DELETE FROM %s" % MIGRATE_TABLE) |
| 89 | self.execute("INSERT INTO %s VALUES (0)" % MIGRATE_TABLE) |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 90 | assert self._database.rowcount == 1 |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 91 | |
| 92 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 93 | def set_db_version(self, version): |
| 94 | assert isinstance(version, int) |
| 95 | self.execute("UPDATE %s SET version=%%s" % MIGRATE_TABLE, |
| 96 | version) |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 97 | assert self._database.rowcount == 1 |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 98 | |
| 99 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 100 | def get_db_version(self): |
| 101 | if not self.check_migrate_table_exists(): |
| 102 | return 0 |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 103 | rows = self.execute("SELECT * FROM %s" % MIGRATE_TABLE) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 104 | if len(rows) == 0: |
| 105 | return 0 |
| 106 | assert len(rows) == 1 and len(rows[0]) == 1 |
| 107 | return rows[0][0] |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 108 | |
| 109 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 110 | def get_migrations(self, minimum_version=None, maximum_version=None): |
| 111 | migrate_files = [filename for filename |
| 112 | in os.listdir(self.migrations_dir) |
| 113 | if re.match(r'^\d\d\d_.*\.py$', filename)] |
| 114 | migrate_files.sort() |
| 115 | migrations = [Migration(filename) for filename in migrate_files] |
| 116 | if minimum_version is not None: |
| 117 | migrations = [migration for migration in migrations |
| 118 | if migration.version >= minimum_version] |
| 119 | if maximum_version is not None: |
| 120 | migrations = [migration for migration in migrations |
| 121 | if migration.version <= maximum_version] |
| 122 | return migrations |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 123 | |
| 124 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 125 | def do_migration(self, migration, migrate_up=True): |
| 126 | print 'Applying migration %s' % migration.name, # no newline |
| 127 | if migrate_up: |
| 128 | print 'up' |
| 129 | assert self.get_db_version() == migration.version - 1 |
| 130 | migration.migrate_up(self) |
| 131 | new_version = migration.version |
| 132 | else: |
| 133 | print 'down' |
| 134 | assert self.get_db_version() == migration.version |
| 135 | migration.migrate_down(self) |
| 136 | new_version = migration.version - 1 |
| 137 | self.set_db_version(new_version) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 138 | |
| 139 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 140 | def migrate_to_version(self, version): |
| 141 | current_version = self.get_db_version() |
| 142 | if current_version < version: |
| 143 | lower, upper = current_version, version |
| 144 | migrate_up = True |
| 145 | else: |
| 146 | lower, upper = version, current_version |
| 147 | migrate_up = False |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 148 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 149 | migrations = self.get_migrations(lower + 1, upper) |
| 150 | if not migrate_up: |
| 151 | migrations.reverse() |
| 152 | for migration in migrations: |
| 153 | self.do_migration(migration, migrate_up) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 154 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 155 | assert self.get_db_version() == version |
| 156 | print 'At version', version |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 157 | |
| 158 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 159 | def get_latest_version(self): |
| 160 | migrations = self.get_migrations() |
| 161 | return migrations[-1].version |
showard | d2d4e2c | 2008-03-12 21:32:46 +0000 | [diff] [blame] | 162 | |
| 163 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 164 | def migrate_to_latest(self): |
| 165 | latest_version = self.get_latest_version() |
| 166 | self.migrate_to_version(latest_version) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 167 | |
| 168 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 169 | def initialize_test_db(self): |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 170 | db_name = self._get_db_name() |
| 171 | test_db_name = 'test_' + db_name |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 172 | # first, connect to no DB so we can create a test DB |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 173 | self._database.connect(db_name='') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 174 | print 'Creating test DB', test_db_name |
| 175 | self.execute('CREATE DATABASE ' + test_db_name) |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 176 | self._database.disconnect() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 177 | # now connect to the test DB |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 178 | self._database.connect(db_name=test_db_name) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 179 | |
| 180 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 181 | def remove_test_db(self): |
| 182 | print 'Removing test DB' |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 183 | self.execute('DROP DATABASE ' + self._get_db_name()) |
| 184 | # reset connection back to real DB |
| 185 | self._database.disconnect() |
| 186 | self._database.connect() |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 187 | |
| 188 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 189 | def get_mysql_args(self): |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 190 | return ('-u %(username)s -p%(password)s -h %(host)s %(db_name)s' % |
| 191 | self._database.get_database_info()) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 192 | |
| 193 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 194 | def migrate_to_version_or_latest(self, version): |
| 195 | if version is None: |
| 196 | self.migrate_to_latest() |
| 197 | else: |
| 198 | self.migrate_to_version(version) |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 199 | |
| 200 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 201 | def do_sync_db(self, version=None): |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 202 | print 'Migration starting for database', self._get_db_name() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 203 | self.migrate_to_version_or_latest(version) |
| 204 | print 'Migration complete' |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 205 | |
| 206 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 207 | def test_sync_db(self, version=None): |
| 208 | """\ |
| 209 | Create a fresh DB and run all migrations on it. |
| 210 | """ |
| 211 | self.initialize_test_db() |
| 212 | try: |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 213 | print 'Starting migration test on DB', self._get_db_name() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 214 | self.migrate_to_version_or_latest(version) |
| 215 | # show schema to the user |
| 216 | os.system('mysqldump %s --no-data=true ' |
| 217 | '--add-drop-table=false' % |
| 218 | self.get_mysql_args()) |
| 219 | finally: |
| 220 | self.remove_test_db() |
| 221 | print 'Test finished successfully' |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 222 | |
| 223 | |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 224 | def simulate_sync_db(self, version=None): |
| 225 | """\ |
| 226 | Create a fresh DB, copy the existing DB to it, and then |
| 227 | try to synchronize it. |
| 228 | """ |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 229 | db_version = self.get_db_version() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 230 | # don't do anything if we're already at the latest version |
| 231 | if db_version == self.get_latest_version(): |
| 232 | print 'Skipping simulation, already at latest version' |
| 233 | return |
| 234 | # get existing data |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 235 | print 'Dumping existing data' |
| 236 | dump_fd, dump_file = tempfile.mkstemp('.migrate_dump') |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 237 | os.system('mysqldump %s >%s' % |
| 238 | (self.get_mysql_args(), dump_file)) |
| 239 | # fill in test DB |
| 240 | self.initialize_test_db() |
| 241 | print 'Filling in test DB' |
| 242 | os.system('mysql %s <%s' % (self.get_mysql_args(), dump_file)) |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 243 | os.close(dump_fd) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 244 | os.remove(dump_file) |
| 245 | try: |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 246 | print 'Starting migration test on DB', self._get_db_name() |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 247 | self.migrate_to_version_or_latest(version) |
| 248 | finally: |
| 249 | self.remove_test_db() |
| 250 | print 'Test finished successfully' |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 251 | |
| 252 | |
mbligh | c2f2445 | 2008-03-31 16:46:13 +0000 | [diff] [blame] | 253 | USAGE = """\ |
| 254 | %s [options] sync|test|simulate|safesync [version] |
| 255 | Options: |
| 256 | -d --database Which database to act on |
| 257 | -a --action Which action to perform"""\ |
| 258 | % sys.argv[0] |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 259 | |
| 260 | |
| 261 | def main(): |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 262 | parser = OptionParser() |
| 263 | parser.add_option("-d", "--database", |
| 264 | help="which database to act on", |
| 265 | dest="database") |
| 266 | parser.add_option("-a", "--action", help="what action to perform", |
| 267 | dest="action") |
showard | 50c0e71 | 2008-09-22 16:20:37 +0000 | [diff] [blame] | 268 | parser.add_option("-f", "--force", help="don't ask for confirmation", |
| 269 | action="store_true") |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 270 | (options, args) = parser.parse_args() |
showard | 0e73c85 | 2008-10-03 10:15:50 +0000 | [diff] [blame] | 271 | database = database_connection.DatabaseConnection(options.database) |
| 272 | database.reconnect_enabled = False |
| 273 | database.connect() |
| 274 | manager = MigrationManager(database, force=options.force) |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 275 | |
| 276 | if len(args) > 0: |
| 277 | if len(args) > 1: |
| 278 | version = int(args[1]) |
| 279 | else: |
| 280 | version = None |
| 281 | if args[0] == 'sync': |
| 282 | manager.do_sync_db(version) |
| 283 | elif args[0] == 'test': |
| 284 | manager.test_sync_db(version) |
| 285 | elif args[0] == 'simulate': |
| 286 | manager.simulate_sync_db(version) |
| 287 | elif args[0] == 'safesync': |
| 288 | print 'Simluating migration' |
| 289 | manager.simulate_sync_db(version) |
| 290 | print 'Performing real migration' |
| 291 | manager.do_sync_db(version) |
| 292 | else: |
| 293 | print USAGE |
| 294 | return |
| 295 | |
| 296 | print USAGE |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 297 | |
| 298 | |
| 299 | if __name__ == '__main__': |
jadmanski | 0afbb63 | 2008-06-06 21:10:57 +0000 | [diff] [blame] | 300 | main() |