mbligh | b5ec487 | 2008-03-14 22:40:37 +0000 | [diff] [blame] | 1 | #!/usr/bin/python2.4 -u |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 2 | |
| 3 | import os, sys, re, subprocess, tempfile |
| 4 | import MySQLdb, MySQLdb.constants.ER |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 5 | from optparse import OptionParser |
mbligh | 9b907d6 | 2008-05-13 17:56:24 +0000 | [diff] [blame^] | 6 | import common |
| 7 | from autotest_lib.client.common_lib import global_config |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 8 | |
| 9 | MIGRATE_TABLE = 'migrate_info' |
| 10 | DEFAULT_MIGRATIONS_DIR = 'migrations' |
| 11 | |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 12 | class Migration(object): |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 13 | def __init__(self, filename): |
| 14 | self.version = int(filename[:3]) |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 15 | self.name = filename[:-3] |
| 16 | self.module = __import__(self.name, globals(), locals(), []) |
| 17 | assert hasattr(self.module, 'migrate_up') |
| 18 | assert hasattr(self.module, 'migrate_down') |
| 19 | |
| 20 | |
| 21 | def migrate_up(self, manager): |
| 22 | self.module.migrate_up(manager) |
| 23 | |
| 24 | |
| 25 | def migrate_down(self, manager): |
| 26 | self.module.migrate_down(manager) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 27 | |
| 28 | |
| 29 | class MigrationManager(object): |
| 30 | connection = None |
| 31 | cursor = None |
| 32 | migrations_dir = None |
| 33 | |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 34 | def __init__(self, database, migrations_dir=None): |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 35 | self.database = database |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 36 | if migrations_dir is None: |
| 37 | migrations_dir = os.path.abspath(DEFAULT_MIGRATIONS_DIR) |
| 38 | self.migrations_dir = migrations_dir |
| 39 | sys.path.append(migrations_dir) |
| 40 | assert os.path.exists(migrations_dir) |
| 41 | |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 42 | self.db_host = None |
| 43 | self.db_name = None |
| 44 | self.username = None |
| 45 | self.password = None |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 46 | |
| 47 | |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 48 | def read_db_info(self): |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 49 | # grab the config file and parse for info |
| 50 | c = global_config.global_config |
mbligh | 9b907d6 | 2008-05-13 17:56:24 +0000 | [diff] [blame^] | 51 | sections = c.get_sections() |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 52 | self.db_host = c.get_config_value(self.database, "host") |
| 53 | self.db_name = c.get_config_value(self.database, "database") |
| 54 | self.username = c.get_config_value(self.database, "user") |
| 55 | self.password = c.get_config_value(self.database, "password") |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 56 | |
| 57 | |
| 58 | def connect(self, host, db_name, username, password): |
| 59 | return MySQLdb.connect(host=host, db=db_name, user=username, |
| 60 | passwd=password) |
| 61 | |
| 62 | |
| 63 | def open_connection(self): |
| 64 | self.connection = self.connect(self.db_host, self.db_name, |
| 65 | self.username, self.password) |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 66 | self.connection.autocommit(True) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 67 | self.cursor = self.connection.cursor() |
| 68 | |
| 69 | |
| 70 | def close_connection(self): |
| 71 | self.connection.close() |
| 72 | |
| 73 | |
| 74 | def execute(self, query, *parameters): |
| 75 | #print 'SQL:', query % parameters |
| 76 | return self.cursor.execute(query, parameters) |
| 77 | |
| 78 | |
| 79 | def execute_script(self, script): |
| 80 | sql_statements = [statement.strip() for statement |
| 81 | in script.split(';')] |
| 82 | for statement in sql_statements: |
| 83 | if statement: |
| 84 | self.execute(statement) |
| 85 | |
| 86 | |
| 87 | def check_migrate_table_exists(self): |
| 88 | try: |
| 89 | self.execute("SELECT * FROM %s" % MIGRATE_TABLE) |
| 90 | return True |
| 91 | except MySQLdb.ProgrammingError, exc: |
| 92 | error_code, _ = exc.args |
| 93 | if error_code == MySQLdb.constants.ER.NO_SUCH_TABLE: |
| 94 | return False |
| 95 | raise |
| 96 | |
| 97 | |
| 98 | def create_migrate_table(self): |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 99 | if not self.check_migrate_table_exists(): |
| 100 | self.execute("CREATE TABLE %s (`version` integer)" % |
| 101 | MIGRATE_TABLE) |
| 102 | else: |
| 103 | self.execute("DELETE FROM %s" % MIGRATE_TABLE) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 104 | self.execute("INSERT INTO %s VALUES (0)" % MIGRATE_TABLE) |
| 105 | assert self.cursor.rowcount == 1 |
| 106 | |
| 107 | |
| 108 | def set_db_version(self, version): |
| 109 | assert isinstance(version, int) |
| 110 | self.execute("UPDATE %s SET version=%%s" % MIGRATE_TABLE, |
| 111 | version) |
| 112 | assert self.cursor.rowcount == 1 |
| 113 | |
| 114 | |
| 115 | def get_db_version(self): |
| 116 | if not self.check_migrate_table_exists(): |
| 117 | return 0 |
| 118 | self.execute("SELECT * FROM %s" % MIGRATE_TABLE) |
| 119 | rows = self.cursor.fetchall() |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 120 | if len(rows) == 0: |
| 121 | return 0 |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 122 | assert len(rows) == 1 and len(rows[0]) == 1 |
| 123 | return rows[0][0] |
| 124 | |
| 125 | |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 126 | def get_migrations(self, minimum_version=None, maximum_version=None): |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 127 | migrate_files = [filename for filename |
| 128 | in os.listdir(self.migrations_dir) |
| 129 | if re.match(r'^\d\d\d_.*\.py$', filename)] |
| 130 | migrate_files.sort() |
| 131 | migrations = [Migration(filename) for filename in migrate_files] |
| 132 | if minimum_version is not None: |
| 133 | migrations = [migration for migration in migrations |
| 134 | if migration.version >= minimum_version] |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 135 | if maximum_version is not None: |
| 136 | migrations = [migration for migration in migrations |
| 137 | if migration.version <= maximum_version] |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 138 | return migrations |
| 139 | |
| 140 | |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 141 | def do_migration(self, migration, migrate_up=True): |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 142 | print 'Applying migration %s' % migration.name, # no newline |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 143 | if migrate_up: |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 144 | print 'up' |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 145 | assert self.get_db_version() == migration.version - 1 |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 146 | migration.migrate_up(self) |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 147 | new_version = migration.version |
| 148 | else: |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 149 | print 'down' |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 150 | assert self.get_db_version() == migration.version |
showard | decbe50 | 2008-03-28 16:31:10 +0000 | [diff] [blame] | 151 | migration.migrate_down(self) |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 152 | new_version = migration.version - 1 |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 153 | self.set_db_version(new_version) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 154 | |
| 155 | |
| 156 | def migrate_to_version(self, version): |
| 157 | current_version = self.get_db_version() |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 158 | if current_version < version: |
| 159 | lower, upper = current_version, version |
| 160 | migrate_up = True |
| 161 | else: |
| 162 | lower, upper = version, current_version |
| 163 | migrate_up = False |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 164 | |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 165 | migrations = self.get_migrations(lower + 1, upper) |
| 166 | if not migrate_up: |
| 167 | migrations.reverse() |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 168 | for migration in migrations: |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 169 | self.do_migration(migration, migrate_up) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 170 | |
| 171 | assert self.get_db_version() == version |
| 172 | print 'At version', version |
| 173 | |
| 174 | |
showard | d2d4e2c | 2008-03-12 21:32:46 +0000 | [diff] [blame] | 175 | def get_latest_version(self): |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 176 | migrations = self.get_migrations() |
showard | d2d4e2c | 2008-03-12 21:32:46 +0000 | [diff] [blame] | 177 | return migrations[-1].version |
| 178 | |
| 179 | |
| 180 | def migrate_to_latest(self): |
| 181 | latest_version = self.get_latest_version() |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 182 | self.migrate_to_version(latest_version) |
| 183 | |
| 184 | |
| 185 | def initialize_test_db(self): |
| 186 | self.read_db_info() |
| 187 | test_db_name = 'test_' + self.db_name |
| 188 | # first, connect to no DB so we can create a test DB |
| 189 | self.db_name = '' |
| 190 | self.open_connection() |
| 191 | print 'Creating test DB', test_db_name |
| 192 | self.execute('CREATE DATABASE ' + test_db_name) |
| 193 | self.close_connection() |
| 194 | # now connect to the test DB |
| 195 | self.db_name = test_db_name |
| 196 | self.open_connection() |
| 197 | |
| 198 | |
| 199 | def remove_test_db(self): |
| 200 | print 'Removing test DB' |
| 201 | self.execute('DROP DATABASE ' + self.db_name) |
| 202 | |
| 203 | |
| 204 | def get_mysql_args(self): |
| 205 | return ('-u %(user)s -p%(password)s -h %(host)s %(db)s' % { |
| 206 | 'user' : self.username, |
| 207 | 'password' : self.password, |
| 208 | 'host' : self.db_host, |
| 209 | 'db' : self.db_name}) |
| 210 | |
| 211 | |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 212 | def migrate_to_version_or_latest(self, version): |
| 213 | if version is None: |
| 214 | self.migrate_to_latest() |
| 215 | else: |
| 216 | self.migrate_to_version(version) |
| 217 | |
| 218 | |
| 219 | def do_sync_db(self, version=None): |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 220 | self.read_db_info() |
| 221 | self.open_connection() |
| 222 | print 'Migration starting for database', self.db_name |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 223 | self.migrate_to_version_or_latest(version) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 224 | print 'Migration complete' |
| 225 | |
| 226 | |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 227 | def test_sync_db(self, version=None): |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 228 | """\ |
| 229 | Create a fresh DB and run all migrations on it. |
| 230 | """ |
| 231 | self.initialize_test_db() |
| 232 | try: |
| 233 | print 'Starting migration test on DB', self.db_name |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 234 | self.migrate_to_version_or_latest(version) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 235 | # show schema to the user |
| 236 | os.system('mysqldump %s --no-data=true ' |
| 237 | '--add-drop-table=false' % |
| 238 | self.get_mysql_args()) |
| 239 | finally: |
| 240 | self.remove_test_db() |
| 241 | print 'Test finished successfully' |
| 242 | |
| 243 | |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 244 | def simulate_sync_db(self, version=None): |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 245 | """\ |
| 246 | Create a fresh DB, copy the existing DB to it, and then |
| 247 | try to synchronize it. |
| 248 | """ |
showard | d2d4e2c | 2008-03-12 21:32:46 +0000 | [diff] [blame] | 249 | self.read_db_info() |
| 250 | self.open_connection() |
| 251 | db_version = self.get_db_version() |
| 252 | self.close_connection() |
| 253 | # don't do anything if we're already at the latest version |
| 254 | if db_version == self.get_latest_version(): |
| 255 | print 'Skipping simulation, already at latest version' |
| 256 | return |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 257 | # get existing data |
| 258 | self.read_db_info() |
| 259 | print 'Dumping existing data' |
| 260 | dump_fd, dump_file = tempfile.mkstemp('.migrate_dump') |
| 261 | os.close(dump_fd) |
| 262 | os.system('mysqldump %s >%s' % |
| 263 | (self.get_mysql_args(), dump_file)) |
| 264 | # fill in test DB |
| 265 | self.initialize_test_db() |
| 266 | print 'Filling in test DB' |
| 267 | os.system('mysql %s <%s' % (self.get_mysql_args(), dump_file)) |
| 268 | os.remove(dump_file) |
| 269 | try: |
| 270 | print 'Starting migration test on DB', self.db_name |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 271 | self.migrate_to_version_or_latest(version) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 272 | finally: |
| 273 | self.remove_test_db() |
| 274 | print 'Test finished successfully' |
| 275 | |
| 276 | |
mbligh | c2f2445 | 2008-03-31 16:46:13 +0000 | [diff] [blame] | 277 | USAGE = """\ |
| 278 | %s [options] sync|test|simulate|safesync [version] |
| 279 | Options: |
| 280 | -d --database Which database to act on |
| 281 | -a --action Which action to perform"""\ |
| 282 | % sys.argv[0] |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 283 | |
| 284 | |
| 285 | def main(): |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 286 | parser = OptionParser() |
| 287 | parser.add_option("-d", "--database", |
| 288 | help="which database to act on", |
| 289 | dest="database") |
| 290 | parser.add_option("-a", "--action", help="what action to perform", |
| 291 | dest="action") |
| 292 | (options, args) = parser.parse_args() |
| 293 | manager = MigrationManager(options.database) |
| 294 | |
| 295 | if len(args) > 0: |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 296 | if len(args) > 1: |
| 297 | version = int(args[1]) |
| 298 | else: |
| 299 | version = None |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 300 | if args[0] == 'sync': |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 301 | manager.do_sync_db(version) |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 302 | elif args[0] == 'test': |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 303 | manager.test_sync_db(version) |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 304 | elif args[0] == 'simulate': |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 305 | manager.simulate_sync_db(version) |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 306 | elif args[0] == 'safesync': |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 307 | print 'Simluating migration' |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 308 | manager.simulate_sync_db(version) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 309 | print 'Performing real migration' |
mbligh | aa383b7 | 2008-03-12 20:11:56 +0000 | [diff] [blame] | 310 | manager.do_sync_db(version) |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 311 | else: |
| 312 | print USAGE |
| 313 | return |
mbligh | b090f14 | 2008-02-27 21:33:46 +0000 | [diff] [blame] | 314 | |
mbligh | e8819cd | 2008-02-15 16:48:40 +0000 | [diff] [blame] | 315 | print USAGE |
| 316 | |
| 317 | |
| 318 | if __name__ == '__main__': |
| 319 | main() |