showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 1 | #!/usr/bin/python2.4 |
| 2 | # |
| 3 | # Copyright 2008 Google Inc. All Rights Reserved. |
| 4 | """ |
| 5 | This utility allows for easy updating, removing and importing |
| 6 | of tests into the autotest_web autotests table. |
| 7 | |
| 8 | Example of updating client side tests: |
| 9 | ./tests.py -t /usr/local/autotest/client/tests |
| 10 | |
| 11 | If for example not all of your control files adhere to the standard outlined at |
| 12 | http://test.kernel.org/autotest/ControlRequirements |
| 13 | |
| 14 | You can force options: |
| 15 | ./tests.py --test-type server -t /usr/local/autotest/server/tests |
| 16 | |
| 17 | |
| 18 | Most options should be fairly self explanatory use --help to display them. |
| 19 | """ |
| 20 | |
| 21 | |
showard | f1175bb | 2009-06-17 19:34:36 +0000 | [diff] [blame] | 22 | import logging, time, re, os, MySQLdb, sys, optparse, compiler |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 23 | import common |
| 24 | from autotest_lib.client.common_lib import control_data, test, global_config |
| 25 | from autotest_lib.client.common_lib import utils |
| 26 | |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 27 | |
showard | f1175bb | 2009-06-17 19:34:36 +0000 | [diff] [blame] | 28 | logging.basicConfig(logging.DEBUG) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 29 | # Global |
| 30 | DRY_RUN = False |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 31 | DEPENDENCIES_NOT_FOUND = set() |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 32 | |
| 33 | def main(argv): |
| 34 | """Main function""" |
| 35 | global DRY_RUN |
| 36 | parser = optparse.OptionParser() |
| 37 | parser.add_option('-c', '--db-clear-tests', |
| 38 | dest='clear_tests', action='store_true', |
| 39 | default=False, |
| 40 | help='Clear client and server tests with invalid control files') |
| 41 | parser.add_option('-d', '--dry-run', |
| 42 | dest='dry_run', action='store_true', default=False, |
| 43 | help='Dry run for operation') |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 44 | parser.add_option('-A', '--add-all', |
| 45 | dest='add_all', action='store_true', |
| 46 | default=False, |
showard | 66a5ef8 | 2008-09-30 16:56:02 +0000 | [diff] [blame] | 47 | help='Add samples, site_tests, tests, and test_suites') |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 48 | parser.add_option('-E', '--add-experimental', |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 49 | dest='add_experimental', action='store_true', |
mbligh | 79410e1 | 2008-11-20 17:59:15 +0000 | [diff] [blame] | 50 | default=True, |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 51 | help='Add experimental tests to frontend') |
| 52 | parser.add_option('-N', '--add-noncompliant', |
| 53 | dest='add_noncompliant', action='store_true', |
| 54 | default=False, |
| 55 | help='Skip any tests that are not compliant') |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 56 | parser.add_option('-p', '--profile-dir', dest='profile_dir', |
| 57 | help='Directory to recursively check for profiles') |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 58 | parser.add_option('-t', '--tests-dir', dest='tests_dir', |
| 59 | help='Directory to recursively check for control.*') |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 60 | parser.add_option('-r', '--control-pattern', dest='control_pattern', |
| 61 | default='^control.*', |
| 62 | help='The pattern to look for in directories for control files') |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 63 | parser.add_option('-v', '--verbose', |
| 64 | dest='verbose', action='store_true', default=False, |
| 65 | help='Run in verbose mode') |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 66 | parser.add_option('-z', '--autotest_dir', dest='autotest_dir', |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 67 | default=os.path.join(os.path.dirname(__file__), '..'), |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 68 | help='Autotest directory root') |
| 69 | options, args = parser.parse_args() |
| 70 | DRY_RUN = options.dry_run |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 71 | # Make sure autotest_dir is the absolute path |
| 72 | options.autotest_dir = os.path.abspath(options.autotest_dir) |
| 73 | |
| 74 | if len(args) > 0: |
| 75 | print "Invalid option(s) provided: ", args |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 76 | parser.print_help() |
| 77 | return 1 |
| 78 | |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 79 | if len(argv) == 1: |
| 80 | update_all(options.autotest_dir, options.add_noncompliant, |
| 81 | options.add_experimental, options.verbose) |
| 82 | db_clean_broken(options.autotest_dir, options.verbose) |
| 83 | return 0 |
| 84 | |
| 85 | if options.add_all: |
| 86 | update_all(options.autotest_dir, options.add_noncompliant, |
| 87 | options.add_experimental, options.verbose) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 88 | if options.clear_tests: |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 89 | db_clean_broken(options.autotest_dir, options.verbose) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 90 | if options.tests_dir: |
showard | 66a5ef8 | 2008-09-30 16:56:02 +0000 | [diff] [blame] | 91 | if ".." in options.tests_dir: |
| 92 | path = os.path.join(os.getcwd(), options.tests_dir) |
showard | 66a5ef8 | 2008-09-30 16:56:02 +0000 | [diff] [blame] | 93 | options.tests_dir = os.path.abspath(path) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 94 | tests = get_tests_from_fs(options.tests_dir, options.control_pattern, |
| 95 | add_noncompliant=options.add_noncompliant) |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 96 | update_tests_in_db(tests, add_experimental=options.add_experimental, |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 97 | add_noncompliant=options.add_noncompliant, |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 98 | autotest_dir=options.autotest_dir, |
| 99 | verbose=options.verbose) |
| 100 | if options.profile_dir: |
| 101 | profilers = get_tests_from_fs(options.profile_dir, '.*py$') |
| 102 | update_profilers_in_db(profilers, verbose=options.verbose, |
| 103 | add_noncompliant=options.add_noncompliant, |
| 104 | description='NA') |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 105 | |
| 106 | |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 107 | def update_all(autotest_dir, add_noncompliant, add_experimental, verbose): |
| 108 | """Function to scan through all tests and add them to the database.""" |
| 109 | for path in [ 'server/tests', 'server/site_tests', 'client/tests', |
| 110 | 'client/site_tests']: |
| 111 | test_path = os.path.join(autotest_dir, path) |
| 112 | if not os.path.exists(test_path): |
| 113 | continue |
| 114 | if verbose: |
| 115 | print "Scanning " + test_path |
| 116 | tests = [] |
| 117 | tests = get_tests_from_fs(test_path, "^control.*", |
| 118 | add_noncompliant=add_noncompliant) |
| 119 | update_tests_in_db(tests, add_experimental=add_experimental, |
| 120 | add_noncompliant=add_noncompliant, |
| 121 | autotest_dir=autotest_dir, |
| 122 | verbose=verbose) |
| 123 | test_suite_path = os.path.join(autotest_dir, 'test_suites') |
| 124 | if os.path.exists(test_suite_path): |
| 125 | if verbose: |
| 126 | print "Scanning " + test_suite_path |
| 127 | tests = get_tests_from_fs(test_suite_path, '.*', |
| 128 | add_noncompliant=add_noncompliant) |
| 129 | update_tests_in_db(tests, add_experimental=add_experimental, |
| 130 | add_noncompliant=add_noncompliant, |
| 131 | autotest_dir=autotest_dir, |
| 132 | verbose=verbose) |
| 133 | sample_path = os.path.join(autotest_dir, 'server/samples') |
| 134 | if os.path.exists(sample_path): |
| 135 | if verbose: |
| 136 | print "Scanning " + sample_path |
| 137 | tests = get_tests_from_fs(sample_path, '.*srv$', |
| 138 | add_noncompliant=add_noncompliant) |
| 139 | update_tests_in_db(tests, add_experimental=add_experimental, |
| 140 | add_noncompliant=add_noncompliant, |
| 141 | autotest_dir=autotest_dir, |
| 142 | verbose=verbose) |
| 143 | |
| 144 | profilers_path = os.path.join(autotest_dir, "client/profilers") |
| 145 | if os.path.exists(profilers_path): |
| 146 | if verbose: |
| 147 | print "Scanning " + profilers_path |
| 148 | profilers = get_tests_from_fs(profilers_path, '.*py$') |
| 149 | update_profilers_in_db(profilers, verbose=verbose, |
| 150 | add_noncompliant=add_noncompliant, |
| 151 | description='NA') |
showard | b6f9553 | 2008-09-30 16:56:31 +0000 | [diff] [blame] | 152 | # Clean bad db entries |
| 153 | db_clean_broken(autotest_dir, verbose) |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 154 | |
| 155 | |
| 156 | def db_clean_broken(autotest_dir, verbose): |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 157 | """Remove tests from autotest_web that do not have valid control files |
| 158 | |
| 159 | Arguments: |
| 160 | tests: a list of control file relative paths used as keys for deletion. |
| 161 | """ |
| 162 | connection=db_connect() |
| 163 | cursor = connection.cursor() |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 164 | # Get tests |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 165 | sql = "SELECT id, path FROM autotests"; |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 166 | cursor.execute(sql) |
| 167 | results = cursor.fetchall() |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 168 | for test_id, path in results: |
| 169 | full_path = os.path.join(autotest_dir, path) |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 170 | if not os.path.isfile(full_path): |
| 171 | if verbose: |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 172 | print "Removing " + path |
| 173 | db_execute(cursor, "DELETE FROM autotests WHERE id=%s" % test_id) |
| 174 | db_execute(cursor, "DELETE FROM autotests_dependency_labels WHERE " |
| 175 | "test_id=%s" % test_id) |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 176 | |
| 177 | # Find profilers that are no longer present |
| 178 | profilers = [] |
| 179 | sql = "SELECT name FROM profilers" |
| 180 | cursor.execute(sql) |
| 181 | results = cursor.fetchall() |
| 182 | for path in results: |
| 183 | full_path = os.path.join(autotest_dir, "client/profilers", path[0]) |
| 184 | if not os.path.exists(full_path): |
| 185 | if verbose: |
| 186 | print "Removing " + path[0] |
showard | b6f9553 | 2008-09-30 16:56:31 +0000 | [diff] [blame] | 187 | sql = "DELETE FROM profilers WHERE name='%s'" % path[0] |
| 188 | db_execute(cursor, sql) |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 189 | |
| 190 | |
| 191 | connection.commit() |
| 192 | connection.close() |
| 193 | |
| 194 | |
| 195 | def update_profilers_in_db(profilers, verbose=False, description='NA', |
| 196 | add_noncompliant=False): |
| 197 | """Update profilers in autotest_web database""" |
| 198 | connection=db_connect() |
| 199 | cursor = connection.cursor() |
| 200 | for profiler in profilers: |
| 201 | name = os.path.basename(profiler).rstrip(".py") |
| 202 | if not profilers[profiler]: |
| 203 | if add_noncompliant: |
| 204 | doc = description |
| 205 | else: |
| 206 | print "Skipping %s, missing docstring" % profiler |
| 207 | else: |
| 208 | doc = profilers[profiler] |
| 209 | # check if test exists |
| 210 | sql = "SELECT name FROM profilers WHERE name='%s'" % name |
| 211 | cursor.execute(sql) |
| 212 | results = cursor.fetchall() |
| 213 | if results: |
| 214 | sql = "UPDATE profilers SET name='%s', description='%s' "\ |
| 215 | "WHERE name='%s'" |
| 216 | sql %= (MySQLdb.escape_string(name), MySQLdb.escape_string(doc), |
| 217 | MySQLdb.escape_string(name)) |
| 218 | else: |
| 219 | # Insert newly into DB |
| 220 | sql = "INSERT into profilers (name, description) VALUES('%s', '%s')" |
| 221 | sql %= (MySQLdb.escape_string(name), MySQLdb.escape_string(doc)) |
| 222 | |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 223 | db_execute(cursor, sql) |
| 224 | |
| 225 | connection.commit() |
| 226 | connection.close() |
| 227 | |
| 228 | |
| 229 | def update_tests_in_db(tests, dry_run=False, add_experimental=False, |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 230 | add_noncompliant=False, verbose=False, |
| 231 | autotest_dir=None): |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 232 | """Update or add each test to the database""" |
| 233 | connection=db_connect() |
| 234 | cursor = connection.cursor() |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 235 | new_test_dicts = [] |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 236 | for test in tests: |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 237 | new_test = {} |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 238 | new_test['path'] = test.replace(autotest_dir, '').lstrip('/') |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 239 | if verbose: |
| 240 | print "Processing " + new_test['path'] |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 241 | # Create a name for the test |
| 242 | for key in dir(tests[test]): |
| 243 | if not key.startswith('__'): |
| 244 | value = getattr(tests[test], key) |
| 245 | if not callable(value): |
| 246 | new_test[key] = value |
| 247 | # This only takes place if --add-noncompliant is provided on the CLI |
| 248 | if 'name' not in new_test: |
| 249 | test_new_test = test.split('/') |
| 250 | if test_new_test[-1] == 'control': |
| 251 | new_test['name'] = test_new_test[-2] |
| 252 | else: |
| 253 | control_name = "%s:%s" |
| 254 | control_name %= (test_new_test[-2], |
| 255 | test_new_test[-1]) |
| 256 | new_test['name'] = control_name.replace('control.', '') |
| 257 | # Experimental Check |
| 258 | if not add_experimental: |
| 259 | if int(new_test['experimental']): |
| 260 | continue |
| 261 | # clean tests for insertion into db |
| 262 | new_test = dict_db_clean(new_test) |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 263 | new_test_dicts.append(new_test) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 264 | sql = "SELECT name,path FROM autotests WHERE path='%s' LIMIT 1" |
| 265 | sql %= new_test['path'] |
| 266 | cursor.execute(sql) |
| 267 | # check for entries already in existence |
| 268 | results = cursor.fetchall() |
| 269 | if results: |
jadmanski | da49cdf | 2009-03-12 14:42:31 +0000 | [diff] [blame] | 270 | sql = ("UPDATE autotests SET name='%s', test_class='%s'," |
| 271 | "description='%s', test_type=%d, path='%s'," |
| 272 | "author='%s', dependencies='%s'," |
| 273 | "experimental=%d, run_verify=%d, test_time=%d," |
| 274 | "test_category='%s', sync_count=%d" |
| 275 | " WHERE path='%s'") |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 276 | sql %= (new_test['name'], new_test['test_class'], new_test['doc'], |
| 277 | int(new_test['test_type']), new_test['path'], |
jadmanski | da49cdf | 2009-03-12 14:42:31 +0000 | [diff] [blame] | 278 | new_test['author'], |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 279 | new_test['dependencies'], int(new_test['experimental']), |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 280 | int(new_test['run_verify']), new_test['time'], |
| 281 | new_test['test_category'], new_test['sync_count'], |
| 282 | new_test['path']) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 283 | else: |
| 284 | # Create a relative path |
| 285 | path = test.replace(autotest_dir, '') |
jadmanski | da49cdf | 2009-03-12 14:42:31 +0000 | [diff] [blame] | 286 | sql = ("INSERT INTO autotests" |
| 287 | "(name, test_class, description, test_type, path, " |
| 288 | "author, dependencies, experimental, " |
| 289 | "run_verify, test_time, test_category, sync_count) " |
| 290 | "VALUES('%s','%s','%s',%d,'%s','%s','%s',%d,%d,%d," |
| 291 | "'%s',%d)") |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 292 | sql %= (new_test['name'], new_test['test_class'], new_test['doc'], |
| 293 | int(new_test['test_type']), new_test['path'], |
jadmanski | da49cdf | 2009-03-12 14:42:31 +0000 | [diff] [blame] | 294 | new_test['author'], new_test['dependencies'], |
| 295 | int(new_test['experimental']), int(new_test['run_verify']), |
| 296 | new_test['time'], new_test['test_category'], |
| 297 | new_test['sync_count']) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 298 | |
| 299 | db_execute(cursor, sql) |
| 300 | |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 301 | add_label_dependencies(new_test_dicts, cursor) |
| 302 | |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 303 | connection.commit() |
| 304 | connection.close() |
| 305 | |
| 306 | |
| 307 | def dict_db_clean(test): |
| 308 | """Take a tests dictionary from update_db and make it pretty for SQL""" |
| 309 | |
| 310 | test_type = { 'client' : 1, |
| 311 | 'server' : 2, } |
| 312 | test_time = { 'short' : 1, |
| 313 | 'medium' : 2, |
| 314 | 'long' : 3, } |
| 315 | |
| 316 | test['name'] = MySQLdb.escape_string(test['name']) |
| 317 | test['author'] = MySQLdb.escape_string(test['author']) |
| 318 | test['test_class'] = MySQLdb.escape_string(test['test_class']) |
| 319 | test['test_category'] = MySQLdb.escape_string(test['test_category']) |
| 320 | test['doc'] = MySQLdb.escape_string(test['doc']) |
| 321 | test['dependencies'] = ", ".join(test['dependencies']) |
| 322 | # TODO Fix when we move from synch_type to sync_count |
| 323 | if test['sync_count'] == 1: |
| 324 | test['synch_type'] = 1 |
| 325 | else: |
| 326 | test['synch_type'] = 2 |
| 327 | try: |
| 328 | test['test_type'] = int(test['test_type']) |
| 329 | if test['test_type'] != 1 and test['test_type'] != 2: |
| 330 | raise Exception('Incorrect number %d for test_type' % |
| 331 | test['test_type']) |
| 332 | except ValueError: |
| 333 | pass |
| 334 | try: |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 335 | test['time'] = int(test['time']) |
| 336 | if test['time'] < 1 or test['time'] > 3: |
| 337 | raise Exception('Incorrect number %d for time' % |
| 338 | test['time']) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 339 | except ValueError: |
| 340 | pass |
| 341 | |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 342 | if str == type(test['time']): |
| 343 | test['time'] = test_time[test['time'].lower()] |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 344 | if str == type(test['test_type']): |
| 345 | test['test_type'] = test_type[test['test_type'].lower()] |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 346 | |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 347 | return test |
| 348 | |
| 349 | |
showard | 989f25d | 2008-10-01 11:38:11 +0000 | [diff] [blame] | 350 | def add_label_dependencies(tests, cursor): |
| 351 | """ |
| 352 | Look at the DEPENDENCIES field for each test and add the proper many-to-many |
| 353 | relationships. |
| 354 | """ |
| 355 | label_name_to_id = get_id_map(cursor, 'labels', 'name') |
| 356 | test_path_to_id = get_id_map(cursor, 'autotests', 'path') |
| 357 | |
| 358 | # clear out old relationships |
| 359 | test_ids = ','.join(str(test_path_to_id[test['path']]) |
| 360 | for test in tests) |
| 361 | db_execute(cursor, |
| 362 | 'DELETE FROM autotests_dependency_labels WHERE test_id IN (%s)' % |
| 363 | test_ids) |
| 364 | |
| 365 | value_pairs = [] |
| 366 | for test in tests: |
| 367 | test_id = test_path_to_id[test['path']] |
| 368 | for label_name in test['dependencies'].split(','): |
| 369 | label_name = label_name.strip().lower() |
| 370 | if not label_name: |
| 371 | continue |
| 372 | if label_name not in label_name_to_id: |
| 373 | log_dependency_not_found(label_name) |
| 374 | continue |
| 375 | label_id = label_name_to_id[label_name] |
| 376 | value_pairs.append('(%s, %s)' % (test_id, label_id)) |
| 377 | |
| 378 | if not value_pairs: |
| 379 | return |
| 380 | |
| 381 | query = ('INSERT INTO autotests_dependency_labels (test_id, label_id) ' |
| 382 | 'VALUES ' + ','.join(value_pairs)) |
| 383 | db_execute(cursor, query) |
| 384 | |
| 385 | |
| 386 | def log_dependency_not_found(label_name): |
| 387 | if label_name in DEPENDENCIES_NOT_FOUND: |
| 388 | return |
| 389 | print 'Dependency %s not found' % label_name |
| 390 | DEPENDENCIES_NOT_FOUND.add(label_name) |
| 391 | |
| 392 | |
| 393 | def get_id_map(cursor, table_name, name_field): |
| 394 | cursor.execute('SELECT id, %s FROM %s' % (name_field, table_name)) |
| 395 | name_to_id = {} |
| 396 | for item_id, item_name in cursor.fetchall(): |
| 397 | name_to_id[item_name] = item_id |
| 398 | return name_to_id |
| 399 | |
| 400 | |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 401 | def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False): |
| 402 | """Find control jobs in location and create one big job |
| 403 | Returns: |
| 404 | dictionary of the form: |
| 405 | tests[file_path] = parsed_object |
| 406 | |
| 407 | """ |
| 408 | tests = {} |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 409 | profilers = False |
| 410 | if 'client/profilers' in parent_dir: |
| 411 | profilers = True |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 412 | for dir in [ parent_dir ]: |
| 413 | files = recursive_walk(dir, control_pattern) |
| 414 | for file in files: |
mbligh | e8474bb | 2009-05-13 21:37:23 +0000 | [diff] [blame] | 415 | if '__init__.py' in file or '.svn' in file: |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 416 | continue |
| 417 | if not profilers: |
| 418 | if not add_noncompliant: |
| 419 | try: |
| 420 | found_test = control_data.parse_control(file, |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 421 | raise_warnings=True) |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 422 | tests[file] = found_test |
| 423 | except control_data.ControlVariableException, e: |
| 424 | print "Skipping %s\n%s" % (file, e) |
| 425 | pass |
| 426 | else: |
| 427 | found_test = control_data.parse_control(file) |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 428 | tests[file] = found_test |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 429 | else: |
mbligh | 322ec1a | 2008-09-26 16:48:10 +0000 | [diff] [blame] | 430 | script = file.rstrip(".py") |
| 431 | tests[file] = compiler.parseFile(file).doc |
showard | 909c7a6 | 2008-07-15 21:52:38 +0000 | [diff] [blame] | 432 | return tests |
| 433 | |
| 434 | |
| 435 | def recursive_walk(path, wildcard): |
| 436 | """Recurisvely go through a directory. |
| 437 | Returns: |
| 438 | A list of files that match wildcard |
| 439 | """ |
| 440 | files = [] |
| 441 | directories = [ path ] |
| 442 | while len(directories)>0: |
| 443 | directory = directories.pop() |
| 444 | for name in os.listdir(directory): |
| 445 | fullpath = os.path.join(directory, name) |
| 446 | if os.path.isfile(fullpath): |
| 447 | # if we are a control file |
| 448 | if re.search(wildcard, name): |
| 449 | files.append(fullpath) |
| 450 | elif os.path.isdir(fullpath): |
| 451 | directories.append(fullpath) |
| 452 | return files |
| 453 | |
| 454 | |
| 455 | def db_connect(): |
| 456 | """Connect to the AUTOTEST_WEB database and return a connect object.""" |
| 457 | c = global_config.global_config |
| 458 | db_host = c.get_config_value('AUTOTEST_WEB', 'host') |
| 459 | db_name = c.get_config_value('AUTOTEST_WEB', 'database') |
| 460 | username = c.get_config_value('AUTOTEST_WEB', 'user') |
| 461 | password = c.get_config_value('AUTOTEST_WEB', 'password') |
| 462 | connection = MySQLdb.connect(host=db_host, db=db_name, |
| 463 | user=username, |
| 464 | passwd=password) |
| 465 | return connection |
| 466 | |
| 467 | |
| 468 | def db_execute(cursor, sql): |
| 469 | """Execute SQL or print out what would be executed if dry_run is defined""" |
| 470 | |
| 471 | if DRY_RUN: |
| 472 | print "Would run: " + sql |
| 473 | else: |
| 474 | cursor.execute(sql) |
| 475 | |
| 476 | |
| 477 | if __name__ == "__main__": |
| 478 | main(sys.argv) |