blob: 00b912ccb9ddfea9375ea1985ade0e3ae5ecb62c [file] [log] [blame]
mbligh7c8ea992009-06-22 19:03:08 +00001#!/usr/bin/python
showard909c7a62008-07-15 21:52:38 +00002#
3# Copyright 2008 Google Inc. All Rights Reserved.
4"""
5This utility allows for easy updating, removing and importing
showardeab66ce2009-12-23 00:03:56 +00006of tests into the autotest_web afe_autotests table.
showard909c7a62008-07-15 21:52:38 +00007
8Example of updating client side tests:
mbligh4b5c31e2009-07-11 00:55:34 +00009./test_importer.py -t /usr/local/autotest/client/tests
showard909c7a62008-07-15 21:52:38 +000010
mbligh4b5c31e2009-07-11 00:55:34 +000011If, for example, not all of your control files adhere to the standard outlined
12at http://autotest.kernel.org/wiki/ControlRequirements, you can force options:
showard909c7a62008-07-15 21:52:38 +000013
mbligh4b5c31e2009-07-11 00:55:34 +000014./test_importer.py --test-type server -t /usr/local/autotest/server/tests
showard909c7a62008-07-15 21:52:38 +000015
mbligh4b5c31e2009-07-11 00:55:34 +000016You would need to pass --add-noncompliant to include such control files,
17however. An easy way to check for compliance is to run in dry mode:
showard909c7a62008-07-15 21:52:38 +000018
mbligh4b5c31e2009-07-11 00:55:34 +000019./test_importer.py --dry-run -t /usr/local/autotest/server/tests/mytest
20
21Or to check a single control file, you can use check_control_file_vars.py.
22
23Running with no options is equivalent to --add-all --db-clear-tests.
24
25Most options should be fairly self explanatory, use --help to display them.
showard909c7a62008-07-15 21:52:38 +000026"""
27
28
showard909c7a62008-07-15 21:52:38 +000029import common
showardcd5131c2010-01-12 18:54:33 +000030import logging, re, os, sys, optparse, compiler
31from autotest_lib.frontend import setup_django_environment
32from autotest_lib.frontend.afe import models
mbligh61be4cd2010-03-30 02:02:40 +000033from autotest_lib.client.common_lib import control_data, utils
lmrc781f552010-05-20 00:36:37 +000034from autotest_lib.client.common_lib import logging_config, logging_manager
showard909c7a62008-07-15 21:52:38 +000035
showard909c7a62008-07-15 21:52:38 +000036
lmrc781f552010-05-20 00:36:37 +000037class TestImporterLoggingConfig(logging_config.LoggingConfig):
38 def configure_logging(self, results_dir=None, verbose=False):
39 super(TestImporterLoggingConfig, self).configure_logging(
40 use_console=True,
41 verbose=verbose)
42
43
showard909c7a62008-07-15 21:52:38 +000044# Global
45DRY_RUN = False
showard989f25d2008-10-01 11:38:11 +000046DEPENDENCIES_NOT_FOUND = set()
showard909c7a62008-07-15 21:52:38 +000047
showardcd5131c2010-01-12 18:54:33 +000048
mbligh61be4cd2010-03-30 02:02:40 +000049def update_all(autotest_dir, add_noncompliant, add_experimental):
50 """
51 Function to scan through all tests and add them to the database.
52
53 This function invoked when no parameters supplied to the command line.
54 It 'synchronizes' the test database with the current contents of the
55 client and server test directories. When test code is discovered
56 in the file system new tests may be added to the db. Likewise,
57 if test code is not found in the filesystem, tests may be removed
58 from the db. The base test directories are hard-coded to client/tests,
59 client/site_tests, server/tests and server/site_tests.
60
61 @param autotest_dir: prepended to path strings (/usr/local/autotest).
62 @param add_noncompliant: attempt adding test with invalid control files.
63 @param add_experimental: add tests with experimental attribute set.
64 """
showardcd5131c2010-01-12 18:54:33 +000065 for path in [ 'server/tests', 'server/site_tests', 'client/tests',
66 'client/site_tests']:
67 test_path = os.path.join(autotest_dir, path)
68 if not os.path.exists(test_path):
69 continue
mbligh61be4cd2010-03-30 02:02:40 +000070 logging.info("Scanning %s", test_path)
showardcd5131c2010-01-12 18:54:33 +000071 tests = []
72 tests = get_tests_from_fs(test_path, "^control.*",
73 add_noncompliant=add_noncompliant)
74 update_tests_in_db(tests, add_experimental=add_experimental,
75 add_noncompliant=add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +000076 autotest_dir=autotest_dir)
showardcd5131c2010-01-12 18:54:33 +000077 test_suite_path = os.path.join(autotest_dir, 'test_suites')
78 if os.path.exists(test_suite_path):
mbligh61be4cd2010-03-30 02:02:40 +000079 logging.info("Scanning %s", test_suite_path)
showardcd5131c2010-01-12 18:54:33 +000080 tests = get_tests_from_fs(test_suite_path, '.*',
81 add_noncompliant=add_noncompliant)
82 update_tests_in_db(tests, add_experimental=add_experimental,
83 add_noncompliant=add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +000084 autotest_dir=autotest_dir)
showardcd5131c2010-01-12 18:54:33 +000085
86 profilers_path = os.path.join(autotest_dir, "client/profilers")
87 if os.path.exists(profilers_path):
mbligh61be4cd2010-03-30 02:02:40 +000088 logging.info("Scanning %s", profilers_path)
showardcd5131c2010-01-12 18:54:33 +000089 profilers = get_tests_from_fs(profilers_path, '.*py$')
mbligh61be4cd2010-03-30 02:02:40 +000090 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant,
showardcd5131c2010-01-12 18:54:33 +000091 description='NA')
92 # Clean bad db entries
mbligh61be4cd2010-03-30 02:02:40 +000093 db_clean_broken(autotest_dir)
showardcd5131c2010-01-12 18:54:33 +000094
95
mbligh61be4cd2010-03-30 02:02:40 +000096def update_samples(autotest_dir, add_noncompliant, add_experimental):
97 """
98 Add only sample tests to the database from the filesystem.
99
100 This function invoked when -S supplied on command line.
101 Only adds tests to the database - does not delete any.
102 Samples tests are formatted slightly differently than other tests.
103
104 @param autotest_dir: prepended to path strings (/usr/local/autotest).
105 @param add_noncompliant: attempt adding test with invalid control files.
106 @param add_experimental: add tests with experimental attribute set.
107 """
showardcd5131c2010-01-12 18:54:33 +0000108 sample_path = os.path.join(autotest_dir, 'server/samples')
109 if os.path.exists(sample_path):
mbligh61be4cd2010-03-30 02:02:40 +0000110 logging.info("Scanning %s", sample_path)
showardcd5131c2010-01-12 18:54:33 +0000111 tests = get_tests_from_fs(sample_path, '.*srv$',
112 add_noncompliant=add_noncompliant)
113 update_tests_in_db(tests, add_experimental=add_experimental,
114 add_noncompliant=add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000115 autotest_dir=autotest_dir)
showardcd5131c2010-01-12 18:54:33 +0000116
117
mbligh61be4cd2010-03-30 02:02:40 +0000118def db_clean_broken(autotest_dir):
119 """
120 Remove tests from autotest_web that do not have valid control files
showardcd5131c2010-01-12 18:54:33 +0000121
mbligh61be4cd2010-03-30 02:02:40 +0000122 This function invoked when -c supplied on the command line and when
123 running update_all(). Removes tests from database which are not
124 found in the filesystem. Also removes profilers which are just
125 a special case of tests.
126
127 @param autotest_dir: prepended to path strings (/usr/local/autotest).
showardcd5131c2010-01-12 18:54:33 +0000128 """
129 for test in models.Test.objects.all():
130 full_path = os.path.join(autotest_dir, test.path)
131 if not os.path.isfile(full_path):
mbligh61be4cd2010-03-30 02:02:40 +0000132 logging.info("Removing %s", test.path)
showardcd5131c2010-01-12 18:54:33 +0000133 _log_or_execute(repr(test), test.delete)
134
135 # Find profilers that are no longer present
136 for profiler in models.Profiler.objects.all():
137 full_path = os.path.join(autotest_dir, "client", "profilers",
138 profiler.name)
139 if not os.path.exists(full_path):
mbligh61be4cd2010-03-30 02:02:40 +0000140 logging.info("Removing %s", profiler.name)
showardcd5131c2010-01-12 18:54:33 +0000141 _log_or_execute(repr(profiler), profiler.delete)
142
143
mbligh61be4cd2010-03-30 02:02:40 +0000144def db_clean_all(autotest_dir):
145 """
146 Remove all tests from autotest_web - very destructive
147
148 This function invoked when -C supplied on the command line.
149 Removes ALL tests from the database.
150
151 @param autotest_dir: prepended to path strings (/usr/local/autotest).
152 """
153 for test in models.Test.objects.all():
154 full_path = os.path.join(autotest_dir, test.path)
155 logging.info("Removing %s", test.path)
156 _log_or_execute(repr(test), test.delete)
157
158 # Find profilers that are no longer present
159 for profiler in models.Profiler.objects.all():
160 full_path = os.path.join(autotest_dir, "client", "profilers",
161 profiler.name)
162 logging.info("Removing %s", profiler.name)
163 _log_or_execute(repr(profiler), profiler.delete)
164
165
166def update_profilers_in_db(profilers, description='NA',
showardcd5131c2010-01-12 18:54:33 +0000167 add_noncompliant=False):
mbligh61be4cd2010-03-30 02:02:40 +0000168 """
169 Add only profilers to the database from the filesystem.
170
171 This function invoked when -p supplied on command line.
172 Only adds profilers to the database - does not delete any.
173 Profilers are formatted slightly differently than tests.
174
175 @param profilers: list of profilers found in the file system.
176 @param description: simple text to satisfy docstring.
177 @param add_noncompliant: attempt adding test with invalid control files.
178 """
showardcd5131c2010-01-12 18:54:33 +0000179 for profiler in profilers:
mblighed0526a2010-04-22 18:26:17 +0000180 name = os.path.basename(profiler)
181 if name.endswith('.py'):
182 name = name[:-3]
showardcd5131c2010-01-12 18:54:33 +0000183 if not profilers[profiler]:
184 if add_noncompliant:
185 doc = description
186 else:
jamesrenfed94092010-05-03 20:52:47 +0000187 logging.warn("Skipping %s, missing docstring", profiler)
mbligh781d2692010-04-21 01:04:28 +0000188 continue
showardcd5131c2010-01-12 18:54:33 +0000189 else:
190 doc = profilers[profiler]
191
192 model = models.Profiler.objects.get_or_create(name=name)[0]
193 model.description = doc
194 _log_or_execute(repr(model), model.save)
195
196
197def update_tests_in_db(tests, dry_run=False, add_experimental=False,
mbligh61be4cd2010-03-30 02:02:40 +0000198 add_noncompliant=False, autotest_dir=None):
199 """
200 Scans through all tests and add them to the database.
201
202 This function invoked when -t supplied and for update_all.
203 When test code is discovered in the file system new tests may be added
204
205 @param tests: list of tests found in the filesystem.
206 @param dry_run: not used at this time.
207 @param add_experimental: add tests with experimental attribute set.
208 @param add_noncompliant: attempt adding test with invalid control files.
209 @param autotest_dir: prepended to path strings (/usr/local/autotest).
210 """
211 site_set_attributes_module = utils.import_site_module(
212 __file__, 'autotest_lib.utils.site_test_importer_attributes')
213
showardcd5131c2010-01-12 18:54:33 +0000214 for test in tests:
215 new_test = models.Test.objects.get_or_create(
216 path=test.replace(autotest_dir, '').lstrip('/'))[0]
mbligh61be4cd2010-03-30 02:02:40 +0000217 logging.info("Processing %s", new_test.path)
showardcd5131c2010-01-12 18:54:33 +0000218
219 # Set the test's attributes
220 data = tests[test]
221 _set_attributes_clean(new_test, data)
222
mbligh61be4cd2010-03-30 02:02:40 +0000223 # Custom Attribute Update
224 if site_set_attributes_module:
225 site_set_attributes_module._set_attributes_custom(new_test, data)
226
showardcd5131c2010-01-12 18:54:33 +0000227 # This only takes place if --add-noncompliant is provided on the CLI
228 if not new_test.name:
229 test_new_test = test.split('/')
230 if test_new_test[-1] == 'control':
231 new_test.name = test_new_test[-2]
232 else:
233 control_name = "%s:%s"
234 control_name %= (test_new_test[-2],
235 test_new_test[-1])
236 new_test.name = control_name.replace('control.', '')
237
238 # Experimental Check
239 if not add_experimental and new_test.experimental:
240 continue
241
242 _log_or_execute(repr(new_test), new_test.save)
243 add_label_dependencies(new_test)
244
245
246def _set_attributes_clean(test, data):
mbligh61be4cd2010-03-30 02:02:40 +0000247 """
248 First pass sets the attributes of the Test object from file system.
showardcd5131c2010-01-12 18:54:33 +0000249
mbligh61be4cd2010-03-30 02:02:40 +0000250 @param test: a test object to be populated for the database.
251 @param data: object with test data from the file system.
252 """
showardcd5131c2010-01-12 18:54:33 +0000253 test_type = { 'client' : 1,
254 'server' : 2, }
255 test_time = { 'short' : 1,
256 'medium' : 2,
257 'long' : 3, }
258
259
260 string_attributes = ('name', 'author', 'test_class', 'test_category',
261 'test_category', 'sync_count')
262 for attribute in string_attributes:
263 setattr(test, attribute, getattr(data, attribute))
264
265 test.description = data.doc
266 test.dependencies = ", ".join(data.dependencies)
267
268 int_attributes = ('experimental', 'run_verify')
269 for attribute in int_attributes:
270 setattr(test, attribute, int(getattr(data, attribute)))
271
272 try:
273 test.test_type = int(data.test_type)
274 if test.test_type != 1 and test.test_type != 2:
275 raise Exception('Incorrect number %d for test_type' %
276 test.test_type)
277 except ValueError:
278 pass
279 try:
280 test.test_time = int(data.time)
281 if test.test_time < 1 or test.time > 3:
282 raise Exception('Incorrect number %d for time' % test.time)
283 except ValueError:
284 pass
285
286 if not test.test_time and str == type(data.time):
287 test.test_time = test_time[data.time.lower()]
288 if not test.test_type and str == type(data.test_type):
289 test.test_type = test_type[data.test_type.lower()]
290
291
292def add_label_dependencies(test):
293 """
mbligh61be4cd2010-03-30 02:02:40 +0000294 Add proper many-to-many relationships from DEPENDENCIES field.
295
296 @param test: test object for the database.
showardcd5131c2010-01-12 18:54:33 +0000297 """
mbligh61be4cd2010-03-30 02:02:40 +0000298
showardcd5131c2010-01-12 18:54:33 +0000299 # clear out old relationships
300 _log_or_execute(repr(test), test.dependency_labels.clear,
301 subject='clear dependencies from')
302
303 for label_name in test.dependencies.split(','):
304 label_name = label_name.strip().lower()
305 if not label_name:
306 continue
307
308 try:
309 label = models.Label.objects.get(name=label_name)
310 except models.Label.DoesNotExist:
311 log_dependency_not_found(label_name)
312 continue
313
314 _log_or_execute(repr(label), test.dependency_labels.add, label,
315 subject='add dependency to %s' % test.name)
316
317
318def log_dependency_not_found(label_name):
mbligh61be4cd2010-03-30 02:02:40 +0000319 """
320 Exception processing when label not found in database.
321
322 @param label_name: from test dependencies.
323 """
showardcd5131c2010-01-12 18:54:33 +0000324 if label_name in DEPENDENCIES_NOT_FOUND:
325 return
mbligh61be4cd2010-03-30 02:02:40 +0000326 logging.info("Dependency %s not found", label_name)
showardcd5131c2010-01-12 18:54:33 +0000327 DEPENDENCIES_NOT_FOUND.add(label_name)
328
329
330def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False):
mbligh61be4cd2010-03-30 02:02:40 +0000331 """
332 Find control files in file system and load a list with their info.
showardcd5131c2010-01-12 18:54:33 +0000333
mbligh61be4cd2010-03-30 02:02:40 +0000334 @param parent_dir: directory to search recursively.
335 @param control_pattern: name format of control file.
336 @param add_noncompliant: ignore control file parse errors.
337
mblighed0526a2010-04-22 18:26:17 +0000338 @return dictionary of the form: tests[file_path] = parsed_object
showardcd5131c2010-01-12 18:54:33 +0000339 """
340 tests = {}
341 profilers = False
342 if 'client/profilers' in parent_dir:
343 profilers = True
344 for dir in [ parent_dir ]:
345 files = recursive_walk(dir, control_pattern)
346 for file in files:
347 if '__init__.py' in file or '.svn' in file:
348 continue
349 if not profilers:
350 if not add_noncompliant:
351 try:
352 found_test = control_data.parse_control(file,
353 raise_warnings=True)
354 tests[file] = found_test
355 except control_data.ControlVariableException, e:
jamesrenfed94092010-05-03 20:52:47 +0000356 logging.warn("Skipping %s\n%s", file, e)
jamesren6e3e9bd2010-04-28 18:07:05 +0000357 except Exception, e:
358 logging.error("Bad %s\n%s", file, e)
showardcd5131c2010-01-12 18:54:33 +0000359 else:
360 found_test = control_data.parse_control(file)
361 tests[file] = found_test
362 else:
showardcd5131c2010-01-12 18:54:33 +0000363 tests[file] = compiler.parseFile(file).doc
364 return tests
365
366
367def recursive_walk(path, wildcard):
mbligh61be4cd2010-03-30 02:02:40 +0000368 """
369 Recursively go through a directory.
370
371 This function invoked by get_tests_from_fs().
372
373 @param path: base directory to start search.
374 @param wildcard: name format to match.
375
mblighed0526a2010-04-22 18:26:17 +0000376 @return A list of files that match wildcard
showardcd5131c2010-01-12 18:54:33 +0000377 """
378 files = []
379 directories = [ path ]
380 while len(directories)>0:
381 directory = directories.pop()
382 for name in os.listdir(directory):
383 fullpath = os.path.join(directory, name)
384 if os.path.isfile(fullpath):
385 # if we are a control file
386 if re.search(wildcard, name):
387 files.append(fullpath)
388 elif os.path.isdir(fullpath):
389 directories.append(fullpath)
390 return files
391
392
393def _log_or_execute(content, func, *args, **kwargs):
mbligh61be4cd2010-03-30 02:02:40 +0000394 """
395 Log a message if dry_run is enabled, or execute the given function.
showardcd5131c2010-01-12 18:54:33 +0000396
mbligh61be4cd2010-03-30 02:02:40 +0000397 Relies on the DRY_RUN global variable.
398
399 @param content: the actual log message.
400 @param func: function to execute if dry_run is not enabled.
401 @param subject: (Optional) The type of log being written. Defaults to
402 the name of the provided function.
showardcd5131c2010-01-12 18:54:33 +0000403 """
404 subject = kwargs.get('subject', func.__name__)
405
406 if DRY_RUN:
mbligh61be4cd2010-03-30 02:02:40 +0000407 logging.info("Would %s: %s", subject, content)
showardcd5131c2010-01-12 18:54:33 +0000408 else:
409 func(*args)
410
411
mbligh61be4cd2010-03-30 02:02:40 +0000412def _create_whitelist_set(whitelist_path):
413 """
414 Create a set with contents from a whitelist file for membership testing.
415
416 @param whitelist_path: full path to the whitelist file.
417
mblighed0526a2010-04-22 18:26:17 +0000418 @return set with files listed one/line - newlines included.
mbligh61be4cd2010-03-30 02:02:40 +0000419 """
420 f = open(whitelist_path, 'r')
421 whitelist_set = set([line.strip() for line in f])
422 f.close()
423 return whitelist_set
424
425
426def update_from_whitelist(whitelist_set, add_experimental, add_noncompliant,
427 autotest_dir):
428 """
429 Scans through all tests in the whitelist and add them to the database.
430
431 This function invoked when -w supplied.
432
433 @param whitelist_set: set of tests in full-path form from a whitelist.
434 @param add_experimental: add tests with experimental attribute set.
435 @param add_noncompliant: attempt adding test with invalid control files.
436 @param autotest_dir: prepended to path strings (/usr/local/autotest).
437 """
438 tests = {}
439 profilers = {}
440 for file_path in whitelist_set:
441 if file_path.find('client/profilers') == -1:
442 try:
443 found_test = control_data.parse_control(file_path,
444 raise_warnings=True)
445 tests[file_path] = found_test
446 except control_data.ControlVariableException, e:
jamesrenfed94092010-05-03 20:52:47 +0000447 logging.warn("Skipping %s\n%s", file, e)
mbligh61be4cd2010-03-30 02:02:40 +0000448 else:
449 profilers[file_path] = compiler.parseFile(file_path).doc
450
451 if len(tests) > 0:
452 update_tests_in_db(tests, add_experimental=add_experimental,
453 add_noncompliant=add_noncompliant,
454 autotest_dir=autotest_dir)
455 if len(profilers) > 0:
456 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant,
457 description='NA')
458
459
showard909c7a62008-07-15 21:52:38 +0000460def main(argv):
461 """Main function"""
mbligh61be4cd2010-03-30 02:02:40 +0000462
showard909c7a62008-07-15 21:52:38 +0000463 global DRY_RUN
464 parser = optparse.OptionParser()
mbligh61be4cd2010-03-30 02:02:40 +0000465 parser.add_option('-c', '--db-clean-tests',
466 dest='clean_tests', action='store_true',
showard909c7a62008-07-15 21:52:38 +0000467 default=False,
mbligh61be4cd2010-03-30 02:02:40 +0000468 help='Clean client and server tests with invalid control files')
469 parser.add_option('-C', '--db-clear-all-tests',
470 dest='clear_all_tests', action='store_true',
471 default=False,
472 help='Clear ALL client and server tests')
showard909c7a62008-07-15 21:52:38 +0000473 parser.add_option('-d', '--dry-run',
474 dest='dry_run', action='store_true', default=False,
475 help='Dry run for operation')
mbligh322ec1a2008-09-26 16:48:10 +0000476 parser.add_option('-A', '--add-all',
477 dest='add_all', action='store_true',
478 default=False,
mbligh25d656c2009-08-24 22:04:02 +0000479 help='Add site_tests, tests, and test_suites')
480 parser.add_option('-S', '--add-samples',
481 dest='add_samples', action='store_true',
482 default=False,
483 help='Add samples.')
mbligh322ec1a2008-09-26 16:48:10 +0000484 parser.add_option('-E', '--add-experimental',
showard909c7a62008-07-15 21:52:38 +0000485 dest='add_experimental', action='store_true',
mbligh79410e12008-11-20 17:59:15 +0000486 default=True,
showard909c7a62008-07-15 21:52:38 +0000487 help='Add experimental tests to frontend')
488 parser.add_option('-N', '--add-noncompliant',
489 dest='add_noncompliant', action='store_true',
490 default=False,
mbligh4b5c31e2009-07-11 00:55:34 +0000491 help='Add non-compliant tests (i.e. tests that do not '
492 'define all required control variables)')
mbligh322ec1a2008-09-26 16:48:10 +0000493 parser.add_option('-p', '--profile-dir', dest='profile_dir',
494 help='Directory to recursively check for profiles')
showard909c7a62008-07-15 21:52:38 +0000495 parser.add_option('-t', '--tests-dir', dest='tests_dir',
496 help='Directory to recursively check for control.*')
showard909c7a62008-07-15 21:52:38 +0000497 parser.add_option('-r', '--control-pattern', dest='control_pattern',
498 default='^control.*',
499 help='The pattern to look for in directories for control files')
mbligh322ec1a2008-09-26 16:48:10 +0000500 parser.add_option('-v', '--verbose',
501 dest='verbose', action='store_true', default=False,
502 help='Run in verbose mode')
mbligh61be4cd2010-03-30 02:02:40 +0000503 parser.add_option('-w', '--whitelist-file', dest='whitelist_file',
504 help='Filename for list of test names that must match')
505 parser.add_option('-z', '--autotest-dir', dest='autotest_dir',
mbligh322ec1a2008-09-26 16:48:10 +0000506 default=os.path.join(os.path.dirname(__file__), '..'),
showard909c7a62008-07-15 21:52:38 +0000507 help='Autotest directory root')
508 options, args = parser.parse_args()
lmrc781f552010-05-20 00:36:37 +0000509
510 logging_manager.configure_logging(TestImporterLoggingConfig(),
511 verbose=options.verbose)
512
showard909c7a62008-07-15 21:52:38 +0000513 DRY_RUN = options.dry_run
jamesrenfed94092010-05-03 20:52:47 +0000514 if DRY_RUN:
515 logging.getLogger().setLevel(logging.WARN)
516
mbligh322ec1a2008-09-26 16:48:10 +0000517 # Make sure autotest_dir is the absolute path
518 options.autotest_dir = os.path.abspath(options.autotest_dir)
519
520 if len(args) > 0:
mbligh61be4cd2010-03-30 02:02:40 +0000521 logging.error("Invalid option(s) provided: %s", args)
showard909c7a62008-07-15 21:52:38 +0000522 parser.print_help()
523 return 1
524
mbligh61be4cd2010-03-30 02:02:40 +0000525 if options.verbose:
526 logging.getLogger().setLevel(logging.DEBUG)
527
528 if len(argv) == 1 or (len(argv) == 2 and options.verbose):
mbligh322ec1a2008-09-26 16:48:10 +0000529 update_all(options.autotest_dir, options.add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000530 options.add_experimental)
531 db_clean_broken(options.autotest_dir)
mbligh322ec1a2008-09-26 16:48:10 +0000532 return 0
533
mbligh61be4cd2010-03-30 02:02:40 +0000534 if options.clear_all_tests:
535 if (options.clean_tests or options.add_all or options.add_samples or
536 options.add_noncompliant):
537 logging.error(
538 "Can only pass --autotest-dir, --dry-run and --verbose with "
539 "--db-clear-all-tests")
540 return 1
541 db_clean_all(options.autotest_dir)
542
543 whitelist_set = None
544 if options.whitelist_file:
545 if options.add_all:
546 logging.error("Cannot pass both --add-all and --whitelist-file")
547 return 1
548 whitelist_path = os.path.abspath(options.whitelist_file)
549 if not os.path.isfile(whitelist_path):
550 logging.error("--whitelist-file (%s) not found", whitelist_path)
551 return 1
552 logging.info("Using whitelist file %s", whitelist_path)
553 whitelist_set = _create_whitelist_set(whitelist_path)
554 update_from_whitelist(whitelist_set,
555 add_experimental=options.add_experimental,
556 add_noncompliant=options.add_noncompliant,
557 autotest_dir=options.autotest_dir)
mbligh322ec1a2008-09-26 16:48:10 +0000558 if options.add_all:
559 update_all(options.autotest_dir, options.add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000560 options.add_experimental)
mbligh25d656c2009-08-24 22:04:02 +0000561 if options.add_samples:
562 update_samples(options.autotest_dir, options.add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000563 options.add_experimental)
showard909c7a62008-07-15 21:52:38 +0000564 if options.tests_dir:
mbligh4b5c31e2009-07-11 00:55:34 +0000565 options.tests_dir = os.path.abspath(options.tests_dir)
showard909c7a62008-07-15 21:52:38 +0000566 tests = get_tests_from_fs(options.tests_dir, options.control_pattern,
567 add_noncompliant=options.add_noncompliant)
mbligh322ec1a2008-09-26 16:48:10 +0000568 update_tests_in_db(tests, add_experimental=options.add_experimental,
showard909c7a62008-07-15 21:52:38 +0000569 add_noncompliant=options.add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000570 autotest_dir=options.autotest_dir)
mbligh322ec1a2008-09-26 16:48:10 +0000571 if options.profile_dir:
572 profilers = get_tests_from_fs(options.profile_dir, '.*py$')
mbligh61be4cd2010-03-30 02:02:40 +0000573 update_profilers_in_db(profilers,
mbligh322ec1a2008-09-26 16:48:10 +0000574 add_noncompliant=options.add_noncompliant,
575 description='NA')
mbligh61be4cd2010-03-30 02:02:40 +0000576 if options.clean_tests:
577 db_clean_broken(options.autotest_dir)
showard909c7a62008-07-15 21:52:38 +0000578
579
showard909c7a62008-07-15 21:52:38 +0000580if __name__ == "__main__":
581 main(sys.argv)