blob: a7db2305de97ee2a2aa24ef45a0e21f1fae8d567 [file] [log] [blame]
mbligh7c8ea992009-06-22 19:03:08 +00001#!/usr/bin/python
showard909c7a62008-07-15 21:52:38 +00002#
3# Copyright 2008 Google Inc. All Rights Reserved.
4"""
5This utility allows for easy updating, removing and importing
showardeab66ce2009-12-23 00:03:56 +00006of tests into the autotest_web afe_autotests table.
showard909c7a62008-07-15 21:52:38 +00007
8Example of updating client side tests:
mbligh4b5c31e2009-07-11 00:55:34 +00009./test_importer.py -t /usr/local/autotest/client/tests
showard909c7a62008-07-15 21:52:38 +000010
mbligh4b5c31e2009-07-11 00:55:34 +000011If, for example, not all of your control files adhere to the standard outlined
12at http://autotest.kernel.org/wiki/ControlRequirements, you can force options:
showard909c7a62008-07-15 21:52:38 +000013
mbligh4b5c31e2009-07-11 00:55:34 +000014./test_importer.py --test-type server -t /usr/local/autotest/server/tests
showard909c7a62008-07-15 21:52:38 +000015
mbligh4b5c31e2009-07-11 00:55:34 +000016You would need to pass --add-noncompliant to include such control files,
17however. An easy way to check for compliance is to run in dry mode:
showard909c7a62008-07-15 21:52:38 +000018
mbligh4b5c31e2009-07-11 00:55:34 +000019./test_importer.py --dry-run -t /usr/local/autotest/server/tests/mytest
20
mbligh4b5c31e2009-07-11 00:55:34 +000021Running with no options is equivalent to --add-all --db-clear-tests.
22
23Most options should be fairly self explanatory, use --help to display them.
showard909c7a62008-07-15 21:52:38 +000024"""
25
26
showard909c7a62008-07-15 21:52:38 +000027import common
showardcd5131c2010-01-12 18:54:33 +000028import logging, re, os, sys, optparse, compiler
beepse19d3032013-05-30 09:22:07 -070029
showardcd5131c2010-01-12 18:54:33 +000030from autotest_lib.frontend import setup_django_environment
31from autotest_lib.frontend.afe import models
Allen Lifce20642017-02-01 17:22:23 -080032from autotest_lib.client.common_lib import control_data
lmrc781f552010-05-20 00:36:37 +000033from autotest_lib.client.common_lib import logging_config, logging_manager
showard909c7a62008-07-15 21:52:38 +000034
showard909c7a62008-07-15 21:52:38 +000035
lmrc781f552010-05-20 00:36:37 +000036class TestImporterLoggingConfig(logging_config.LoggingConfig):
Aviv Keshet6f455262013-03-01 16:02:29 -080037 #pylint: disable-msg=C0111
lmrc781f552010-05-20 00:36:37 +000038 def configure_logging(self, results_dir=None, verbose=False):
39 super(TestImporterLoggingConfig, self).configure_logging(
40 use_console=True,
41 verbose=verbose)
42
43
showard909c7a62008-07-15 21:52:38 +000044# Global
45DRY_RUN = False
showard989f25d2008-10-01 11:38:11 +000046DEPENDENCIES_NOT_FOUND = set()
showard909c7a62008-07-15 21:52:38 +000047
showardcd5131c2010-01-12 18:54:33 +000048
mbligh61be4cd2010-03-30 02:02:40 +000049def update_all(autotest_dir, add_noncompliant, add_experimental):
50 """
51 Function to scan through all tests and add them to the database.
52
53 This function invoked when no parameters supplied to the command line.
54 It 'synchronizes' the test database with the current contents of the
55 client and server test directories. When test code is discovered
56 in the file system new tests may be added to the db. Likewise,
57 if test code is not found in the filesystem, tests may be removed
58 from the db. The base test directories are hard-coded to client/tests,
59 client/site_tests, server/tests and server/site_tests.
60
61 @param autotest_dir: prepended to path strings (/usr/local/autotest).
62 @param add_noncompliant: attempt adding test with invalid control files.
63 @param add_experimental: add tests with experimental attribute set.
64 """
showardcd5131c2010-01-12 18:54:33 +000065 for path in [ 'server/tests', 'server/site_tests', 'client/tests',
66 'client/site_tests']:
67 test_path = os.path.join(autotest_dir, path)
68 if not os.path.exists(test_path):
69 continue
mbligh61be4cd2010-03-30 02:02:40 +000070 logging.info("Scanning %s", test_path)
showardcd5131c2010-01-12 18:54:33 +000071 tests = []
72 tests = get_tests_from_fs(test_path, "^control.*",
73 add_noncompliant=add_noncompliant)
74 update_tests_in_db(tests, add_experimental=add_experimental,
75 add_noncompliant=add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +000076 autotest_dir=autotest_dir)
showardcd5131c2010-01-12 18:54:33 +000077 test_suite_path = os.path.join(autotest_dir, 'test_suites')
78 if os.path.exists(test_suite_path):
mbligh61be4cd2010-03-30 02:02:40 +000079 logging.info("Scanning %s", test_suite_path)
showardcd5131c2010-01-12 18:54:33 +000080 tests = get_tests_from_fs(test_suite_path, '.*',
81 add_noncompliant=add_noncompliant)
82 update_tests_in_db(tests, add_experimental=add_experimental,
83 add_noncompliant=add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +000084 autotest_dir=autotest_dir)
showardcd5131c2010-01-12 18:54:33 +000085
86 profilers_path = os.path.join(autotest_dir, "client/profilers")
87 if os.path.exists(profilers_path):
mbligh61be4cd2010-03-30 02:02:40 +000088 logging.info("Scanning %s", profilers_path)
showardcd5131c2010-01-12 18:54:33 +000089 profilers = get_tests_from_fs(profilers_path, '.*py$')
mbligh61be4cd2010-03-30 02:02:40 +000090 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant,
showardcd5131c2010-01-12 18:54:33 +000091 description='NA')
92 # Clean bad db entries
mbligh61be4cd2010-03-30 02:02:40 +000093 db_clean_broken(autotest_dir)
showardcd5131c2010-01-12 18:54:33 +000094
95
mbligh61be4cd2010-03-30 02:02:40 +000096def update_samples(autotest_dir, add_noncompliant, add_experimental):
97 """
98 Add only sample tests to the database from the filesystem.
99
100 This function invoked when -S supplied on command line.
101 Only adds tests to the database - does not delete any.
102 Samples tests are formatted slightly differently than other tests.
103
104 @param autotest_dir: prepended to path strings (/usr/local/autotest).
105 @param add_noncompliant: attempt adding test with invalid control files.
106 @param add_experimental: add tests with experimental attribute set.
107 """
showardcd5131c2010-01-12 18:54:33 +0000108 sample_path = os.path.join(autotest_dir, 'server/samples')
109 if os.path.exists(sample_path):
mbligh61be4cd2010-03-30 02:02:40 +0000110 logging.info("Scanning %s", sample_path)
showardcd5131c2010-01-12 18:54:33 +0000111 tests = get_tests_from_fs(sample_path, '.*srv$',
112 add_noncompliant=add_noncompliant)
113 update_tests_in_db(tests, add_experimental=add_experimental,
114 add_noncompliant=add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000115 autotest_dir=autotest_dir)
showardcd5131c2010-01-12 18:54:33 +0000116
117
mbligh61be4cd2010-03-30 02:02:40 +0000118def db_clean_broken(autotest_dir):
119 """
120 Remove tests from autotest_web that do not have valid control files
showardcd5131c2010-01-12 18:54:33 +0000121
mbligh61be4cd2010-03-30 02:02:40 +0000122 This function invoked when -c supplied on the command line and when
123 running update_all(). Removes tests from database which are not
124 found in the filesystem. Also removes profilers which are just
125 a special case of tests.
126
127 @param autotest_dir: prepended to path strings (/usr/local/autotest).
showardcd5131c2010-01-12 18:54:33 +0000128 """
129 for test in models.Test.objects.all():
130 full_path = os.path.join(autotest_dir, test.path)
131 if not os.path.isfile(full_path):
mbligh61be4cd2010-03-30 02:02:40 +0000132 logging.info("Removing %s", test.path)
showardcd5131c2010-01-12 18:54:33 +0000133 _log_or_execute(repr(test), test.delete)
134
135 # Find profilers that are no longer present
136 for profiler in models.Profiler.objects.all():
137 full_path = os.path.join(autotest_dir, "client", "profilers",
138 profiler.name)
139 if not os.path.exists(full_path):
mbligh61be4cd2010-03-30 02:02:40 +0000140 logging.info("Removing %s", profiler.name)
showardcd5131c2010-01-12 18:54:33 +0000141 _log_or_execute(repr(profiler), profiler.delete)
142
143
mbligh61be4cd2010-03-30 02:02:40 +0000144def db_clean_all(autotest_dir):
145 """
146 Remove all tests from autotest_web - very destructive
147
148 This function invoked when -C supplied on the command line.
149 Removes ALL tests from the database.
150
151 @param autotest_dir: prepended to path strings (/usr/local/autotest).
152 """
153 for test in models.Test.objects.all():
154 full_path = os.path.join(autotest_dir, test.path)
155 logging.info("Removing %s", test.path)
156 _log_or_execute(repr(test), test.delete)
157
158 # Find profilers that are no longer present
159 for profiler in models.Profiler.objects.all():
160 full_path = os.path.join(autotest_dir, "client", "profilers",
161 profiler.name)
162 logging.info("Removing %s", profiler.name)
163 _log_or_execute(repr(profiler), profiler.delete)
164
165
166def update_profilers_in_db(profilers, description='NA',
showardcd5131c2010-01-12 18:54:33 +0000167 add_noncompliant=False):
mbligh61be4cd2010-03-30 02:02:40 +0000168 """
169 Add only profilers to the database from the filesystem.
170
171 This function invoked when -p supplied on command line.
172 Only adds profilers to the database - does not delete any.
173 Profilers are formatted slightly differently than tests.
174
175 @param profilers: list of profilers found in the file system.
176 @param description: simple text to satisfy docstring.
177 @param add_noncompliant: attempt adding test with invalid control files.
178 """
showardcd5131c2010-01-12 18:54:33 +0000179 for profiler in profilers:
mblighed0526a2010-04-22 18:26:17 +0000180 name = os.path.basename(profiler)
181 if name.endswith('.py'):
182 name = name[:-3]
showardcd5131c2010-01-12 18:54:33 +0000183 if not profilers[profiler]:
184 if add_noncompliant:
185 doc = description
186 else:
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -0700187 logging.warning("Skipping %s, missing docstring", profiler)
mbligh781d2692010-04-21 01:04:28 +0000188 continue
showardcd5131c2010-01-12 18:54:33 +0000189 else:
190 doc = profilers[profiler]
191
192 model = models.Profiler.objects.get_or_create(name=name)[0]
193 model.description = doc
194 _log_or_execute(repr(model), model.save)
195
196
Allen Lifce20642017-02-01 17:22:23 -0800197def _set_attributes_custom(test, data):
198 # We set the test name to the dirname of the control file.
199 test_new_name = test.path.split('/')
200 if test_new_name[-1] == 'control' or test_new_name[-1] == 'control.srv':
201 test.name = test_new_name[-2]
202 else:
203 control_name = "%s:%s"
204 control_name %= (test_new_name[-2],
205 test_new_name[-1])
206 test.name = re.sub('control.*\.', '', control_name)
207
208 # We set verify to always False (0).
209 test.run_verify = 0
210
211 if hasattr(data, 'test_parameters'):
212 for para_name in data.test_parameters:
213 test_parameter = models.TestParameter.objects.get_or_create(
214 test=test, name=para_name)[0]
215 test_parameter.save()
216
217
showardcd5131c2010-01-12 18:54:33 +0000218def update_tests_in_db(tests, dry_run=False, add_experimental=False,
mbligh61be4cd2010-03-30 02:02:40 +0000219 add_noncompliant=False, autotest_dir=None):
220 """
221 Scans through all tests and add them to the database.
222
223 This function invoked when -t supplied and for update_all.
224 When test code is discovered in the file system new tests may be added
225
226 @param tests: list of tests found in the filesystem.
227 @param dry_run: not used at this time.
228 @param add_experimental: add tests with experimental attribute set.
229 @param add_noncompliant: attempt adding test with invalid control files.
230 @param autotest_dir: prepended to path strings (/usr/local/autotest).
231 """
showardcd5131c2010-01-12 18:54:33 +0000232 for test in tests:
233 new_test = models.Test.objects.get_or_create(
234 path=test.replace(autotest_dir, '').lstrip('/'))[0]
mbligh61be4cd2010-03-30 02:02:40 +0000235 logging.info("Processing %s", new_test.path)
showardcd5131c2010-01-12 18:54:33 +0000236
237 # Set the test's attributes
238 data = tests[test]
239 _set_attributes_clean(new_test, data)
240
mbligh61be4cd2010-03-30 02:02:40 +0000241 # Custom Attribute Update
Allen Lifce20642017-02-01 17:22:23 -0800242 _set_attributes_custom(new_test, data)
mbligh61be4cd2010-03-30 02:02:40 +0000243
showardcd5131c2010-01-12 18:54:33 +0000244 # This only takes place if --add-noncompliant is provided on the CLI
245 if not new_test.name:
246 test_new_test = test.split('/')
247 if test_new_test[-1] == 'control':
248 new_test.name = test_new_test[-2]
249 else:
250 control_name = "%s:%s"
251 control_name %= (test_new_test[-2],
252 test_new_test[-1])
253 new_test.name = control_name.replace('control.', '')
254
255 # Experimental Check
256 if not add_experimental and new_test.experimental:
257 continue
258
259 _log_or_execute(repr(new_test), new_test.save)
260 add_label_dependencies(new_test)
261
Eric Li861b2d52011-02-04 14:50:35 -0800262 # save TestParameter
263 for para_name in data.test_parameters:
264 test_parameter = models.TestParameter.objects.get_or_create(
265 test=new_test, name=para_name)[0]
266 test_parameter.save()
267
showardcd5131c2010-01-12 18:54:33 +0000268
269def _set_attributes_clean(test, data):
mbligh61be4cd2010-03-30 02:02:40 +0000270 """
271 First pass sets the attributes of the Test object from file system.
showardcd5131c2010-01-12 18:54:33 +0000272
mbligh61be4cd2010-03-30 02:02:40 +0000273 @param test: a test object to be populated for the database.
274 @param data: object with test data from the file system.
275 """
showardcd5131c2010-01-12 18:54:33 +0000276 test_time = { 'short' : 1,
277 'medium' : 2,
278 'long' : 3, }
279
280
281 string_attributes = ('name', 'author', 'test_class', 'test_category',
282 'test_category', 'sync_count')
283 for attribute in string_attributes:
284 setattr(test, attribute, getattr(data, attribute))
285
286 test.description = data.doc
287 test.dependencies = ", ".join(data.dependencies)
288
Aviv Keshet3dd8beb2013-05-13 17:36:04 -0700289 try:
290 test.test_type = control_data.CONTROL_TYPE.get_value(data.test_type)
291 except AttributeError:
292 raise Exception('Unknown test_type %s for test %s', data.test_type,
293 data.name)
294
showardcd5131c2010-01-12 18:54:33 +0000295 int_attributes = ('experimental', 'run_verify')
296 for attribute in int_attributes:
297 setattr(test, attribute, int(getattr(data, attribute)))
298
299 try:
showardcd5131c2010-01-12 18:54:33 +0000300 test.test_time = int(data.time)
301 if test.test_time < 1 or test.time > 3:
302 raise Exception('Incorrect number %d for time' % test.time)
303 except ValueError:
304 pass
305
306 if not test.test_time and str == type(data.time):
307 test.test_time = test_time[data.time.lower()]
showardcd5131c2010-01-12 18:54:33 +0000308
Aviv Keshet6f455262013-03-01 16:02:29 -0800309 test.test_retry = data.retries
310
showardcd5131c2010-01-12 18:54:33 +0000311
312def add_label_dependencies(test):
313 """
mbligh61be4cd2010-03-30 02:02:40 +0000314 Add proper many-to-many relationships from DEPENDENCIES field.
315
316 @param test: test object for the database.
showardcd5131c2010-01-12 18:54:33 +0000317 """
mbligh61be4cd2010-03-30 02:02:40 +0000318
showardcd5131c2010-01-12 18:54:33 +0000319 # clear out old relationships
320 _log_or_execute(repr(test), test.dependency_labels.clear,
321 subject='clear dependencies from')
322
323 for label_name in test.dependencies.split(','):
324 label_name = label_name.strip().lower()
325 if not label_name:
326 continue
327
328 try:
329 label = models.Label.objects.get(name=label_name)
330 except models.Label.DoesNotExist:
331 log_dependency_not_found(label_name)
332 continue
333
334 _log_or_execute(repr(label), test.dependency_labels.add, label,
335 subject='add dependency to %s' % test.name)
336
337
338def log_dependency_not_found(label_name):
mbligh61be4cd2010-03-30 02:02:40 +0000339 """
340 Exception processing when label not found in database.
341
342 @param label_name: from test dependencies.
343 """
showardcd5131c2010-01-12 18:54:33 +0000344 if label_name in DEPENDENCIES_NOT_FOUND:
345 return
mbligh61be4cd2010-03-30 02:02:40 +0000346 logging.info("Dependency %s not found", label_name)
showardcd5131c2010-01-12 18:54:33 +0000347 DEPENDENCIES_NOT_FOUND.add(label_name)
348
349
350def get_tests_from_fs(parent_dir, control_pattern, add_noncompliant=False):
mbligh61be4cd2010-03-30 02:02:40 +0000351 """
352 Find control files in file system and load a list with their info.
showardcd5131c2010-01-12 18:54:33 +0000353
mbligh61be4cd2010-03-30 02:02:40 +0000354 @param parent_dir: directory to search recursively.
355 @param control_pattern: name format of control file.
356 @param add_noncompliant: ignore control file parse errors.
357
mblighed0526a2010-04-22 18:26:17 +0000358 @return dictionary of the form: tests[file_path] = parsed_object
showardcd5131c2010-01-12 18:54:33 +0000359 """
360 tests = {}
361 profilers = False
362 if 'client/profilers' in parent_dir:
363 profilers = True
364 for dir in [ parent_dir ]:
365 files = recursive_walk(dir, control_pattern)
366 for file in files:
367 if '__init__.py' in file or '.svn' in file:
368 continue
369 if not profilers:
370 if not add_noncompliant:
371 try:
372 found_test = control_data.parse_control(file,
373 raise_warnings=True)
374 tests[file] = found_test
375 except control_data.ControlVariableException, e:
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -0700376 logging.warning("Skipping %s\n%s", file, e)
jamesren6e3e9bd2010-04-28 18:07:05 +0000377 except Exception, e:
378 logging.error("Bad %s\n%s", file, e)
showardcd5131c2010-01-12 18:54:33 +0000379 else:
380 found_test = control_data.parse_control(file)
381 tests[file] = found_test
382 else:
showardcd5131c2010-01-12 18:54:33 +0000383 tests[file] = compiler.parseFile(file).doc
384 return tests
385
386
387def recursive_walk(path, wildcard):
mbligh61be4cd2010-03-30 02:02:40 +0000388 """
389 Recursively go through a directory.
390
391 This function invoked by get_tests_from_fs().
392
393 @param path: base directory to start search.
394 @param wildcard: name format to match.
395
mblighed0526a2010-04-22 18:26:17 +0000396 @return A list of files that match wildcard
showardcd5131c2010-01-12 18:54:33 +0000397 """
398 files = []
399 directories = [ path ]
400 while len(directories)>0:
401 directory = directories.pop()
402 for name in os.listdir(directory):
403 fullpath = os.path.join(directory, name)
404 if os.path.isfile(fullpath):
405 # if we are a control file
406 if re.search(wildcard, name):
407 files.append(fullpath)
408 elif os.path.isdir(fullpath):
409 directories.append(fullpath)
410 return files
411
412
413def _log_or_execute(content, func, *args, **kwargs):
mbligh61be4cd2010-03-30 02:02:40 +0000414 """
415 Log a message if dry_run is enabled, or execute the given function.
showardcd5131c2010-01-12 18:54:33 +0000416
mbligh61be4cd2010-03-30 02:02:40 +0000417 Relies on the DRY_RUN global variable.
418
419 @param content: the actual log message.
420 @param func: function to execute if dry_run is not enabled.
421 @param subject: (Optional) The type of log being written. Defaults to
422 the name of the provided function.
showardcd5131c2010-01-12 18:54:33 +0000423 """
424 subject = kwargs.get('subject', func.__name__)
425
426 if DRY_RUN:
mbligh61be4cd2010-03-30 02:02:40 +0000427 logging.info("Would %s: %s", subject, content)
showardcd5131c2010-01-12 18:54:33 +0000428 else:
429 func(*args)
430
431
mbligh61be4cd2010-03-30 02:02:40 +0000432def _create_whitelist_set(whitelist_path):
433 """
434 Create a set with contents from a whitelist file for membership testing.
435
436 @param whitelist_path: full path to the whitelist file.
437
mblighed0526a2010-04-22 18:26:17 +0000438 @return set with files listed one/line - newlines included.
mbligh61be4cd2010-03-30 02:02:40 +0000439 """
440 f = open(whitelist_path, 'r')
441 whitelist_set = set([line.strip() for line in f])
442 f.close()
443 return whitelist_set
444
445
446def update_from_whitelist(whitelist_set, add_experimental, add_noncompliant,
447 autotest_dir):
448 """
449 Scans through all tests in the whitelist and add them to the database.
450
451 This function invoked when -w supplied.
452
453 @param whitelist_set: set of tests in full-path form from a whitelist.
454 @param add_experimental: add tests with experimental attribute set.
455 @param add_noncompliant: attempt adding test with invalid control files.
456 @param autotest_dir: prepended to path strings (/usr/local/autotest).
457 """
458 tests = {}
459 profilers = {}
460 for file_path in whitelist_set:
461 if file_path.find('client/profilers') == -1:
462 try:
463 found_test = control_data.parse_control(file_path,
464 raise_warnings=True)
465 tests[file_path] = found_test
466 except control_data.ControlVariableException, e:
Ilja H. Friedel04be2bd2014-05-07 21:29:59 -0700467 logging.warning("Skipping %s\n%s", file, e)
mbligh61be4cd2010-03-30 02:02:40 +0000468 else:
469 profilers[file_path] = compiler.parseFile(file_path).doc
470
471 if len(tests) > 0:
472 update_tests_in_db(tests, add_experimental=add_experimental,
473 add_noncompliant=add_noncompliant,
474 autotest_dir=autotest_dir)
475 if len(profilers) > 0:
476 update_profilers_in_db(profilers, add_noncompliant=add_noncompliant,
477 description='NA')
478
479
showard909c7a62008-07-15 21:52:38 +0000480def main(argv):
Aviv Keshet6f455262013-03-01 16:02:29 -0800481 """Main function
482 @param argv: List of command line parameters.
483 """
mbligh61be4cd2010-03-30 02:02:40 +0000484
showard909c7a62008-07-15 21:52:38 +0000485 global DRY_RUN
486 parser = optparse.OptionParser()
mbligh61be4cd2010-03-30 02:02:40 +0000487 parser.add_option('-c', '--db-clean-tests',
488 dest='clean_tests', action='store_true',
showard909c7a62008-07-15 21:52:38 +0000489 default=False,
mbligh61be4cd2010-03-30 02:02:40 +0000490 help='Clean client and server tests with invalid control files')
491 parser.add_option('-C', '--db-clear-all-tests',
492 dest='clear_all_tests', action='store_true',
493 default=False,
494 help='Clear ALL client and server tests')
showard909c7a62008-07-15 21:52:38 +0000495 parser.add_option('-d', '--dry-run',
496 dest='dry_run', action='store_true', default=False,
497 help='Dry run for operation')
mbligh322ec1a2008-09-26 16:48:10 +0000498 parser.add_option('-A', '--add-all',
499 dest='add_all', action='store_true',
500 default=False,
mbligh25d656c2009-08-24 22:04:02 +0000501 help='Add site_tests, tests, and test_suites')
502 parser.add_option('-S', '--add-samples',
503 dest='add_samples', action='store_true',
504 default=False,
505 help='Add samples.')
mbligh322ec1a2008-09-26 16:48:10 +0000506 parser.add_option('-E', '--add-experimental',
showard909c7a62008-07-15 21:52:38 +0000507 dest='add_experimental', action='store_true',
mbligh79410e12008-11-20 17:59:15 +0000508 default=True,
Dan Shi5cbd0f82013-03-29 09:22:55 -0700509 help='Add experimental tests to frontend, works only '
510 'with -A (--add-all) option')
showard909c7a62008-07-15 21:52:38 +0000511 parser.add_option('-N', '--add-noncompliant',
512 dest='add_noncompliant', action='store_true',
513 default=False,
mbligh4b5c31e2009-07-11 00:55:34 +0000514 help='Add non-compliant tests (i.e. tests that do not '
Dan Shi5cbd0f82013-03-29 09:22:55 -0700515 'define all required control variables), works '
516 'only with -A (--add-all) option')
mbligh322ec1a2008-09-26 16:48:10 +0000517 parser.add_option('-p', '--profile-dir', dest='profile_dir',
518 help='Directory to recursively check for profiles')
showard909c7a62008-07-15 21:52:38 +0000519 parser.add_option('-t', '--tests-dir', dest='tests_dir',
520 help='Directory to recursively check for control.*')
showard909c7a62008-07-15 21:52:38 +0000521 parser.add_option('-r', '--control-pattern', dest='control_pattern',
522 default='^control.*',
523 help='The pattern to look for in directories for control files')
mbligh322ec1a2008-09-26 16:48:10 +0000524 parser.add_option('-v', '--verbose',
525 dest='verbose', action='store_true', default=False,
526 help='Run in verbose mode')
mbligh61be4cd2010-03-30 02:02:40 +0000527 parser.add_option('-w', '--whitelist-file', dest='whitelist_file',
528 help='Filename for list of test names that must match')
529 parser.add_option('-z', '--autotest-dir', dest='autotest_dir',
mbligh322ec1a2008-09-26 16:48:10 +0000530 default=os.path.join(os.path.dirname(__file__), '..'),
showard909c7a62008-07-15 21:52:38 +0000531 help='Autotest directory root')
532 options, args = parser.parse_args()
lmrc781f552010-05-20 00:36:37 +0000533
534 logging_manager.configure_logging(TestImporterLoggingConfig(),
535 verbose=options.verbose)
536
showard909c7a62008-07-15 21:52:38 +0000537 DRY_RUN = options.dry_run
jamesrenfed94092010-05-03 20:52:47 +0000538 if DRY_RUN:
539 logging.getLogger().setLevel(logging.WARN)
540
Dan Shi5cbd0f82013-03-29 09:22:55 -0700541 if len(argv) > 1 and options.add_noncompliant and not options.add_all:
542 logging.error('-N (--add-noncompliant) must be ran with option -A '
543 '(--add-All).')
544 return 1
545
546 if len(argv) > 1 and options.add_experimental and not options.add_all:
547 logging.error('-E (--add-experimental) must be ran with option -A '
548 '(--add-All).')
549 return 1
550
mbligh322ec1a2008-09-26 16:48:10 +0000551 # Make sure autotest_dir is the absolute path
552 options.autotest_dir = os.path.abspath(options.autotest_dir)
553
554 if len(args) > 0:
mbligh61be4cd2010-03-30 02:02:40 +0000555 logging.error("Invalid option(s) provided: %s", args)
showard909c7a62008-07-15 21:52:38 +0000556 parser.print_help()
557 return 1
558
mbligh61be4cd2010-03-30 02:02:40 +0000559 if options.verbose:
560 logging.getLogger().setLevel(logging.DEBUG)
561
562 if len(argv) == 1 or (len(argv) == 2 and options.verbose):
mbligh322ec1a2008-09-26 16:48:10 +0000563 update_all(options.autotest_dir, options.add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000564 options.add_experimental)
565 db_clean_broken(options.autotest_dir)
mbligh322ec1a2008-09-26 16:48:10 +0000566 return 0
567
mbligh61be4cd2010-03-30 02:02:40 +0000568 if options.clear_all_tests:
569 if (options.clean_tests or options.add_all or options.add_samples or
570 options.add_noncompliant):
571 logging.error(
572 "Can only pass --autotest-dir, --dry-run and --verbose with "
573 "--db-clear-all-tests")
574 return 1
575 db_clean_all(options.autotest_dir)
576
577 whitelist_set = None
578 if options.whitelist_file:
579 if options.add_all:
580 logging.error("Cannot pass both --add-all and --whitelist-file")
581 return 1
582 whitelist_path = os.path.abspath(options.whitelist_file)
583 if not os.path.isfile(whitelist_path):
584 logging.error("--whitelist-file (%s) not found", whitelist_path)
585 return 1
586 logging.info("Using whitelist file %s", whitelist_path)
587 whitelist_set = _create_whitelist_set(whitelist_path)
588 update_from_whitelist(whitelist_set,
589 add_experimental=options.add_experimental,
590 add_noncompliant=options.add_noncompliant,
591 autotest_dir=options.autotest_dir)
mbligh322ec1a2008-09-26 16:48:10 +0000592 if options.add_all:
593 update_all(options.autotest_dir, options.add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000594 options.add_experimental)
mbligh25d656c2009-08-24 22:04:02 +0000595 if options.add_samples:
596 update_samples(options.autotest_dir, options.add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000597 options.add_experimental)
showard909c7a62008-07-15 21:52:38 +0000598 if options.tests_dir:
mbligh4b5c31e2009-07-11 00:55:34 +0000599 options.tests_dir = os.path.abspath(options.tests_dir)
showard909c7a62008-07-15 21:52:38 +0000600 tests = get_tests_from_fs(options.tests_dir, options.control_pattern,
601 add_noncompliant=options.add_noncompliant)
mbligh322ec1a2008-09-26 16:48:10 +0000602 update_tests_in_db(tests, add_experimental=options.add_experimental,
showard909c7a62008-07-15 21:52:38 +0000603 add_noncompliant=options.add_noncompliant,
mbligh61be4cd2010-03-30 02:02:40 +0000604 autotest_dir=options.autotest_dir)
mbligh322ec1a2008-09-26 16:48:10 +0000605 if options.profile_dir:
606 profilers = get_tests_from_fs(options.profile_dir, '.*py$')
mbligh61be4cd2010-03-30 02:02:40 +0000607 update_profilers_in_db(profilers,
mbligh322ec1a2008-09-26 16:48:10 +0000608 add_noncompliant=options.add_noncompliant,
609 description='NA')
mbligh61be4cd2010-03-30 02:02:40 +0000610 if options.clean_tests:
611 db_clean_broken(options.autotest_dir)
showard909c7a62008-07-15 21:52:38 +0000612
613
showard909c7a62008-07-15 21:52:38 +0000614if __name__ == "__main__":
Alex Miller62154932013-08-12 11:04:31 -0700615 sys.exit(main(sys.argv))