Initial commit of a new testsuite feature: test categories.
This feature allows us to group test cases into logical groups (categories), and to only run a subset of test cases based on these categories.
Each test-case can have a new method getCategories(self): which returns a list of strings that are the categories to which the test case belongs.
If a test-case does not provide its own categories, we will look for categories in the class that contains the test case.
If that fails too, the default implementation looks for a .category file, which contains a comma separated list of strings.
The test suite will recurse look for .categories up until the top level directory (which we guarantee will have an empty .category file).
The driver dotest.py has a new --category <foo> option, which can be repeated, and specifies which categories of tests you want to run.
(example: ./dotest.py --category objc --category expression)
All tests that do not belong to any specified category will be skipped. Other filtering options still exist and should not interfere with category filtering.
A few tests have been categorized. Feel free to categorize others, and to suggest new categories that we could want to use.
All categories need to be validly defined in dotest.py, or the test suite will refuse to run when you use them as arguments to --category.
In the end, failures will be reported on a per-category basis, as well as in the usual format.
This is the very first stage of this feature. Feel free to chime in with ideas for improvements!
git-svn-id: https://llvm.org/svn/llvm-project/lldb/trunk@164403 91177308-0d34-0410-b5e6-96231b3b80d8
diff --git a/test/dotest.py b/test/dotest.py
index 7b6685d..c69b9dc 100755
--- a/test/dotest.py
+++ b/test/dotest.py
@@ -66,6 +66,17 @@
# Global variables:
#
+# Dictionary of categories
+# When you define a new category for your testcases, be sure to add it here, or the test suite
+# will gladly complain as soon as you try to use it. This allows us to centralize which categories
+# exist, and to provide a description for each one
+validCategories = {
+'dataformatters':'Tests related to the type command and the data formatters subsystem',
+'expression':'Tests related to the expression parser',
+'objc':'Tests related to the Objective-C programming language support',
+'pyapi':'Tests related to the Python API'
+}
+
# The test suite.
suite = unittest2.TestSuite()
@@ -94,6 +105,13 @@
# The dictionary as a result of sourcing blacklistFile.
blacklistConfig = {}
+# The list of categories we said we care about
+categoriesList = None
+# set to true if we are going to use categories for cherry-picking test cases
+useCategories = False
+# use this to track per-category failures
+failuresPerCategory = {}
+
# The config file is optional.
configFile = None
@@ -296,6 +314,9 @@
global dont_do_dwarf_test
global blacklist
global blacklistConfig
+ global categoriesList
+ global validCategories
+ global useCategories
global configFile
global archs
global compilers
@@ -353,6 +374,7 @@
X('-l', "Don't skip long running tests")
group.add_argument('-p', metavar='pattern', help='Specify a regexp filename pattern for inclusion in the test suite')
group.add_argument('-X', metavar='directory', help="Exclude a directory from consideration for test discovery. -X types => if 'types' appear in the pathname components of a potential testfile, it will be ignored")
+ group.add_argument('-G', '--category', metavar='category', action='append', dest='categoriesList', help=textwrap.dedent('''Specify categories of test cases of interest. Can be specified more than once.'''))
# Configuration options
group = parser.add_argument_group('Configuration options')
@@ -400,6 +422,16 @@
else:
archs = [platform_machine]
+ if args.categoriesList:
+ for category in args.categoriesList:
+ if not(category in validCategories):
+ print "fatal error: category '" + category + "' is not a valid category - edit dotest.py or correct your invocation"
+ sys.exit(1)
+ categoriesList = set(args.categoriesList)
+ useCategories = True
+ else:
+ categoriesList = []
+
if args.compilers:
compilers = args.compilers
else:
@@ -1228,7 +1260,34 @@
else:
return str(test)
+ def getCategoriesForTest(self,test):
+ if hasattr(test,"getCategories"):
+ test_categories = test.getCategories()
+ elif inspect.ismethod(test) and test.__self__ != None and hasattr(test.__self__,"getCategories"):
+ test_categories = test.__self__.getCategories()
+ else:
+ test_categories = []
+ if test_categories == None:
+ test_categories = []
+ return test_categories
+
+ def shouldSkipBecauseOfCategories(self,test):
+ global useCategories
+ import inspect
+ if useCategories:
+ global categoriesList
+ test_categories = self.getCategoriesForTest(test)
+ if len(test_categories) == 0 or len(categoriesList & set(test_categories)) == 0:
+ return True
+ return False
+
+ def hardMarkAsSkipped(self,test):
+ getattr(test, test._testMethodName).__func__.__unittest_skip__ = True
+ getattr(test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run"
+
def startTest(self, test):
+ if self.shouldSkipBecauseOfCategories(test):
+ self.hardMarkAsSkipped(test)
self.counter += 1
if self.showAll:
self.stream.write(self.fmt % self.counter)
@@ -1244,11 +1303,19 @@
def addFailure(self, test, err):
global sdir_has_content
+ global failuresPerCategory
sdir_has_content = True
super(LLDBTestResult, self).addFailure(test, err)
method = getattr(test, "markFailure", None)
if method:
method()
+ if useCategories:
+ test_categories = self.getCategoriesForTest(test)
+ for category in test_categories:
+ if category in failuresPerCategory:
+ failuresPerCategory[category] = failuresPerCategory[category] + 1
+ else:
+ failuresPerCategory[category] = 1
def addExpectedFailure(self, test, err):
global sdir_has_content
@@ -1296,6 +1363,11 @@
sys.stderr.write("Session logs for test failures/errors/unexpected successes"
" can be found in directory '%s'\n" % sdir_name)
+if useCategories and len(failuresPerCategory) > 0:
+ sys.stderr.write("Failures per category:\n")
+ for category in failuresPerCategory:
+ sys.stderr.write("%s - %d\n" % (category,failuresPerCategory[category]))
+
fname = os.path.join(sdir_name, "TestFinished")
with open(fname, "w") as f:
print >> f, "Test finished at: %s\n" % datetime.datetime.now().strftime("%Y-%m-%d-%H_%M_%S")