tests: Update document validator to check for valid tests
This adds a set of checks to the document validator that will iterate through
all of the checks documented in vk_validation_layer_details.md and compare the
names of the test(s) for each check against the names of the actual tests in
layer_validation_tests.cpp.
Currently, all checks not ending in "_NONE" that don't have a valid test
recorded in the table will report a warning with a final count of the warnings
given. There are initially 131 such warnings, but I suspect at least a handful
of them are implemented but just need to be correctly documented.
Once all the test holes are filled in we can promote the warnings to errors
to make sure that any new checks are both documented and have a valid test
implemented.
Rough support of wild-carded testnames such as "CreatePipeline*NotConsumed" is
included.
diff --git a/vk_layer_documentation_generate.py b/vk_layer_documentation_generate.py
index a14e16b..5e79369 100755
--- a/vk_layer_documentation_generate.py
+++ b/vk_layer_documentation_generate.py
@@ -85,6 +85,7 @@
builtin_headers = [layer_inputs[ln]['header'] for ln in layer_inputs]
builtin_source = [layer_inputs[ln]['source'] for ln in layer_inputs]
+builtin_tests = ['tests/layer_validation_tests.cpp', ]
# List of extensions in layers that are included in documentation, but not in vulkan.py API set
layer_extension_functions = ['objTrackGetObjects', 'objTrackGetObjectsOfType']
@@ -93,6 +94,7 @@
parser = argparse.ArgumentParser(description='Generate layer documenation from source.')
parser.add_argument('--in_headers', required=False, default=builtin_headers, help='The input layer header files from which code will be generated.')
parser.add_argument('--in_source', required=False, default=builtin_source, help='The input layer source files from which code will be generated.')
+ parser.add_argument('--test_source', required=False, default=builtin_tests, help='The input test source files from which code will be generated.')
parser.add_argument('--layer_doc', required=False, default='layers/vk_validation_layer_details.md', help='Existing layer document to be validated against actual layers.')
parser.add_argument('--validate', action='store_true', default=False, help='Validate that there are no mismatches between layer documentation and source. This includes cross-checking the validation checks, and making sure documented Vulkan API calls exist.')
parser.add_argument('--print_structs', action='store_true', default=False, help='Primarily a debug option that prints out internal data structs used to generate layer docs.')
@@ -105,10 +107,12 @@
def __init__(self):
self.GREEN = '\033[0;32m'
self.RED = '\033[0;31m'
+ self.YELLOW = '\033[1;33m'
self.ENDC = '\033[0m'
if 'Linux' != platform.system():
self.GREEN = ''
self.RED = ''
+ self.YELLOW = ''
self.ENDC = ''
def green(self):
@@ -117,9 +121,43 @@
def red(self):
return self.RED
+ def yellow(self):
+ return self.YELLOW
+
def endc(self):
return self.ENDC
+# Class to parse the validation layer test source and store testnames
+class TestParser:
+ def __init__(self, test_file_list, test_group_name='VkLayerTest'):
+ self.test_files = test_file_list
+ self.tests_set = set()
+ self.test_trigger_txt = 'TEST_F(%s' % test_group_name
+
+ # Parse test files into internal data struct
+ def parse(self):
+ # For each test file, parse test names into set
+ grab_next_line = False # handle testname on separate line than wildcard
+ for test_file in self.test_files:
+ with open(test_file) as tf:
+ for line in tf:
+ if True in [line.strip().startswith(comment) for comment in ['//', '/*']]:
+ continue
+
+ if self.test_trigger_txt in line:
+ #print('Test wildcard in line: %s' % (line))
+ testname = line.split(',')[-1]
+ testname = testname.strip().strip(' {)')
+ #print('Inserting test: "%s"' % (testname))
+ if ('' == testname):
+ grab_next_line = True
+ continue
+ self.tests_set.add(testname)
+ if grab_next_line: # test name on its own line
+ grab_next_line = False
+ testname = testname.strip().strip(' {)')
+ self.tests_set.add(testname)
+
# Class to parse the layer source code and store details in internal data structs
class LayerParser:
def __init__(self, header_file_list, source_file_list):
@@ -145,7 +183,6 @@
# For now skipping lines starting w/ comment, may use these to capture
# documentation in the future
continue
-
# Find enums
if store_enum:
if '}' in line: # we're done with enum definition
@@ -251,13 +288,16 @@
self.layer_doc_dict[layer_name] = {}
self.layer_doc_dict[layer_name]['overview'] = ''
- # Verify that checks and api references in layer doc match reality
+ # Verify that checks, tests and api references in layer doc match reality
# Report API calls from doc that are not found in API
# Report checks from doc that are not in actual layers
# Report checks from layers that are not captured in doc
- def validate(self, layer_dict):
+ # Report checks from doc that do not have a valid test
+ def validate(self, layer_dict, tests_set):
+ #print("tests_set: %s" % (tests_set))
# Count number of errors found and return it
errors_found = 0
+ warnings_found = 0
# First we'll go through the doc datastructures and flag any issues
for chk in self.enum_list:
doc_layer_found = False
@@ -269,6 +309,7 @@
if not doc_layer_found:
print(self.txt_color.red() + 'Actual layers do not contain documented check: %s' % (chk) + self.txt_color.endc())
errors_found += 1
+
# Now go through API names in doc and verify they're real
# First we're going to transform proto names from vulkan.py into single list
core_api_names = [p.name for p in vulkan.core.protos]
@@ -284,6 +325,24 @@
if api[2:] not in api_names and api not in layer_extension_functions:
print(self.txt_color.red() + 'Doc references invalid function: %s' % (api) + self.txt_color.endc())
errors_found += 1
+ # For now warn on missing or invalid tests
+ for test in self.layer_doc_dict[ln][chk]['tests']:
+ if '*' in test:
+ # naive way to handle wildcards, just make sure we have matches on parts
+ test_parts = test.split('*')
+ for part in test_parts:
+ part_found = False
+ for t in tests_set:
+ if part in t:
+ part_found = True
+ break
+ if not part_found:
+ print(self.txt_color.yellow() + 'Validation check %s has missing or invalid test : %s' % (chk, test))
+ warnings_found += 1
+ break
+ elif test not in tests_set and not chk.endswith('_NONE'):
+ print(self.txt_color.yellow() + 'Validation check %s has missing or invalid test : %s' % (chk, test))
+ warnings_found += 1
# Now go through all of the actual checks in the layers and make sure they're covered in the doc
for ln in layer_dict:
for chk in layer_dict[ln]['CHECKS']:
@@ -291,7 +350,7 @@
print(self.txt_color.red() + 'Doc is missing check: %s' % (chk) + self.txt_color.endc())
errors_found += 1
- return errors_found
+ return (errors_found, warnings_found)
# Print all of the checks captured in the doc
def print_checks(self):
@@ -304,6 +363,9 @@
layer_parser = LayerParser(opts.in_headers, opts.in_source)
# Parse files into internal data structs
layer_parser.parse()
+ # Parse test files
+ test_parser = TestParser(opts.test_source)
+ test_parser.parse()
# Generate requested types of output
if opts.print_structs: # Print details of internal data structs
@@ -315,9 +377,13 @@
layer_doc.print_checks()
if opts.validate:
- num_errors = layer_doc.validate(layer_parser.layer_dict)
+ (num_errors, num_warnings) = layer_doc.validate(layer_parser.layer_dict, test_parser.tests_set)
+ txt_color = bcolors()
+ if (0 == num_warnings):
+ print(txt_color.green() + 'No warning cases found between %s and implementation' % (os.path.basename(opts.layer_doc)) + txt_color.endc())
+ else:
+ print(txt_color.yellow() + 'Found %s warnings. See above for details' % num_warnings)
if (0 == num_errors):
- txt_color = bcolors()
print(txt_color.green() + 'No mismatches found between %s and implementation' % (os.path.basename(opts.layer_doc)) + txt_color.endc())
else:
return num_errors