layers: adjust VU database/scripts for impl flag
Since the VU database audit, "U" is no longer valid as a value for
the check_implemented status; only "Y" or "N" are valid.
Adjust the spec.py generator script to default to "N" for newly added
Valid Usages.
Adjust database stats script to emit an error message if the
check_implemented value is anything other than "Y" or "N".
Added some whitespace between functional areas.
Update the database to change recently VU lines from "U" to "N".
Change-Id: I72929362d67ddea862cb380e1bfcd0b475531d56
diff --git a/layers/vk_validation_stats.py b/layers/vk_validation_stats.py
index f34fb06..acf73ae 100755
--- a/layers/vk_validation_stats.py
+++ b/layers/vk_validation_stats.py
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
-# Copyright (c) 2015-2016 The Khronos Group Inc.
-# Copyright (c) 2015-2016 Valve Corporation
-# Copyright (c) 2015-2016 LunarG, Inc.
-# Copyright (c) 2015-2016 Google Inc.
+# Copyright (c) 2015-2017 The Khronos Group Inc.
+# Copyright (c) 2015-2017 Valve Corporation
+# Copyright (c) 2015-2017 LunarG, Inc.
+# Copyright (c) 2015-2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -89,6 +89,7 @@
self.db_implemented_enums = [] # list of all error enums claiming to be implemented in database file
self.db_unimplemented_implicit = [] # list of all implicit checks that aren't marked implemented
self.db_enum_to_tests = {} # dict where enum is key to lookup list of tests implementing the enum
+ self.db_invalid_implemented = [] # list of checks with invalid check_implemented flags
#self.src_implemented_enums
def read(self):
"""Read a database file into internal data structures, format of each line is <enum><implemented Y|N?><testname><api><errormsg><notes>"""
@@ -120,6 +121,8 @@
self.db_implemented_enums.append(error_enum)
elif 'implicit' in note: # only make note of non-implemented implicit checks
self.db_unimplemented_implicit.append(error_enum)
+ if implemented not in ['Y', 'N']:
+ self.db_invalid_implemented.append(error_enum)
if testname.lower() not in ['unknown', 'none']:
self.db_enum_to_tests[error_enum] = testname.split(',')
#if len(self.db_enum_to_tests[error_enum]) > 1:
@@ -288,12 +291,23 @@
# Process stats - Just doing this inline in main, could make a fancy class to handle
# all the processing of data and then get results from that
txt_color = bcolors()
+
print("Validation Statistics")
# First give number of checks in db & header and report any discrepancies
db_enums = len(val_db.db_dict.keys())
hdr_enums = len(val_header.enums)
print(" Database file includes %d unique checks" % (db_enums))
print(" Header file declares %d unique checks" % (hdr_enums))
+
+ # Report any checks that have an invalid check_implemented flag
+ if len(val_db.db_invalid_implemented) > 0:
+ result = 1
+ print(txt_color.red() + "The following checks have an invalid check_implemented flag (must be 'Y' or 'N'):" + txt_color.endc())
+ for invalid_imp_enum in val_db.db_invalid_implemented:
+ check_implemented = val_db.db_dict[invalid_imp_enum]['check_implemented']
+ print(txt_color.red() + " %s has check_implemented flag '%s'" % (invalid_imp_enum, check_implemented) + txt_color.endc())
+
+ # Report details about how well the Database and Header are synchronized.
tmp_db_dict = val_db.db_dict
db_missing = []
for enum in val_header.enums:
@@ -312,6 +326,7 @@
print(txt_color.red() + " The following checks are in database but haven't been declared in the header:" + txt_color.endc())
for extra_enum in tmp_db_dict:
print(txt_color.red() + " %s" % (extra_enum) + txt_color.endc())
+
# Report out claimed implemented checks vs. found actual implemented checks
imp_not_found = [] # Checks claimed to implemented in DB file but no source found
imp_not_claimed = [] # Checks found implemented but not claimed to be in DB
@@ -325,6 +340,7 @@
if src_enum not in val_db.db_implemented_enums:
imp_not_claimed.append(src_enum)
print(" Database file claims that %d checks (%s) are implemented in source." % (len(val_db.db_implemented_enums), "{0:.0f}%".format(float(len(val_db.db_implemented_enums))/db_enums * 100)))
+
if len(val_db.db_unimplemented_implicit) > 0:
print(" Database file claims %d implicit checks (%s) that are not implemented." % (len(val_db.db_unimplemented_implicit), "{0:.0f}%".format(float(len(val_db.db_unimplemented_implicit))/db_enums * 100)))
total_checks = len(val_db.db_implemented_enums) + len(val_db.db_unimplemented_implicit)
@@ -342,6 +358,7 @@
print(txt_color.red() + " The following checks are implemented in source, but not claimed to be in Database:" + txt_color.endc())
for imp_enum in imp_not_claimed:
print(txt_color.red() + " %s" % (imp_enum) + txt_color.endc())
+
if multiple_uses:
print(txt_color.yellow() + " Note that some checks are used multiple times. These may be good candidates for new valid usage spec language." + txt_color.endc())
print(txt_color.yellow() + " Here is a list of each check used multiple times with its number of uses:" + txt_color.endc())
@@ -350,6 +367,7 @@
print(txt_color.yellow() + " %s: %d uses in file,line:" % (enum, val_source.enum_count_dict[enum]['count']) + txt_color.endc())
for file_line in val_source.enum_count_dict[enum]['file_line']:
print(txt_color.yellow() + " \t%s" % (file_line) + txt_color.endc())
+
# Now check that tests claimed to be implemented are actual test names
bad_testnames = []
tests_missing_enum = {} # Report tests that don't use validation error enum to check for error case
@@ -374,6 +392,7 @@
print(txt_color.yellow() + " Testname %s does not explicitly check for these ids:" % (testname) + txt_color.endc())
for enum in tests_missing_enum[testname]:
print(txt_color.yellow() + " %s" % (enum) + txt_color.endc())
+
# TODO : Go through all enums found in the test file and make sure they're correctly documented in the database file
print(" Database file claims that %d checks have tests written." % len(val_db.db_enum_to_tests))
if len(bad_testnames) == 0: