scripts: repair and update validation_stats.py
Parses validation headers registry file 'validusage.json',
layer source code and layer test code and reports various
statistics and consistency checks.
Deletes the checked-in error database file
vk_validation_error_database.txt in favor of the ability
to generate an up-to-date version on demand.
Change-Id: I604b972cbb3c6954d8f321ec9e9c214455e38765
diff --git a/scripts/vk_validation_stats.py b/scripts/vk_validation_stats.py
index 05b9125..00bdd2e 100755
--- a/scripts/vk_validation_stats.py
+++ b/scripts/vk_validation_stats.py
@@ -1,8 +1,8 @@
#!/usr/bin/env python3
-# Copyright (c) 2015-2017 The Khronos Group Inc.
-# Copyright (c) 2015-2017 Valve Corporation
-# Copyright (c) 2015-2017 LunarG, Inc.
-# Copyright (c) 2015-2017 Google Inc.
+# Copyright (c) 2015-2018 The Khronos Group Inc.
+# Copyright (c) 2015-2018 Valve Corporation
+# Copyright (c) 2015-2018 LunarG, Inc.
+# Copyright (c) 2015-2018 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@@ -17,40 +17,35 @@
# limitations under the License.
#
# Author: Tobin Ehlis <tobine@google.com>
+# Author: Dave Houlton <daveh@lunarg.com>
import argparse
import os
import sys
+import operator
import platform
+import json
+import re
+import csv
+import html
+from collections import defaultdict
-# vk_validation_stats.py overview
-#
-# usage:
-# python vk_validation_stats.py [verbose]
-#
-# Arguments:
-# verbose - enables verbose output, including VUID duplicates
-#
-# This script is intended to generate statistics on the state of validation code
-# based on information parsed from the source files and the database file
-# Here's what it currently does:
-# 1. Parse vk_validation_error_database.txt to store claimed state of validation checks
-# 2. Parse vk_validation_error_messages.h to verify the actual checks in header vs. the
-# claimed state of the checks
-# 3. Parse source files to identify which checks are implemented and verify that this
-# exactly matches the list of checks claimed to be implemented in the database
-# 4. Parse test file(s) and verify that reported tests exist
-# 5. Report out stats on number of checks, implemented checks, and duplicated checks
-#
-# If a mis-match is found during steps 2, 3, or 4, then the script exits w/ a non-zero error code
-# otherwise, the script will exit(0)
-#
-# TODO:
-# 1. Would also like to report out number of existing checks that don't yet use new, unique enum
-# 2. Could use notes to store custom fields (like TODO) and print those out here
-# 3. Update test code to check if tests use new, unique enums to check for errors instead of strings
+verbose_mode = False
+txt_db = False
+csv_db = False
+html_db = False
+txt_filename = "validation_error_database.txt"
+csv_filename = "validation_error_database.csv"
+html_filename = "validation_error_database.html"
+# header_file = '../layers/vk_validation_error_messages.h'
+test_file = '../tests/layer_validation_tests.cpp'
+vuid_prefixes = ['VUID-', 'UNASSIGNED-']
-db_file = '../layers/vk_validation_error_database.txt'
+# Hard-coded flags that could be command line args, if we decide that's useful
+# replace KHR vuids with non-KHR during consistency checking
+dealias_khr = True
+ignore_unassigned = True # These are not found in layer code unless they appear explicitly (most don't), so produce false positives
+
generated_layer_source_directories = [
'build',
'dbuild',
@@ -68,122 +63,164 @@
'../layers/shader_validation.cpp',
'../layers/buffer_validation.cpp',
]
-header_file = '../layers/vk_validation_error_messages.h'
-# TODO : Don't hardcode linux path format if we want this to run on windows
-test_file = '../tests/layer_validation_tests.cpp'
-# List of enums that are allowed to be used more than once so don't warn on their duplicates
-duplicate_exceptions = [
-'VALIDATION_ERROR_258004ea', # This covers the broad case that all child objects must be destroyed at DestroyInstance time
-'VALIDATION_ERROR_24a002f4', # This covers the broad case that all child objects must be destroyed at DestroyDevice time
-'VALIDATION_ERROR_0280006e', # Obj tracker check makes sure non-null framebuffer is valid & CV check makes sure it's compatible w/ renderpass framebuffer
-'VALIDATION_ERROR_12200682', # This is an aliasing error that we report twice, for each of the two allocations that are aliasing
-'VALIDATION_ERROR_1060d201', # Covers valid shader module handle for both Compute & Graphics pipelines
-'VALIDATION_ERROR_0c20c601', # This is a case for VkMappedMemoryRange struct that is used by both Flush & Invalidate MappedMemoryRange
-'VALIDATION_ERROR_0a400c01', # This is a blanket case for all invalid image aspect bit errors. The spec link has appropriate details for all separate cases.
-'VALIDATION_ERROR_0a8007fc', # This case covers two separate checks which are done independently
-'VALIDATION_ERROR_0a800800', # This case covers two separate checks which are done independently
-'VALIDATION_ERROR_15c0028a', # This is a descriptor set write update error that we use for a couple copy cases as well
-'VALIDATION_ERROR_1bc002de', # Single error for mis-matched stageFlags of vkCmdPushConstants() that is flagged for no stage flags & mis-matched flags
-'VALIDATION_ERROR_1880000e', # Handles both depth/stencil & compressed image errors for vkCmdClearColorImage()
-'VALIDATION_ERROR_0a600152', # Used for the mipLevel check of both dst & src images on vkCmdCopyImage call
-'VALIDATION_ERROR_0a600154', # Used for the arraySize check of both dst & src images on vkCmdCopyImage call
-'VALIDATION_ERROR_1500099e', # Used for both x & y bounds of viewport
-'VALIDATION_ERROR_1d8004a6', # Used for both x & y value of scissors to make sure they're not negative
-'VALIDATION_ERROR_1462ec01', # Surface of VkSwapchainCreateInfoKHR must be valid when creating both single or shared swapchains
-'VALIDATION_ERROR_1460de01', # oldSwapchain of VkSwapchainCreateInfoKHR must be valid when creating both single or shared swapchains
-'VALIDATION_ERROR_146009f2', # Single error for both imageFormat & imageColorSpace requirements when creating swapchain
-'VALIDATION_ERROR_15c00294', # Used twice for the same error codepath as both a param & to set a variable, so not really a duplicate
-]
-class ValidationDatabase:
- def __init__(self, filename=db_file):
- self.db_file = filename
- self.delimiter = '~^~'
- self.db_dict = {} # complete dict of all db values per error enum
- # specialized data structs with slices of complete dict
- self.db_implemented_enums = [] # list of all error enums claiming to be implemented in database file
- self.db_unimplemented_implicit = [] # list of all implicit checks that aren't marked implemented
- self.db_enum_to_tests = {} # dict where enum is key to lookup list of tests implementing the enum
- self.db_invalid_implemented = [] # list of checks with invalid check_implemented flags
- #self.src_implemented_enums
- def read(self):
- """Read a database file into internal data structures, format of each line is <enum><implemented Y|N?><testname><api><errormsg><notes>"""
- #db_dict = {} # This is a simple db of just enum->errormsg, the same as is created from spec
- #max_id = 0
- with open(self.db_file, "r", encoding="utf8") as infile:
- for line in infile:
- line = line.strip()
- if line.startswith('#') or '' == line:
- continue
- db_line = line.split(self.delimiter)
- if len(db_line) != 8:
- print("ERROR: Bad database line doesn't have 8 elements: %s" % (line))
- error_enum = db_line[0]
- implemented = db_line[1]
- testname = db_line[2]
- api = db_line[3]
- vuid_string = db_line[4]
- core_ext = db_line[5]
- error_str = db_line[6]
- note = db_line[7]
- # Read complete database contents into our class var for later use
- self.db_dict[error_enum] = {}
- self.db_dict[error_enum]['check_implemented'] = implemented
- self.db_dict[error_enum]['testname'] = testname
- self.db_dict[error_enum]['api'] = api
- self.db_dict[error_enum]['vuid_string'] = vuid_string
- self.db_dict[error_enum]['core_ext'] = core_ext
- self.db_dict[error_enum]['error_string'] = error_str
- self.db_dict[error_enum]['note'] = note
- # Now build custom data structs
- if 'Y' == implemented:
- self.db_implemented_enums.append(error_enum)
- elif 'implicit' in note: # only make note of non-implemented implicit checks
- self.db_unimplemented_implicit.append(error_enum)
- if implemented not in ['Y', 'N']:
- self.db_invalid_implemented.append(error_enum)
- if testname.lower() not in ['unknown', 'none', 'nottestable']:
- self.db_enum_to_tests[error_enum] = testname.split(',')
- #if len(self.db_enum_to_tests[error_enum]) > 1:
- # print "Found check %s that has multiple tests: %s" % (error_enum, self.db_enum_to_tests[error_enum])
- #else:
- # print "Check %s has single test: %s" % (error_enum, self.db_enum_to_tests[error_enum])
- #unique_id = int(db_line[0].split('_')[-1])
- #if unique_id > max_id:
- # max_id = unique_id
- #print "Found %d total enums in database" % (len(self.db_dict.keys()))
- #print "Found %d enums claiming to be implemented in source" % (len(self.db_implemented_enums))
- #print "Found %d enums claiming to have tests implemented" % (len(self.db_enum_to_tests.keys()))
+khr_aliases = {
+ 'VUID-vkBindBufferMemory2KHR-device-parameter' : 'VUID-vkBindBufferMemory2-device-parameter',
+ 'VUID-vkBindBufferMemory2KHR-pBindInfos-parameter' : 'VUID-vkBindBufferMemory2-pBindInfos-parameter',
+ 'VUID-vkBindImageMemory2KHR-device-parameter' : 'VUID-vkBindImageMemory2-device-parameter',
+ 'VUID-vkBindImageMemory2KHR-pBindInfos-parameter' : 'VUID-vkBindImageMemory2-pBindInfos-parameter',
+ 'VUID-vkCmdDispatchBaseKHR-commandBuffer-parameter' : 'VUID-vkCmdDispatchBase-commandBuffer-parameter',
+ 'VUID-vkCmdSetDeviceMaskKHR-commandBuffer-parameter' : 'VUID-vkCmdSetDeviceMask-commandBuffer-parameter',
+ 'VUID-vkCreateDescriptorUpdateTemplateKHR-device-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-device-parameter',
+ 'VUID-vkCreateDescriptorUpdateTemplateKHR-pDescriptorUpdateTemplate-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-pDescriptorUpdateTemplate-parameter',
+ 'VUID-vkCreateSamplerYcbcrConversionKHR-device-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-device-parameter',
+ 'VUID-vkCreateSamplerYcbcrConversionKHR-pYcbcrConversion-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-pYcbcrConversion-parameter',
+ 'VUID-vkDestroyDescriptorUpdateTemplateKHR-descriptorUpdateTemplate-parameter' : 'VUID-vkDestroyDescriptorUpdateTemplate-descriptorUpdateTemplate-parameter',
+ 'VUID-vkDestroyDescriptorUpdateTemplateKHR-descriptorUpdateTemplate-parent' : 'VUID-vkDestroyDescriptorUpdateTemplate-descriptorUpdateTemplate-parent',
+ 'VUID-vkDestroyDescriptorUpdateTemplateKHR-device-parameter' : 'VUID-vkDestroyDescriptorUpdateTemplate-device-parameter',
+ 'VUID-vkDestroySamplerYcbcrConversionKHR-device-parameter' : 'VUID-vkDestroySamplerYcbcrConversion-device-parameter',
+ 'VUID-vkDestroySamplerYcbcrConversionKHR-ycbcrConversion-parameter' : 'VUID-vkDestroySamplerYcbcrConversion-ycbcrConversion-parameter',
+ 'VUID-vkDestroySamplerYcbcrConversionKHR-ycbcrConversion-parent' : 'VUID-vkDestroySamplerYcbcrConversion-ycbcrConversion-parent',
+ 'VUID-vkEnumeratePhysicalDeviceGroupsKHR-instance-parameter' : 'VUID-vkEnumeratePhysicalDeviceGroups-instance-parameter',
+ 'VUID-vkEnumeratePhysicalDeviceGroupsKHR-pPhysicalDeviceGroupProperties-parameter' : 'VUID-vkEnumeratePhysicalDeviceGroups-pPhysicalDeviceGroupProperties-parameter',
+ 'VUID-vkGetBufferMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetBufferMemoryRequirements2-device-parameter',
+ 'VUID-vkGetDescriptorSetLayoutSupportKHR-device-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-device-parameter',
+ 'VUID-vkGetDeviceGroupPeerMemoryFeaturesKHR-device-parameter' : 'VUID-vkGetDeviceGroupPeerMemoryFeatures-device-parameter',
+ 'VUID-vkGetDeviceGroupPeerMemoryFeaturesKHR-pPeerMemoryFeatures-parameter' : 'VUID-vkGetDeviceGroupPeerMemoryFeatures-pPeerMemoryFeatures-parameter',
+ 'VUID-vkGetImageMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetImageMemoryRequirements2-device-parameter',
+ 'VUID-vkGetImageSparseMemoryRequirements2KHR-device-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-device-parameter',
+ 'VUID-vkGetImageSparseMemoryRequirements2KHR-pSparseMemoryRequirements-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-pSparseMemoryRequirements-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-physicalDevice-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-physicalDevice-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-physicalDevice-parameter',
+ 'VUID-vkGetPhysicalDeviceFeatures2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceFeatures2-physicalDevice-parameter',
+ 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-format-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-format-parameter',
+ 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-physicalDevice-parameter',
+ 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-physicalDevice-parameter',
+ 'VUID-vkGetPhysicalDeviceMemoryProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceMemoryProperties2-physicalDevice-parameter',
+ 'VUID-vkGetPhysicalDeviceProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceProperties2-physicalDevice-parameter',
+ 'VUID-vkGetPhysicalDeviceQueueFamilyProperties2KHR-pQueueFamilyProperties-parameter' : 'VUID-vkGetPhysicalDeviceQueueFamilyProperties2-pQueueFamilyProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-pProperties-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-pProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-physicalDevice-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-physicalDevice-parameter',
+ 'VUID-vkTrimCommandPoolKHR-commandPool-parameter' : 'VUID-vkTrimCommandPool-commandPool-parameter',
+ 'VUID-vkTrimCommandPoolKHR-commandPool-parent' : 'VUID-vkTrimCommandPool-commandPool-parent',
+ 'VUID-vkTrimCommandPoolKHR-device-parameter' : 'VUID-vkTrimCommandPool-device-parameter',
+ 'VUID-vkTrimCommandPoolKHR-flags-zerobitmask' : 'VUID-vkTrimCommandPool-flags-zerobitmask',
+ 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorSet-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorSet-parameter',
+ 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorUpdateTemplate-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorUpdateTemplate-parameter',
+ 'VUID-vkUpdateDescriptorSetWithTemplateKHR-descriptorUpdateTemplate-parent' : 'VUID-vkUpdateDescriptorSetWithTemplate-descriptorUpdateTemplate-parent',
+ 'VUID-vkUpdateDescriptorSetWithTemplateKHR-device-parameter' : 'VUID-vkUpdateDescriptorSetWithTemplate-device-parameter',
+ 'VUID-vkCreateDescriptorUpdateTemplateKHR-pCreateInfo-parameter' : 'VUID-vkCreateDescriptorUpdateTemplate-pCreateInfo-parameter',
+ 'VUID-vkCreateSamplerYcbcrConversionKHR-pCreateInfo-parameter' : 'VUID-vkCreateSamplerYcbcrConversion-pCreateInfo-parameter',
+ 'VUID-vkGetBufferMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetBufferMemoryRequirements2-pInfo-parameter',
+ 'VUID-vkGetBufferMemoryRequirements2KHR-pMemoryRequirements-parameter' : 'VUID-vkGetBufferMemoryRequirements2-pMemoryRequirements-parameter',
+ 'VUID-vkGetDescriptorSetLayoutSupportKHR-pCreateInfo-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-pCreateInfo-parameter',
+ 'VUID-vkGetDescriptorSetLayoutSupportKHR-pSupport-parameter' : 'VUID-vkGetDescriptorSetLayoutSupport-pSupport-parameter',
+ 'VUID-vkGetImageMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetImageMemoryRequirements2-pInfo-parameter',
+ 'VUID-vkGetImageMemoryRequirements2KHR-pMemoryRequirements-parameter' : 'VUID-vkGetImageMemoryRequirements2-pMemoryRequirements-parameter',
+ 'VUID-vkGetImageSparseMemoryRequirements2KHR-pInfo-parameter' : 'VUID-vkGetImageSparseMemoryRequirements2-pInfo-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-pExternalBufferInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-pExternalBufferInfo-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalBufferPropertiesKHR-pExternalBufferProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalBufferProperties-pExternalBufferProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-pExternalFenceInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-pExternalFenceInfo-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalFencePropertiesKHR-pExternalFenceProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalFenceProperties-pExternalFenceProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-pExternalSemaphoreInfo-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-pExternalSemaphoreInfo-parameter',
+ 'VUID-vkGetPhysicalDeviceExternalSemaphorePropertiesKHR-pExternalSemaphoreProperties-parameter' : 'VUID-vkGetPhysicalDeviceExternalSemaphoreProperties-pExternalSemaphoreProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceFeatures2KHR-pFeatures-parameter' : 'VUID-vkGetPhysicalDeviceFeatures2-pFeatures-parameter',
+ 'VUID-vkGetPhysicalDeviceFormatProperties2KHR-pFormatProperties-parameter' : 'VUID-vkGetPhysicalDeviceFormatProperties2-pFormatProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-pImageFormatInfo-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-pImageFormatInfo-parameter',
+ 'VUID-vkGetPhysicalDeviceImageFormatProperties2KHR-pImageFormatProperties-parameter' : 'VUID-vkGetPhysicalDeviceImageFormatProperties2-pImageFormatProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceMemoryProperties2KHR-pMemoryProperties-parameter' : 'VUID-vkGetPhysicalDeviceMemoryProperties2-pMemoryProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceProperties2KHR-pProperties-parameter' : 'VUID-vkGetPhysicalDeviceProperties2-pProperties-parameter',
+ 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2KHR-pFormatInfo-parameter' : 'VUID-vkGetPhysicalDeviceSparseImageFormatProperties2-pFormatInfo-parameter' }
-class ValidationHeader:
- def __init__(self, filename=header_file):
- self.filename = header_file
- self.enums = []
+def printHelp():
+ print ("Usage:")
+ print (" python vk_validation_stats.py <json_file>")
+ print (" [ -c ]")
+ print (" [ -todo ]")
+ print (" [ -vuid <vuid_name> ]")
+ print (" [ -text [ <text_out_filename>] ]")
+ print (" [ -csv [ <csv_out_filename>] ]")
+ print (" [ -html [ <html_out_filename>] ]")
+ print (" [ -verbose ]")
+ print (" [ -help ]")
+ print ("\n The vk_validation_stats script parses validation layer source files to")
+ print (" determine the set of valid usage checks and tests currently implemented,")
+ print (" and generates coverage values by comparing against the full set of valid")
+ print (" usage identifiers in the Vulkan-Headers registry file 'validusage.json'")
+ print ("\nArguments: ")
+ print (" <json-file> (required) registry file 'validusage.json'")
+ print (" -c report consistency warnings")
+ print (" -todo report unimplemented VUIDs")
+ print (" -vuid <vuid_name> report status of individual VUID <vuid_name>")
+ print (" -text [filename] output the error database text to <text_database_filename>,")
+ print (" defaults to 'validation_error_database.txt'")
+ print (" -csv [filename] output the error database in csv to <csv_database_filename>,")
+ print (" defaults to 'validation_error_database.csv'")
+ print (" -html [filename] output the error database in html to <html_database_filename>,")
+ print (" defaults to 'validation_error_database.html'")
+ print (" -verbose show your work (to stdout)")
+
+class ValidationJSON:
+ def __init__(self, filename):
+ self.filename = filename
+ self.explicit_vuids = set()
+ self.implicit_vuids = set()
+ self.all_vuids = set()
+ self.vuid_db = defaultdict(list) # Maps VUID string to list of json-data dicts
+ self.apiversion = ""
+ self.re_striptags = re.compile('<.*?>')
+ self.duplicate_vuids = set()
+
def read(self):
- """Read unique error enum header file into internal data structures"""
- grab_enums = False
- with open(self.filename, "r") as infile:
- for line in infile:
- line = line.strip()
- if 'enum UNIQUE_VALIDATION_ERROR_CODE {' in line:
- grab_enums = True
- continue
- if grab_enums:
- if 'VALIDATION_ERROR_MAX_ENUM' in line:
- grab_enums = False
- break # done
- elif 'VALIDATION_ERROR_UNDEFINED' in line:
- continue
- elif 'VALIDATION_ERROR_' in line:
- enum = line.split(' = ')[0]
- self.enums.append(enum)
- #print "Found %d error enums. First is %s and last is %s." % (len(self.enums), self.enums[0], self.enums[-1])
+ self.json_dict = {}
+ if os.path.isfile(self.filename):
+ json_file = open(self.filename, 'r')
+ self.json_dict = json.load(json_file)
+ json_file.close()
+ if len(self.json_dict) == 0:
+ print("Error: Error loading validusage.json file <%s>" % self.filename)
+ sys.exit(-1)
+ try:
+ version = self.json_dict['version info']
+ validation = self.json_dict['validation']
+ self.apiversion = version['api version']
+ except:
+ print("Error: Failure parsing validusage.json object")
+ sys.exit(-1)
+
+ # Parse vuid from json into local databases
+ for apiname in validation.keys():
+ # print("entrypoint:%s"%apiname)
+ apidict = validation[apiname]
+ for ext in apidict.keys():
+ vlist = apidict[ext]
+ for ventry in vlist:
+ vuid_string = ventry['vuid']
+ if (vuid_string[-5:-1].isdecimal()):
+ self.explicit_vuids.add(vuid_string) # explicit end in 5 numeric chars
+ vtype = 'explicit'
+ else:
+ self.implicit_vuids.add(vuid_string) # otherwise, implicit
+ vtype = 'implicit'
+ vuid_text = ventry['text']
+ stripped = re.sub(self.re_striptags, '', vuid_text) # strip tags
+ stripped = html.unescape(stripped) # unescape html literals (only partially works - because asciiDoctor?)
+ self.vuid_db[vuid_string].append({'api':apiname, 'ext':ext, 'type':vtype, 'text':stripped})
+ self.all_vuids = self.explicit_vuids | self.implicit_vuids
+ self.duplicate_vuids = set({v for v in self.vuid_db if len(self.vuid_db[v]) > 1})
class ValidationSource:
def __init__(self, source_file_list, generated_source_file_list, generated_source_directories):
self.source_files = source_file_list
self.generated_source_files = generated_source_file_list
self.generated_source_dirs = generated_source_directories
+ self.vuid_count_dict = {} # dict of vuid values to the count of how much they're used, and location of where they're used
+ self.duplicated_checks = 0
+ self.explicit_vuids = set()
+ self.implicit_vuids = set()
+ self.unassigned_vuids = set()
+ self.all_vuids = set()
if len(self.generated_source_files) > 0:
qualified_paths = []
@@ -196,14 +233,13 @@
if len(self.generated_source_files) != len(qualified_paths):
print("Error: Unable to locate one or more of the following source files in the %s directories" % (", ".join(generated_source_directories)))
print(self.generated_source_files)
- print("Skipping documentation validation test")
+ print("Failed to locate one or more codegen files in layer source code - cannot proceed.")
exit(1)
else:
self.source_files.extend(qualified_paths)
- self.enum_count_dict = {} # dict of enum values to the count of how much they're used, and location of where they're used
def parse(self):
- duplicate_checks = 0
+ prepend = None
for sf in self.source_files:
line_num = 0
with open(sf) as f:
@@ -211,241 +247,450 @@
line_num = line_num + 1
if True in [line.strip().startswith(comment) for comment in ['//', '/*']]:
continue
- # Find enums
- #if 'VALIDATION_ERROR_' in line and True not in [ignore in line for ignore in ['[VALIDATION_ERROR_', 'UNIQUE_VALIDATION_ERROR_CODE']]:
- if 'VALIDATION_ERROR_' in line:
- # Need to isolate the validation error enum
- #print("Line has check:%s" % (line))
+ # Find vuid strings
+ if prepend != None:
+ line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char
+ prepend = None
+ if any(prefix in line for prefix in vuid_prefixes):
line_list = line.split()
- enum_list = []
+
+ # A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list
+ broken_vuid = line_list[-1].strip('"')
+ if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'):
+ prepend = line
+ continue
+
+ vuid_list = []
for str in line_list:
- if 'VALIDATION_ERROR_' in str and True not in [ignore_str in str for ignore_str in ['[VALIDATION_ERROR_', 'VALIDATION_ERROR_UNDEFINED', 'UNIQUE_VALIDATION_ERROR_CODE']]:
- enum_list.append(str.strip(',);{}'))
- #break
- for enum in enum_list:
- if enum != '':
- if enum not in self.enum_count_dict:
- self.enum_count_dict[enum] = {}
- self.enum_count_dict[enum]['count'] = 1
- self.enum_count_dict[enum]['file_line'] = []
- self.enum_count_dict[enum]['file_line'].append('%s,%d' % (sf, line_num))
- #print "Found enum %s implemented for first time in file %s" % (enum, sf)
- else:
- self.enum_count_dict[enum]['count'] = self.enum_count_dict[enum]['count'] + 1
- self.enum_count_dict[enum]['file_line'].append('%s,%d' % (sf, line_num))
- #print "Found enum %s implemented for %d time in file %s" % (enum, self.enum_count_dict[enum], sf)
- duplicate_checks = duplicate_checks + 1
- #else:
- #print("Didn't find actual check in line:%s" % (line))
- #print "Found %d unique implemented checks and %d are duplicated at least once" % (len(self.enum_count_dict.keys()), duplicate_checks)
+ if any(prefix in str for prefix in vuid_prefixes):
+ vuid_list.append(str.strip(',);{}"'))
+ for vuid in vuid_list:
+ if vuid not in self.vuid_count_dict:
+ self.vuid_count_dict[vuid] = {}
+ self.vuid_count_dict[vuid]['count'] = 1
+ self.vuid_count_dict[vuid]['file_line'] = []
+ else:
+ if self.vuid_count_dict[vuid]['count'] == 1: # only count first time duplicated
+ self.duplicated_checks = self.duplicated_checks + 1
+ self.vuid_count_dict[vuid]['count'] = self.vuid_count_dict[vuid]['count'] + 1
+ self.vuid_count_dict[vuid]['file_line'].append('%s,%d' % (sf, line_num))
+ # Sort vuids by type
+ for vuid in self.vuid_count_dict.keys():
+ if (vuid.startswith('VUID-')):
+ if (vuid[-5:-1].isdecimal()):
+ self.explicit_vuids.add(vuid) # explicit end in 5 numeric chars
+ else:
+ self.implicit_vuids.add(vuid)
+ elif (vuid.startswith('UNASSIGNED-')):
+ self.unassigned_vuids.add(vuid)
+ else:
+ print("Unable to categorize VUID: %s" % vuid)
+ print("Confused while parsing VUIDs in layer source code - cannot proceed. (FIXME)")
+ exit(-1)
+ self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids
# Class to parse the validation layer test source and store testnames
-# TODO: Enhance class to detect use of unique error enums in the test
-class TestParser:
+class ValidationTests:
def __init__(self, test_file_list, test_group_name=['VkLayerTest', 'VkPositiveLayerTest', 'VkWsiEnabledLayerTest']):
self.test_files = test_file_list
- self.test_to_errors = {} # Dict where testname maps to list of error enums found in that test
self.test_trigger_txt_list = []
for tg in test_group_name:
self.test_trigger_txt_list.append('TEST_F(%s' % tg)
- #print('Test trigger test list: %s' % (self.test_trigger_txt_list))
+ self.explicit_vuids = set()
+ self.implicit_vuids = set()
+ self.unassigned_vuids = set()
+ self.all_vuids = set()
+ #self.test_to_vuids = {} # Map test name to VUIDs tested
+ self.vuid_to_tests = defaultdict(set) # Map VUIDs to set of test names where implemented
# Parse test files into internal data struct
def parse(self):
# For each test file, parse test names into set
grab_next_line = False # handle testname on separate line than wildcard
testname = ''
+ prepend = None
for test_file in self.test_files:
with open(test_file) as tf:
for line in tf:
if True in [line.strip().startswith(comment) for comment in ['//', '/*']]:
continue
- if True in [ttt in line for ttt in self.test_trigger_txt_list]:
- #print('Test wildcard in line: %s' % (line))
+ # if line ends in a broken VUID string, fix that before proceeding
+ if prepend != None:
+ line = prepend[:-2] + line.lstrip().lstrip('"') # join lines skipping CR, whitespace and trailing/leading quote char
+ prepend = None
+ if any(prefix in line for prefix in vuid_prefixes):
+ line_list = line.split()
+
+ # A VUID string that has been broken by clang will start with a vuid prefix and end with -, and will be last in the list
+ broken_vuid = line_list[-1].strip('"')
+ if any(broken_vuid.startswith(prefix) for prefix in vuid_prefixes) and broken_vuid.endswith('-'):
+ prepend = line
+ continue
+
+ if any(ttt in line for ttt in self.test_trigger_txt_list):
testname = line.split(',')[-1]
testname = testname.strip().strip(' {)')
- #print('Inserting test: "%s"' % (testname))
if ('' == testname):
grab_next_line = True
continue
- self.test_to_errors[testname] = []
+ #self.test_to_vuids[testname] = []
if grab_next_line: # test name on its own line
grab_next_line = False
testname = testname.strip().strip(' {)')
- self.test_to_errors[testname] = []
- if ' VALIDATION_ERROR_' in line:
- line_list = line.split()
+ #self.test_to_vuids[testname] = []
+ if any(prefix in line for prefix in vuid_prefixes):
+ line_list = re.split('[\s{}[\]()"]+',line)
for sub_str in line_list:
- if 'VALIDATION_ERROR_' in sub_str and True not in [ignore_str in sub_str for ignore_str in ['VALIDATION_ERROR_UNDEFINED', 'UNIQUE_VALIDATION_ERROR_CODE', 'VALIDATION_ERROR_MAX_ENUM']]:
- #print("Trying to add enums for line: %s" % ())
- #print("Adding enum %s to test %s" % (sub_str.strip(',);'), testname))
- self.test_to_errors[testname].append(sub_str.strip(',);'))
+ if any(prefix in sub_str for prefix in vuid_prefixes):
+ vuid_str = sub_str.strip(',);:"')
+ self.vuid_to_tests[vuid_str].add(testname)
+ #self.test_to_vuids[testname].append(vuid_str)
+ if (vuid_str.startswith('VUID-')):
+ if (vuid_str[-5:-1].isdecimal()):
+ self.explicit_vuids.add(vuid_str) # explicit end in 5 numeric chars
+ else:
+ self.implicit_vuids.add(vuid_str)
+ elif (vuid_str.startswith('UNASSIGNED-')):
+ self.unassigned_vuids.add(vuid_str)
+ else:
+ print("Unable to categorize VUID: %s" % vuid_str)
+ print("Confused while parsing VUIDs in test code - cannot proceed. (FIXME)")
+ exit(-1)
+ self.all_vuids = self.explicit_vuids | self.implicit_vuids | self.unassigned_vuids
-# Little helper class for coloring cmd line output
-class bcolors:
+# Class to do consistency checking
+#
+class Consistency:
+ def __init__(self, all_json, all_checks, all_tests):
+ self.valid = all_json
+ self.checks = all_checks
+ self.tests = all_tests
- def __init__(self):
- self.GREEN = '\033[0;32m'
- self.RED = '\033[0;31m'
- self.YELLOW = '\033[1;33m'
- self.ENDC = '\033[0m'
- if 'Linux' != platform.system():
- self.GREEN = ''
- self.RED = ''
- self.YELLOW = ''
- self.ENDC = ''
+ if (dealias_khr):
+ dk = set()
+ for vuid in self.checks:
+ if vuid in khr_aliases:
+ dk.add(khr_aliases[vuid])
+ else:
+ dk.add(vuid)
+ self.checks = dk
- def green(self):
- return self.GREEN
+ dk = set()
+ for vuid in self.tests:
+ if vuid in khr_aliases:
+ dk.add(khr_aliases[vuid])
+ else:
+ dk.add(vuid)
+ self.tests = dk
- def red(self):
- return self.RED
+ # Report undefined VUIDs in source code
+ def undef_vuids_in_layer_code(self):
+ undef_set = self.checks - self.valid
+ undef_set.discard('VUID-Undefined') # don't report Undefined
+ if ignore_unassigned:
+ unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')})
+ undef_set = undef_set - unassigned
+ if (len(undef_set) > 0):
+ print("\nFollowing VUIDs found in layer code are not defined in validusage.json (%d):" % len(undef_set))
+ undef = list(undef_set)
+ undef.sort()
+ for vuid in undef:
+ print(" %s" % vuid)
+ return False
+ return True
- def yellow(self):
- return self.YELLOW
+ # Report undefined VUIDs in tests
+ def undef_vuids_in_tests(self):
+ undef_set = self.tests - self.valid
+ undef_set.discard('VUID-Undefined') # don't report Undefined
+ if ignore_unassigned:
+ unassigned = set({uv for uv in undef_set if uv.startswith('UNASSIGNED-')})
+ undef_set = undef_set - unassigned
+ if (len(undef_set) > 0):
+ ok = False
+ print("\nFollowing VUIDs found in layer tests are not defined in validusage.json (%d):" % len(undef_set))
+ undef = list(undef_set)
+ undef.sort()
+ for vuid in undef:
+ print(" %s" % vuid)
+ return False
+ return True
- def endc(self):
- return self.ENDC
+ # Report vuids in tests that are not in source
+ def vuids_tested_not_checked(self):
+ undef_set = self.tests - self.checks
+ undef_set.discard('VUID-Undefined') # don't report Undefined
+ if ignore_unassigned:
+ unassigned = set()
+ for vuid in undef_set:
+ if vuid.startswith('UNASSIGNED-'):
+ unassigned.add(vuid)
+ undef_set = undef_set - unassigned
+ if (len(undef_set) > 0):
+ ok = False
+ print("\nFollowing VUIDs found in tests but are not checked in layer code (%d):" % len(undef_set))
+ undef = list(undef_set)
+ undef.sort()
+ for vuid in undef:
+ print(" %s" % vuid)
+ return False
+ return True
+
+ # TODO: Explicit checked VUIDs which have no test
+ # def explicit_vuids_checked_not_tested(self):
+
+
+# Class to output database in various flavors
+#
+class OutputDatabase:
+ def __init__(self, val_json, val_source, val_tests):
+ self.vj = val_json
+ self.vs = val_source
+ self.vt = val_tests
+
+ def dump_txt(self):
+ print("\n Dumping database to text file: %s" % txt_filename)
+ with open (txt_filename, 'w') as txt:
+ txt.write("## VUID Database\n")
+ txt.write("## Format: VUID_NAME | CHECKED | TEST | TYPE | API/STRUCT | EXTENSION | VUID_TEXT\n##\n")
+ vuid_list = list(self.vj.all_vuids)
+ vuid_list.sort()
+ for vuid in vuid_list:
+ db_list = self.vj.vuid_db[vuid]
+ db_list.sort(key=operator.itemgetter('ext')) # sort list to ease diffs of output file
+ for db_entry in db_list:
+ checked = 'N'
+ if vuid in self.vs.all_vuids:
+ checked = 'Y'
+ test = 'None'
+ if vuid in self.vt.vuid_to_tests:
+ test_list = list(self.vt.vuid_to_tests[vuid])
+ test_list.sort() # sort tests, for diff-ability
+ sep = ', '
+ test = sep.join(test_list)
+
+ txt.write("%s | %s | %s | %s | %s | %s | %s\n" % (vuid, checked, test, db_entry['type'], db_entry['api'], db_entry['ext'], db_entry['text']))
+
+ def dump_csv(self):
+ print("\n Dumping database to csv file: %s" % csv_filename)
+ with open (csv_filename, 'w', newline='') as csvfile:
+ cw = csv.writer(csvfile)
+ cw.writerow(['VUID_NAME','CHECKED','TEST','TYPE','API/STRUCT','EXTENSION','VUID_TEXT'])
+ vuid_list = list(self.vj.all_vuids)
+ vuid_list.sort()
+ for vuid in vuid_list:
+ for db_entry in self.vj.vuid_db[vuid]:
+ row = [vuid]
+ if vuid in self.vs.all_vuids:
+ row.append('Y')
+ else:
+ row.append('N')
+ test = 'None'
+ if vuid in self.vt.vuid_to_tests:
+ sep = ', '
+ test = sep.join(self.vt.vuid_to_tests[vuid])
+ row.append(test)
+ row.append(db_entry['type'])
+ row.append(db_entry['api'])
+ row.append(db_entry['ext'])
+ row.append(db_entry['text'])
+ cw.writerow(row)
+
+ def dump_html(self):
+ print("\n Dumping database to html file: %s" % html_filename)
+ preamble = '<!DOCTYPE html>\n<html>\n<head>\n<style>\ntable, th, td {\n border: 1px solid black;\n border-collapse: collapse; \n}\n</style>\n<body>\n<h2>Valid Usage Database</h2>\n<font size="2" face="Arial">\n<table style="width:100%">\n'
+ headers = '<tr><th>VUID NAME</th><th>CHECKED</th><th>TEST</th><th>TYPE</th><th>API/STRUCT</th><th>EXTENSION</th><th>VUID TEXT</th></tr>\n'
+ with open (html_filename, 'w') as hfile:
+ hfile.write(preamble)
+ hfile.write(headers)
+ vuid_list = list(self.vj.all_vuids)
+ vuid_list.sort()
+ for vuid in vuid_list:
+ for db_entry in self.vj.vuid_db[vuid]:
+ hfile.write('<tr><th>%s</th>' % vuid)
+ checked = '<span style="color:red;">N</span>'
+ if vuid in self.vs.all_vuids:
+ checked = '<span style="color:limegreen;">Y</span>'
+ hfile.write('<th>%s</th>' % checked)
+ test = 'None'
+ if vuid in self.vt.vuid_to_tests:
+ sep = ', '
+ test = sep.join(self.vt.vuid_to_tests[vuid])
+ hfile.write('<th>%s</th>' % test)
+ hfile.write('<th>%s</th>' % db_entry['type'])
+ hfile.write('<th>%s</th>' % db_entry['api'])
+ hfile.write('<th>%s</th>' % db_entry['ext'])
+ hfile.write('<th>%s</th></tr>\n' % db_entry['text'])
+ hfile.write('</table>\n</body>\n</html>\n')
def main(argv):
+ global verbose_mode
+ global txt_filename
+ global csv_filename
+ global html_filename
+
+ run_consistency = False
+ report_unimplemented = False
+ get_vuid_status = ''
+ txt_out = False
+ csv_out = False
+ html_out = False
+
+ if (1 > len(argv)):
+ printHelp()
+ sys.exit()
+
+ # Parse script args
+ json_filename = argv[0]
+ i = 1
+ while (i < len(argv)):
+ arg = argv[i]
+ i = i + 1
+ if (arg == '-c'):
+ run_consistency = True
+ elif (arg == '-vuid'):
+ get_vuid_status = argv[i]
+ i = i + 1
+ elif (arg == '-todo'):
+ report_unimplemented = True
+ elif (arg == '-txt'):
+ txt_out = True
+ # Set filename if supplied, else use default
+ if i < len(argv) and not argv[i].startswith('-'):
+ txt_filename = argv[i]
+ i = i + 1
+ elif (arg == '-csv'):
+ csv_out = True
+ # Set filename if supplied, else use default
+ if i < len(argv) and not argv[i].startswith('-'):
+ csv_filename = argv[i]
+ i = i + 1
+ elif (arg == '-html'):
+ html_out = True
+ # Set filename if supplied, else use default
+ if i < len(argv) and not argv[i].startswith('-'):
+ html_filename = argv[i]
+ i = i + 1
+ elif (arg in ['-verbose']):
+ verbose_mode = True
+ elif (arg in ['-help', '-h']):
+ printHelp()
+ sys.exit()
+ else:
+ print("Unrecognized argument: %s\n" % arg)
+ printHelp()
+ sys.exit()
+
result = 0 # Non-zero result indicates an error case
- verbose_mode = 'verbose' in sys.argv
- # parse db
- val_db = ValidationDatabase()
- val_db.read()
- # parse header
- val_header = ValidationHeader()
- val_header.read()
- # Create parser for layer files
+
+ # Parse validusage json
+ val_json = ValidationJSON(json_filename)
+ val_json.read()
+ exp_json = len(val_json.explicit_vuids)
+ imp_json = len(val_json.implicit_vuids)
+ all_json = len(val_json.all_vuids)
+ if verbose_mode:
+ print("Found %d unique error vuids in validusage.json file." % all_json)
+ print(" %d explicit" % exp_json)
+ print(" %d implicit" % imp_json)
+ if len(val_json.duplicate_vuids) > 0:
+ print("%d VUIDs appear in validusage.json more than once." % len(val_json.duplicate_vuids))
+ for vuid in val_json.duplicate_vuids:
+ print(" %s" % vuid)
+ for ext in val_json.vuid_db[vuid]:
+ print(" with extension: %s" % ext['ext'])
+
+ # Parse layer source files
val_source = ValidationSource(layer_source_files, generated_layer_source_files, generated_layer_source_directories)
val_source.parse()
+ exp_checks = len(val_source.explicit_vuids)
+ imp_checks = len(val_source.implicit_vuids)
+ all_checks = len(val_source.vuid_count_dict.keys())
+ if verbose_mode:
+ print("Found %d unique vuid checks in layer source code." % all_checks)
+ print(" %d explicit" % exp_checks)
+ print(" %d implicit" % imp_checks)
+ print(" %d unassigned" % len(val_source.unassigned_vuids))
+ print(" %d checks are implemented more that once" % val_source.duplicated_checks)
+
# Parse test files
- test_parser = TestParser([test_file, ])
- test_parser.parse()
-
- # Process stats - Just doing this inline in main, could make a fancy class to handle
- # all the processing of data and then get results from that
- txt_color = bcolors()
+ val_tests = ValidationTests([test_file, ])
+ val_tests.parse()
+ exp_tests = len(val_tests.explicit_vuids)
+ imp_tests = len(val_tests.implicit_vuids)
+ all_tests = len(val_tests.all_vuids)
if verbose_mode:
- print("Validation Statistics")
- else:
- print("Validation/Documentation Consistency Test")
- # First give number of checks in db & header and report any discrepancies
- db_enums = len(val_db.db_dict.keys())
- hdr_enums = len(val_header.enums)
- if verbose_mode:
- print(" Database file includes %d unique checks" % (db_enums))
- print(" Header file declares %d unique checks" % (hdr_enums))
+ print("Found %d unique error vuids in test file %s." % (all_tests, test_file))
+ print(" %d explicit" % exp_tests)
+ print(" %d implicit" % imp_tests)
+ print(" %d unassigned" % len(val_tests.unassigned_vuids))
- # Report any checks that have an invalid check_implemented flag
- if len(val_db.db_invalid_implemented) > 0:
- result = 1
- print(txt_color.red() + "The following checks have an invalid check_implemented flag (must be 'Y' or 'N'):" + txt_color.endc())
- for invalid_imp_enum in val_db.db_invalid_implemented:
- check_implemented = val_db.db_dict[invalid_imp_enum]['check_implemented']
- print(txt_color.red() + " %s has check_implemented flag '%s'" % (invalid_imp_enum, check_implemented) + txt_color.endc())
+ # Process stats
+ print("\nValidation Statistics (using validusage.json version %s)" % val_json.apiversion)
+ print(" VUIDs defined in JSON file: %04d explicit, %04d implicit, %04d total." % (exp_json, imp_json, all_json))
+ print(" VUIDs checked in layer code: %04d explicit, %04d implicit, %04d total." % (exp_checks, imp_checks, all_checks))
+ print(" VUIDs tested in layer tests: %04d explicit, %04d implicit, %04d total." % (exp_tests, imp_tests, all_tests))
+
+ print("\nVUID check coverage")
+ print(" Explicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * exp_checks / exp_json), exp_checks, exp_json))
+ print(" Implicit VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * imp_checks / imp_json), imp_checks, imp_json))
+ print(" Overall VUIDs checked: %.1f%% (%d checked vs %d defined)" % ((100.0 * all_checks / all_json), all_checks, all_json))
- # Report details about how well the Database and Header are synchronized.
- tmp_db_dict = val_db.db_dict
- db_missing = []
- for enum in val_header.enums:
- if not tmp_db_dict.pop(enum, False):
- db_missing.append(enum)
- if db_enums == hdr_enums and len(db_missing) == 0 and len(tmp_db_dict.keys()) == 0:
- if verbose_mode:
- print(txt_color.green() + " Database and Header match, GREAT!" + txt_color.endc())
- else:
- print(txt_color.red() + " Uh oh, Database doesn't match Header :(" + txt_color.endc())
- result = 1
- if len(db_missing) != 0:
- print(txt_color.red() + " The following checks are in header but missing from database:" + txt_color.endc())
- for missing_enum in db_missing:
- print(txt_color.red() + " %s" % (missing_enum) + txt_color.endc())
- if len(tmp_db_dict.keys()) != 0:
- print(txt_color.red() + " The following checks are in database but haven't been declared in the header:" + txt_color.endc())
- for extra_enum in tmp_db_dict:
- print(txt_color.red() + " %s" % (extra_enum) + txt_color.endc())
+ print("\nVUID test coverage")
+ print(" Explicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * exp_tests / exp_checks), exp_tests, exp_checks))
+ print(" Implicit VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * imp_tests / imp_checks), imp_tests, imp_checks))
+ print(" Overall VUIDs tested: %.1f%% (%d tested vs %d checks)" % ((100.0 * all_tests / all_checks), all_tests, all_checks))
- # Report out claimed implemented checks vs. found actual implemented checks
- imp_not_found = [] # Checks claimed to implemented in DB file but no source found
- imp_not_claimed = [] # Checks found implemented but not claimed to be in DB
- multiple_uses = False # Flag if any enums are used multiple times
- for db_imp in val_db.db_implemented_enums:
- if db_imp not in val_source.enum_count_dict:
- imp_not_found.append(db_imp)
- for src_enum in val_source.enum_count_dict:
- if val_source.enum_count_dict[src_enum]['count'] > 1 and src_enum not in duplicate_exceptions:
- multiple_uses = True
- if src_enum not in val_db.db_implemented_enums:
- imp_not_claimed.append(src_enum)
- if verbose_mode:
- print(" Database file claims that %d checks (%s) are implemented in source." % (len(val_db.db_implemented_enums), "{0:.0f}%".format(float(len(val_db.db_implemented_enums))/db_enums * 100)))
-
- if len(val_db.db_unimplemented_implicit) > 0 and verbose_mode:
- print(" Database file claims %d implicit checks (%s) that are not implemented." % (len(val_db.db_unimplemented_implicit), "{0:.0f}%".format(float(len(val_db.db_unimplemented_implicit))/db_enums * 100)))
- total_checks = len(val_db.db_implemented_enums) + len(val_db.db_unimplemented_implicit)
- print(" If all implicit checks are handled by parameter validation this is a total of %d (%s) checks covered." % (total_checks, "{0:.0f}%".format(float(total_checks)/db_enums * 100)))
- if len(imp_not_found) == 0 and len(imp_not_claimed) == 0:
- if verbose_mode:
- print(txt_color.green() + " All claimed Database implemented checks have been found in source, and no source checks aren't claimed in Database, GREAT!" + txt_color.endc())
- else:
- result = 1
- print(txt_color.red() + " Uh oh, Database claimed implemented don't match Source :(" + txt_color.endc())
- if len(imp_not_found) != 0:
- print(txt_color.red() + " The following %d checks are claimed to be implemented in Database, but weren't found in source:" % (len(imp_not_found)) + txt_color.endc())
- for not_imp_enum in imp_not_found:
- print(txt_color.red() + " %s" % (not_imp_enum) + txt_color.endc())
- if len(imp_not_claimed) != 0:
- print(txt_color.red() + " The following checks are implemented in source, but not claimed to be in Database:" + txt_color.endc())
- for imp_enum in imp_not_claimed:
- print(txt_color.red() + " %s" % (imp_enum) + txt_color.endc())
-
- if multiple_uses and verbose_mode:
- print(txt_color.yellow() + " Note that some checks are used multiple times. These may be good candidates for new valid usage spec language." + txt_color.endc())
- print(txt_color.yellow() + " Here is a list of each check used multiple times with its number of uses:" + txt_color.endc())
- for enum in val_source.enum_count_dict:
- if val_source.enum_count_dict[enum]['count'] > 1 and enum not in duplicate_exceptions:
- print(txt_color.yellow() + " %s: %d uses in file,line:" % (enum, val_source.enum_count_dict[enum]['count']) + txt_color.endc())
- for file_line in val_source.enum_count_dict[enum]['file_line']:
- print(txt_color.yellow() + " \t%s" % (file_line) + txt_color.endc())
-
- # Now check that tests claimed to be implemented are actual test names
- bad_testnames = []
- tests_missing_enum = {} # Report tests that don't use validation error enum to check for error case
- for enum in val_db.db_enum_to_tests:
- for testname in val_db.db_enum_to_tests[enum]:
- if testname not in test_parser.test_to_errors:
- bad_testnames.append(testname)
+ # Report status of a single VUID
+ if len(get_vuid_status) > 1:
+ print("\n\nChecking status of <%s>" % get_vuid_status);
+ if get_vuid_status not in val_json.all_vuids:
+ print(' Not a valid VUID string.')
+ else:
+ if get_vuid_status in val_source.explicit_vuids:
+ print(' Implemented!')
+ line_list = val_source.vuid_count_dict[get_vuid_status]['file_line']
+ for line in line_list:
+ print(' => %s' % line)
else:
- enum_found = False
- for test_enum in test_parser.test_to_errors[testname]:
- if test_enum == enum:
- #print("Found test that correctly checks for enum: %s" % (enum))
- enum_found = True
- if not enum_found:
- #print("Test %s is not using enum %s to check for error" % (testname, enum))
- if testname not in tests_missing_enum:
- tests_missing_enum[testname] = []
- tests_missing_enum[testname].append(enum)
- if tests_missing_enum and verbose_mode:
- print(txt_color.yellow() + " \nThe following tests do not use their reported enums to check for the validation error. You may want to update these to pass the expected enum to SetDesiredFailureMsg:" + txt_color.endc())
- for testname in tests_missing_enum:
- print(txt_color.yellow() + " Testname %s does not explicitly check for these ids:" % (testname) + txt_color.endc())
- for enum in tests_missing_enum[testname]:
- print(txt_color.yellow() + " %s" % (enum) + txt_color.endc())
+ print(' Not implemented.')
+ if get_vuid_status in val_tests.explicit_vuids:
+ print(' Has a test!')
+ test_list = val_tests.vuid_to_tests[get_vuid_status]
+ for test in test_list:
+ print(' => %s' % test)
+ else:
+ print(' Not tested.')
- # TODO : Go through all enums found in the test file and make sure they're correctly documented in the database file
- if verbose_mode:
- print(" Database file claims that %d checks have tests written." % len(val_db.db_enum_to_tests))
- if len(bad_testnames) == 0:
- if verbose_mode:
- print(txt_color.green() + " All claimed tests have valid names. That's good!" + txt_color.endc())
- else:
- print(txt_color.red() + " The following testnames in Database appear to be invalid:")
- result = 1
- for bt in bad_testnames:
- print(txt_color.red() + " %s" % (bt) + txt_color.endc())
+ # Report unimplemented explicit VUIDs
+ if report_unimplemented:
+ unim_explicit = val_json.explicit_vuids - val_source.explicit_vuids
+ print("\n\n%d explicit VUID checks remain unimplemented:" % len(unim_explicit))
+ ulist = list(unim_explicit)
+ ulist.sort()
+ for vuid in ulist:
+ print(" => %s" % vuid)
+
+ # Consistency tests
+ if run_consistency:
+ print("\n\nRunning consistency tests...")
+ con = Consistency(val_json.all_vuids, val_source.all_vuids, val_tests.all_vuids)
+ ok = con.undef_vuids_in_layer_code()
+ ok &= con.undef_vuids_in_tests()
+ ok &= con.vuids_tested_not_checked()
+
+ if ok:
+ print(" OK! No inconsistencies found.")
+
+ # Output database in requested format(s)
+ db_out = OutputDatabase(val_json, val_source, val_tests)
+ if txt_out:
+ db_out.dump_txt()
+ if csv_out:
+ db_out.dump_csv()
+ if html_out:
+ db_out.dump_html()
return result