scripts: Move layer scripts into scripts directory
Simplifies sharing code among python modules, keeps everything together.
- Moved python scripts out of layers dir into scripts dir
- Updated relative paths for input/output files in layers dir
- Updated doc validation tests to use correct paths
Change-Id: I57e6ad6f551d4fb652583315fe3dccb23b187db1
diff --git a/scripts/spec.py b/scripts/spec.py
new file mode 100644
index 0000000..de34632
--- /dev/null
+++ b/scripts/spec.py
@@ -0,0 +1,357 @@
+#!/usr/bin/python -i
+
+import sys
+try:
+ import urllib.request as urllib2
+except ImportError:
+ import urllib2
+from bs4 import BeautifulSoup
+import json
+import vuid_mapping
+
+#############################
+# spec.py script
+#
+# Overview - this script is intended to generate validation error codes and message strings from the json spec file
+# that contains all of the valid usage statements. In addition to generating the header file, it provides a number of
+# corrollary services to aid in generating/updating the header.
+#
+# Ideal flow - Pull the valid usage text and IDs from the spec json, pull the IDs from the validation error database,
+# then update the database with any new IDs from the json file and generate new database and header file.
+#
+# TODO:
+# 1. When VUs go away (in error DB, but not in json) need to report them and remove from DB as deleted
+#
+#############################
+
+
+out_filename = "../layers/vk_validation_error_messages.h" # can override w/ '-out <filename>' option
+db_filename = "../layers/vk_validation_error_database.txt" # can override w/ '-gendb <filename>' option
+json_filename = None # con pass in w/ '-json <filename> option
+gen_db = False # set to True when '-gendb <filename>' option provided
+json_compare = False # compare existing DB to json file input
+json_url = "https://www.khronos.org/registry/vulkan/specs/1.0-extensions/validation/validusage.json"
+read_json = False
+# This is the root spec link that is used in error messages to point users to spec sections
+#old_spec_url = "https://www.khronos.org/registry/vulkan/specs/1.0/xhtml/vkspec.html"
+spec_url = "https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html"
+core_url = "https://www.khronos.org/registry/vulkan/specs/1.0/html/vkspec.html"
+ext_url = "https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html"
+# After the custom validation error message, this is the prefix for the standard message that includes the
+# spec valid usage language as well as the link to nearest section of spec to that language
+error_msg_prefix = "The spec valid usage text states "
+validation_error_enum_name = "VALIDATION_ERROR_"
+
+def printHelp():
+ print ("Usage: python spec.py [-out <headerfile.h>] [-gendb <databasefile.txt>] [-update] [-json <json_file>] [-help]")
+ print ("\n Default script behavior is to parse the specfile and generate a header of unique error enums and corresponding error messages based on the specfile.\n")
+ print (" Default specfile is from online at %s" % (spec_url))
+ print (" Default headerfile is %s" % (out_filename))
+ print (" Default databasefile is %s" % (db_filename))
+ print ("\nIf '-gendb' option is specified then a database file is generated to default file or <databasefile.txt> if supplied. The database file stores")
+ print (" the list of enums and their error messages.")
+ print ("\nIf '-update' option is specified this triggers the master flow to automate updating header and database files using default db file as baseline")
+ print (" and online spec file as the latest. The default header and database files will be updated in-place for review and commit to the git repo.")
+ print ("\nIf '-json' option is used trigger the script to load in data from a json file.")
+ print ("\nIf '-json-file' option is it will point to a local json file, else '%s' is used from the web." % (json_url))
+
+def get8digithex(dec_num):
+ """Convert a decimal # into an 8-digit hex"""
+ if dec_num > 4294967295:
+ print ("ERROR: Decimal # %d can't be represented in 8 hex digits" % (dec_num))
+ sys.exit()
+ hex_num = hex(dec_num)
+ return hex_num[2:].zfill(8)
+
+class Specification:
+ def __init__(self):
+ self.tree = None
+ self.error_db_dict = {} # dict of previous error values read in from database file
+ self.delimiter = '~^~' # delimiter for db file
+ # Global dicts used for tracking spec updates from old to new VUs
+ self.orig_no_link_msg_dict = {} # Pair of API,Original msg w/o spec link to ID list mapping
+ self.orig_core_msg_dict = {} # Pair of API,Original core msg (no link or section) to ID list mapping
+ self.last_mapped_id = -10 # start as negative so we don't hit an accidental sequence
+ self.orig_test_imp_enums = set() # Track old enums w/ tests and/or implementation to flag any that aren't carried fwd
+ # Dict of data from json DB
+ # Key is API,<short_msg> which leads to dict w/ following values
+ # 'ext' -> <core|<ext_name>>
+ # 'string_vuid' -> <string_vuid>
+ # 'number_vuid' -> <numerical_vuid>
+ self.json_db = {}
+ self.json_missing = 0
+ self.struct_to_func_map = {} # Map structs to the API func that they fall under in the spec
+ self.duplicate_json_key_count = 0
+ self.copyright = """/* THIS FILE IS GENERATED. DO NOT EDIT. */
+
+/*
+ * Vulkan
+ *
+ * Copyright (c) 2016 Google Inc.
+ * Copyright (c) 2016 LunarG, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * Author: Tobin Ehlis <tobine@google.com>
+ */"""
+
+ def readJSON(self):
+ """Read in JSON file"""
+ if json_filename is not None:
+ with open(json_filename) as jsf:
+ self.json_data = json.load(jsf, encoding='utf-8')
+ else:
+ response = urllib2.urlopen(json_url).read().decode('utf-8')
+ self.json_data = json.loads(response)
+
+ def parseJSON(self):
+ """Parse JSON VUIDs into data struct"""
+ # Format of JSON file is:
+ # "API": { "core|EXT": [ {"vuid": "<id>", "text": "<VU txt>"}]},
+ # "VK_KHX_external_memory" & "VK_KHX_device_group" - extension case (vs. "core")
+ for top_level in sorted(self.json_data):
+ if "validation" == top_level:
+ for api in sorted(self.json_data[top_level]):
+ for ext in sorted(self.json_data[top_level][api]):
+ for vu_txt_dict in self.json_data[top_level][api][ext]:
+ print ("Looking at dict for api:ext entry %s:%s" % (api, ext))
+ vuid = vu_txt_dict['vuid']
+ vutxt = vu_txt_dict['text']
+ #print ("%s:%s:%s:%s" % (api, ext, vuid, vutxt))
+ #print ("VUTXT orig:%s" % (vutxt))
+ just_txt = BeautifulSoup(vutxt, 'html.parser')
+ #print ("VUTXT only:%s" % (just_txt.get_text()))
+ num_vuid = vuid_mapping.convertVUID(vuid)
+ self.json_db[vuid] = {}
+ self.json_db[vuid]['ext'] = ext
+ self.json_db[vuid]['number_vuid'] = num_vuid
+ self.json_db[vuid]['struct_func'] = api
+ just_txt = just_txt.get_text().strip()
+ unicode_map = {
+ u"\u2019" : "'",
+ u"\u2192" : "->",
+ }
+ for um in unicode_map:
+ just_txt = just_txt.replace(um, unicode_map[um])
+ self.json_db[vuid]['vu_txt'] = just_txt.replace("\\", "")
+ print ("Spec vu txt:%s" % (self.json_db[vuid]['vu_txt']))
+ #sys.exit()
+
+ def compareJSON(self):
+ """Compare parsed json file with existing data read in from DB file"""
+ json_db_set = set()
+ for vuid in self.json_db: # pull entries out and see which fields we're missing from error_db
+ json_db_set.add(vuid)
+ for enum in self.error_db_dict:
+ vuid_string = self.error_db_dict[enum]['vuid_string']
+ if vuid_string not in self.json_db:
+ #print ("Full string for %s is:%s" % (enum, full_error_string))
+ print ("WARN: Couldn't find vuid_string in json db:%s" % (vuid_string))
+ self.json_missing = self.json_missing + 1
+ self.error_db_dict[enum]['ext'] = 'core'
+ # TODO: Currently GL843 tracks 2 VUs that are missing from json incorrectly
+ # Fix will land in 1.0.51 spec. After that we should take some alternative
+ # action here to indicate that VUs have gone away.
+ # Can have a removed_enums set that we add to and report to user
+ #sys.exit()
+ else:
+ json_db_set.remove(vuid_string)
+ self.error_db_dict[enum]['ext'] = self.json_db[vuid_string]['ext']
+ if 'core' == self.json_db[vuid_string]['ext'] or '!' in self.json_db[vuid_string]['ext']:
+ spec_link = "%s#%s" % (core_url, vuid_string)
+ else:
+ spec_link = "%s#%s" % (ext_url, vuid_string)
+ self.error_db_dict[enum]['error_msg'] = "%s'%s' (%s)" % (error_msg_prefix, self.json_db[vuid_string]['vu_txt'], spec_link)
+ print ("Updated error_db error_msg:%s" % (self.error_db_dict[enum]['error_msg']))
+ #sys.exit()
+ print ("These json DB entries are not in error DB:")
+ for extra_vuid in json_db_set:
+ print ("\t%s" % (extra_vuid))
+ # Add these missing entries into the error_db
+ # Create link into core or ext spec as needed
+ if 'core' == self.json_db[extra_vuid]['ext'] or '!' in self.json_db[extra_vuid]['ext']:
+ spec_link = "%s#%s" % (core_url, extra_vuid)
+ else:
+ spec_link = "%s#%s" % (ext_url, extra_vuid)
+ error_enum = "%s%s" % (validation_error_enum_name, get8digithex(self.json_db[extra_vuid]['number_vuid']))
+ self.error_db_dict[error_enum] = {}
+ self.error_db_dict[error_enum]['check_implemented'] = 'N'
+ self.error_db_dict[error_enum]['testname'] = 'None'
+ self.error_db_dict[error_enum]['api'] = self.json_db[extra_vuid]['struct_func']
+ self.error_db_dict[error_enum]['vuid_string'] = extra_vuid
+ self.error_db_dict[error_enum]['error_msg'] = "%s'%s' (%s)" % (error_msg_prefix, self.json_db[extra_vuid]['vu_txt'], spec_link)
+ self.error_db_dict[error_enum]['note'] = ''
+ self.error_db_dict[error_enum]['ext'] = self.json_db[extra_vuid]['ext']
+ implicit = False
+ last_segment = extra_vuid.split("-")[-1]
+ if last_segment in vuid_mapping.implicit_type_map:
+ implicit = True
+ elif not last_segment.isdigit(): # Explicit ids should only have digits in last segment
+ print ("ERROR: Found last segment of val error ID that isn't in implicit map and doesn't have numbers in last segment: %s" % (last_segment))
+ sys.exit()
+ self.error_db_dict[error_enum]['implicit'] = implicit
+
+ def genHeader(self, header_file):
+ """Generate a header file based on the contents of a parsed spec"""
+ print ("Generating header %s..." % (header_file))
+ file_contents = []
+ file_contents.append(self.copyright)
+ file_contents.append('\n#pragma once')
+ file_contents.append('\n// Disable auto-formatting for generated file')
+ file_contents.append('// clang-format off')
+ file_contents.append('\n#include <unordered_map>')
+ file_contents.append('\n// enum values for unique validation error codes')
+ file_contents.append('// Corresponding validation error message for each enum is given in the mapping table below')
+ file_contents.append('// When a given error occurs, these enum values should be passed to the as the messageCode')
+ file_contents.append('// parameter to the PFN_vkDebugReportCallbackEXT function')
+ enum_decl = ['enum UNIQUE_VALIDATION_ERROR_CODE {\n VALIDATION_ERROR_UNDEFINED = -1,']
+ error_string_map = ['static std::unordered_map<int, char const *const> validation_error_map{']
+ enum_value = 0
+ max_enum_val = 0
+ for enum in sorted(self.error_db_dict):
+ #print ("Header enum is %s" % (enum))
+ # TMP: Use updated value
+ vuid_str = self.error_db_dict[enum]['vuid_string']
+ if vuid_str in self.json_db:
+ enum_value = self.json_db[vuid_str]['number_vuid']
+ else:
+ enum_value = vuid_mapping.convertVUID(vuid_str)
+ new_enum = "%s%s" % (validation_error_enum_name, get8digithex(enum_value))
+ enum_decl.append(' %s = 0x%s,' % (new_enum, get8digithex(enum_value)))
+ error_string_map.append(' {%s, "%s"},' % (new_enum, self.error_db_dict[enum]['error_msg']))
+ max_enum_val = max(max_enum_val, enum_value)
+ enum_decl.append(' %sMAX_ENUM = %d,' % (validation_error_enum_name, max_enum_val + 1))
+ enum_decl.append('};')
+ error_string_map.append('};\n')
+ file_contents.extend(enum_decl)
+ file_contents.append('\n// Mapping from unique validation error enum to the corresponding error message')
+ file_contents.append('// The error message should be appended to the end of a custom error message that is passed')
+ file_contents.append('// as the pMessage parameter to the PFN_vkDebugReportCallbackEXT function')
+ file_contents.extend(error_string_map)
+ #print ("File contents: %s" % (file_contents))
+ with open(header_file, "w") as outfile:
+ outfile.write("\n".join(file_contents))
+ def genDB(self, db_file):
+ """Generate a database of check_enum, check_coded?, testname, API, VUID_string, core|ext, error_string, notes"""
+ db_lines = []
+ # Write header for database file
+ db_lines.append("# This is a database file with validation error check information")
+ db_lines.append("# Comments are denoted with '#' char")
+ db_lines.append("# The format of the lines is:")
+ db_lines.append("# <error_enum>%s<check_implemented>%s<testname>%s<api>%s<vuid_string>%s<core|ext>%s<errormsg>%s<note>" % (self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter, self.delimiter))
+ db_lines.append("# error_enum: Unique error enum for this check of format %s<uniqueid>" % validation_error_enum_name)
+ db_lines.append("# check_implemented: 'Y' if check has been implemented in layers, or 'N' for not implemented")
+ db_lines.append("# testname: Name of validation test for this check, 'Unknown' for unknown, 'None' if not implemented, or 'NotTestable' if cannot be implemented")
+ db_lines.append("# api: Vulkan API function that this check is related to")
+ db_lines.append("# vuid_string: Unique string to identify this check")
+ db_lines.append("# core|ext: Either 'core' for core spec or some extension string that indicates the extension required for this VU to be relevant")
+ db_lines.append("# errormsg: The unique error message for this check that includes spec language and link")
+ db_lines.append("# note: Free txt field with any custom notes related to the check in question")
+ for enum in sorted(self.error_db_dict):
+ print ("Gen DB for enum %s" % (enum))
+ implicit = self.error_db_dict[enum]['implicit']
+ implemented = self.error_db_dict[enum]['check_implemented']
+ testname = self.error_db_dict[enum]['testname']
+ note = self.error_db_dict[enum]['note']
+ core_ext = self.error_db_dict[enum]['ext']
+ self.error_db_dict[enum]['vuid_string'] = self.error_db_dict[enum]['vuid_string']
+ if implicit and 'implicit' not in note: # add implicit note
+ if '' != note:
+ note = "implicit, %s" % (note)
+ else:
+ note = "implicit"
+ db_lines.append("%s%s%s%s%s%s%s%s%s%s%s%s%s%s%s" % (enum, self.delimiter, implemented, self.delimiter, testname, self.delimiter, self.error_db_dict[enum]['api'], self.delimiter, self.error_db_dict[enum]['vuid_string'], self.delimiter, core_ext, self.delimiter, self.error_db_dict[enum]['error_msg'], self.delimiter, note))
+ db_lines.append("\n") # newline at end of file
+ print ("Generating database file %s" % (db_file))
+ with open(db_file, "w") as outfile:
+ outfile.write("\n".join(db_lines))
+ def readDB(self, db_file):
+ """Read a db file into a dict, refer to genDB function above for format of each line"""
+ with open(db_file, "r") as infile:
+ for line in infile:
+ line = line.strip()
+ if line.startswith('#') or '' == line:
+ continue
+ db_line = line.split(self.delimiter)
+ if len(db_line) != 8:
+ print ("ERROR: Bad database line doesn't have 8 elements: %s" % (line))
+ error_enum = db_line[0]
+ implemented = db_line[1]
+ testname = db_line[2]
+ api = db_line[3]
+ vuid_str = db_line[4]
+ core_ext = db_line[5]
+ error_str = db_line[6]
+ note = db_line[7]
+ # Also read complete database contents into our class var for later use
+ self.error_db_dict[error_enum] = {}
+ self.error_db_dict[error_enum]['check_implemented'] = implemented
+ self.error_db_dict[error_enum]['testname'] = testname
+ self.error_db_dict[error_enum]['api'] = api
+ self.error_db_dict[error_enum]['vuid_string'] = vuid_str
+ self.error_db_dict[error_enum]['ext'] = core_ext
+ self.error_db_dict[error_enum]['error_msg'] = error_str
+ self.error_db_dict[error_enum]['note'] = note
+ implicit = False
+ last_segment = vuid_str.split("-")[-1]
+ if last_segment in vuid_mapping.implicit_type_map:
+ implicit = True
+ elif not last_segment.isdigit(): # Explicit ids should only have digits in last segment
+ print ("ERROR: Found last segment of val error ID that isn't in implicit map and doesn't have numbers in last segment: %s" % (last_segment))
+ sys.exit()
+ self.error_db_dict[error_enum]['implicit'] = implicit
+if __name__ == "__main__":
+ i = 1
+ use_online = True # Attempt to grab spec from online by default
+ while (i < len(sys.argv)):
+ arg = sys.argv[i]
+ i = i + 1
+ if (arg == '-json-file'):
+ json_filename = sys.argv[i]
+ i = i + 1
+ elif (arg == '-json'):
+ read_json = True
+ elif (arg == '-json-compare'):
+ json_compare = True
+ elif (arg == '-out'):
+ out_filename = sys.argv[i]
+ i = i + 1
+ elif (arg == '-gendb'):
+ gen_db = True
+ # Set filename if supplied, else use default
+ if i < len(sys.argv) and not sys.argv[i].startswith('-'):
+ db_filename = sys.argv[i]
+ i = i + 1
+ elif (arg == '-update'):
+ read_json = True
+ json_compare = True
+ gen_db = True
+ elif (arg in ['-help', '-h']):
+ printHelp()
+ sys.exit()
+ spec = Specification()
+ if read_json:
+ spec.readJSON()
+ spec.parseJSON()
+ #sys.exit()
+ if (json_compare):
+ # Read in current spec info from db file
+ (orig_err_msg_dict) = spec.readDB(db_filename)
+ spec.compareJSON()
+ print ("Found %d missing db entries in json db" % (spec.json_missing))
+ print ("Found %d duplicate json entries" % (spec.duplicate_json_key_count))
+ spec.genDB(db_filename)
+ if (gen_db):
+ spec.genDB(db_filename)
+ print ("Writing out file (-out) to '%s'" % (out_filename))
+ spec.genHeader(out_filename)