Tobin Ehlis | d22e3ff | 2015-03-27 11:47:10 -0600 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | # |
Courtney Goeltzenleuchter | d8e229c | 2015-04-08 15:36:08 -0600 | [diff] [blame] | 3 | # VK |
Tobin Ehlis | d22e3ff | 2015-03-27 11:47:10 -0600 | [diff] [blame] | 4 | # |
| 5 | # Copyright (C) 2014 LunarG, Inc. |
| 6 | # |
| 7 | # Permission is hereby granted, free of charge, to any person obtaining a |
| 8 | # copy of this software and associated documentation files (the "Software"), |
| 9 | # to deal in the Software without restriction, including without limitation |
| 10 | # the rights to use, copy, modify, merge, publish, distribute, sublicense, |
| 11 | # and/or sell copies of the Software, and to permit persons to whom the |
| 12 | # Software is furnished to do so, subject to the following conditions: |
| 13 | # |
| 14 | # The above copyright notice and this permission notice shall be included |
| 15 | # in all copies or substantial portions of the Software. |
| 16 | # |
| 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL |
| 20 | # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
| 22 | # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER |
| 23 | # DEALINGS IN THE SOFTWARE. |
| 24 | import argparse |
| 25 | import subprocess |
| 26 | import sys |
| 27 | |
| 28 | # layer_test_suite.py overview |
| 29 | # This script runs tests and verifies results |
| 30 | # It's intended to wrap tests run with layers and verify the following: |
| 31 | # 1. Any expected layers are inserted |
| 32 | # 2. Any expected layer errors are output |
| 33 | # 3. No unexpected errors are output |
| 34 | |
| 35 | def handle_args(): |
| 36 | parser = argparse.ArgumentParser(description='Run Vulkan test suite and report errors.') |
| 37 | parser.add_argument('--script_name', required=False, default='./run_all_tests_with_layers.sh', help='The script file to be executed and have its output checked.') |
| 38 | return parser.parse_args() |
| 39 | |
| 40 | expected_layers = ['DrawState', 'MemTracker', 'ParamChecker', 'ObjectTracker'] |
| 41 | |
| 42 | # Format of this dict is <testname> key points to a list of expected error text |
| 43 | # The start of each line of output for any given test is compared against the expected error txt |
| 44 | expected_errors = {'XglRenderTest.CubeWithVertexFetchAndMVP' : ['{OBJTRACK}ERROR : OBJ ERROR : DEPTH_STENCIL_VIEW', |
| 45 | '{OBJTRACK}ERROR : OBJ ERROR : GPU_MEMORY', |
| 46 | '{OBJTRACK}ERROR : OBJ ERROR : IMAGE'], |
| 47 | 'XglRenderTest.CubeWithVertexFetchAndMVPAndTexture' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER', |
| 48 | '{OBJTRACK}ERROR : OBJ ERROR : DEPTH_STENCIL_VIEW', |
| 49 | '{OBJTRACK}ERROR : OBJ ERROR : GPU_MEMORY', |
| 50 | '{OBJTRACK}ERROR : OBJ ERROR : IMAGE'], |
| 51 | 'XglTest.Fence' : ['{OBJTRACK}ERROR : OBJECT VALIDATION WARNING: FENCE'], |
Courtney Goeltzenleuchter | d8e229c | 2015-04-08 15:36:08 -0600 | [diff] [blame] | 52 | #'XglRenderTest.VKTriangle_OutputLocation' : ['{OBJTRACK}ERROR : vkQueueSubmit Memory reference count'], |
Tobin Ehlis | d22e3ff | 2015-03-27 11:47:10 -0600 | [diff] [blame] | 53 | 'XglRenderTest.TriangleWithVertexFetch' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'], |
| 54 | 'XglRenderTest.TriangleMRT' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'], |
| 55 | 'XglRenderTest.QuadWithIndexedVertexFetch' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER', '{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'], |
| 56 | 'XglRenderTest.GreyandRedCirclesonBlue' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'], |
| 57 | 'XglRenderTest.RedCirclesonBlue' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'], |
| 58 | 'XglRenderTest.GreyCirclesonBlueFade' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'], |
| 59 | 'XglRenderTest.GreyCirclesonBlueDiscard' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'], |
| 60 | 'XglRenderTest.TriVertFetchAndVertID' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'], |
| 61 | 'XglRenderTest.TriVertFetchDeadAttr' : ['{OBJTRACK}ERROR : OBJ ERROR : CMD_BUFFER'],} |
| 62 | |
| 63 | # Verify that expected errors are hit |
| 64 | # Return True if all expected errors for any matched tests are found and no unexpected errors are found |
| 65 | # Return False if a test with expected errors doesn't have all expected errors and/or any unexpected errors occur |
| 66 | def verify_errors(out_lines): |
| 67 | print "Verifying expected errors and making sure no unexpected errors occur." |
| 68 | result = True |
| 69 | # If we hit a testname line, then catch test name |
| 70 | # If testname is in dict, then get expected errors |
| 71 | # Pop expected errors off of the list |
| 72 | # for any expected errors remaining, flag an issue |
| 73 | testname = '' |
| 74 | ee_list = [] |
| 75 | errors_found = 0 |
| 76 | check_expected_errors = False |
| 77 | for line in out_lines: |
| 78 | if '[ RUN ]' in line: |
| 79 | testname = line[line.find(']')+1:].split()[0].strip() |
| 80 | #print "Found testname %s" % testname |
| 81 | if testname in expected_errors: |
| 82 | ee_list = expected_errors[testname]; |
| 83 | #print "testname %s has %i expected errors in ee_list %s" % (testname, len(ee_list), ", ".join(ee_list)) |
| 84 | check_expected_errors = True |
| 85 | elif '[ OK ]' in line: |
| 86 | if len(ee_list) > 0: |
| 87 | print "ERROR : Failed to find expected error(s) for %s: %s" % (testname, ", ".join(ee_list)) |
| 88 | result = False |
| 89 | check_expected_errors = False |
| 90 | ee_list = [] |
| 91 | errors_found = 0 |
| 92 | elif check_expected_errors: |
| 93 | if True in [line.startswith(ex_err) for ex_err in ee_list]: |
| 94 | for ee in ee_list: |
| 95 | if line.startswith(ee): |
| 96 | #print "Found expected error %s in line %s" % (ee, line) |
| 97 | errors_found += 1 |
| 98 | ee_list.remove(ee) |
| 99 | if len(ee_list) == 0: |
| 100 | print "Found all %i expected errors for testname %s" % (errors_found, testname) |
| 101 | elif 'ERROR' in line: |
| 102 | print "ERROR : Found unexpected error %s for test %s" % (line, testname) |
| 103 | result = False |
| 104 | return result |
| 105 | |
| 106 | def verify_layers(out_lines): |
| 107 | for line in out_lines: |
| 108 | if line.startswith("Inserting layer "): |
| 109 | layer_name = line.split()[2] |
| 110 | #print "Found layer %s" % (layer_name) |
| 111 | if layer_name in expected_layers: |
| 112 | expected_layers.remove(layer_name) |
| 113 | if 0 == len(expected_layers): |
| 114 | print "Found all expected layers" |
| 115 | return True |
| 116 | print "Failed to find layers: %s" % ", ".join(expected_layers) |
| 117 | return False |
| 118 | |
| 119 | def main(argv=None): |
| 120 | # parse args |
| 121 | opts = handle_args() |
| 122 | # run test and capture output |
| 123 | print "Running test script %s" % (opts.script_name) |
| 124 | test_result = subprocess.check_output(opts.script_name, stderr=subprocess.STDOUT) #, shell=True) |
| 125 | result_lines = test_result.split("\n") |
| 126 | if verify_layers(result_lines): |
| 127 | if verify_errors(result_lines): |
| 128 | print "Verify errors completed successfully" |
| 129 | else: |
| 130 | print "Verify errors FAILED! See ERROR messages above" |
| 131 | else: |
| 132 | print "Verify errors FAILED! See ERROR messages above" |
| 133 | print "Done running test suite" |
| 134 | # verify output |
| 135 | |
| 136 | |
| 137 | if __name__ == "__main__": |
| 138 | sys.exit(main()) |