| # -*- coding: utf-8 -*- |
| |
| #------------------------------------------------------------------------- |
| # drawElements Quality Program utilities |
| # -------------------------------------- |
| # |
| # Copyright 2015 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| #------------------------------------------------------------------------- |
| |
| from build.common import * |
| from build.config import * |
| from build.build import * |
| |
| import os |
| import sys |
| import string |
| import socket |
| import fnmatch |
| from datetime import datetime |
| |
| BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly")) |
| BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build") |
| BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs") |
| BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs") |
| |
| EXECUTOR_PATH = "executor/executor" |
| LOG_TO_CSV_PATH = "executor/testlog-to-csv" |
| EXECSERVER_PATH = "execserver/execserver" |
| |
| CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data") |
| |
| COMPARE_NUM_RESULTS = 4 |
| COMPARE_REPORT_NAME = "nightly-report.html" |
| |
| COMPARE_REPORT_TMPL = ''' |
| <html> |
| <head> |
| <title>${TITLE}</title> |
| <style type="text/css"> |
| <!-- |
| body { font: serif; font-size: 1em; } |
| table { border-spacing: 0; border-collapse: collapse; } |
| td { border-width: 1px; border-style: solid; border-color: #808080; } |
| .Header { font-weight: bold; font-size: 1em; border-style: none; } |
| .CasePath { } |
| .Pass { background: #80ff80; } |
| .Fail { background: #ff4040; } |
| .QualityWarning { background: #ffff00; } |
| .CompabilityWarning { background: #ffff00; } |
| .Pending { background: #808080; } |
| .Running { background: #d3d3d3; } |
| .NotSupported { background: #ff69b4; } |
| .ResourceError { background: #ff4040; } |
| .InternalError { background: #ff1493; } |
| .Canceled { background: #808080; } |
| .Crash { background: #ffa500; } |
| .Timeout { background: #ffa500; } |
| .Disabled { background: #808080; } |
| .Missing { background: #808080; } |
| .Ignored { opacity: 0.5; } |
| --> |
| </style> |
| </head> |
| <body> |
| <h1>${TITLE}</h1> |
| <table> |
| ${RESULTS} |
| </table> |
| </body> |
| </html> |
| ''' |
| |
| class NightlyRunConfig: |
| def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []): |
| self.name = name |
| self.buildConfig = buildConfig |
| self.generator = generator |
| self.binaryName = binaryName |
| self.testset = testset |
| self.args = args |
| self.exclude = exclude |
| self.ignore = ignore |
| |
| def getBinaryPath(self, basePath): |
| return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath)) |
| |
| class NightlyBuildConfig(BuildConfig): |
| def __init__(self, name, buildType, args): |
| BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args) |
| |
| class TestCaseResult: |
| def __init__ (self, name, statusCode): |
| self.name = name |
| self.statusCode = statusCode |
| |
| class MultiResult: |
| def __init__ (self, name, statusCodes): |
| self.name = name |
| self.statusCodes = statusCodes |
| |
| class BatchResult: |
| def __init__ (self, name): |
| self.name = name |
| self.results = [] |
| |
| def parseResultCsv (data): |
| lines = data.splitlines()[1:] |
| results = [] |
| |
| for line in lines: |
| items = line.split(",") |
| results.append(TestCaseResult(items[0], items[1])) |
| |
| return results |
| |
| def readTestCaseResultsFromCSV (filename): |
| return parseResultCsv(readFile(filename)) |
| |
| def readBatchResultFromCSV (filename, batchResultName = None): |
| batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename)) |
| batchResult.results = readTestCaseResultsFromCSV(filename) |
| return batchResult |
| |
| def getResultTimestamp (): |
| return datetime.now().strftime("%Y-%m-%d-%H-%M") |
| |
| def getCompareFilenames (logsDir): |
| files = [] |
| for file in os.listdir(logsDir): |
| fullPath = os.path.join(logsDir, file) |
| if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"): |
| files.append(fullPath) |
| files.sort() |
| |
| return files[-COMPARE_NUM_RESULTS:] |
| |
| def parseAsCSV (logPath, config): |
| args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath] |
| proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| out, err = proc.communicate() |
| return out |
| |
| def computeUnifiedTestCaseList (batchResults): |
| caseList = [] |
| caseSet = set() |
| |
| for batchResult in batchResults: |
| for result in batchResult.results: |
| if not result.name in caseSet: |
| caseList.append(result.name) |
| caseSet.add(result.name) |
| |
| return caseList |
| |
| def computeUnifiedResults (batchResults): |
| |
| def genResultMap (batchResult): |
| resMap = {} |
| for result in batchResult.results: |
| resMap[result.name] = result |
| return resMap |
| |
| resultMap = [genResultMap(r) for r in batchResults] |
| caseList = computeUnifiedTestCaseList(batchResults) |
| results = [] |
| |
| for caseName in caseList: |
| statusCodes = [] |
| |
| for i in range(0, len(batchResults)): |
| result = resultMap[i][caseName] if caseName in resultMap[i] else None |
| statusCode = result.statusCode if result != None else 'Missing' |
| statusCodes.append(statusCode) |
| |
| results.append(MultiResult(caseName, statusCodes)) |
| |
| return results |
| |
| def allStatusCodesEqual (result): |
| firstCode = result.statusCodes[0] |
| for i in range(1, len(result.statusCodes)): |
| if result.statusCodes[i] != firstCode: |
| return False |
| return True |
| |
| def computeDiffResults (unifiedResults): |
| diff = [] |
| for result in unifiedResults: |
| if not allStatusCodesEqual(result): |
| diff.append(result) |
| return diff |
| |
| def genCompareReport (batchResults, title, ignoreCases): |
| class TableRow: |
| def __init__ (self, testCaseName, innerHTML): |
| self.testCaseName = testCaseName |
| self.innerHTML = innerHTML |
| |
| unifiedResults = computeUnifiedResults(batchResults) |
| diffResults = computeDiffResults(unifiedResults) |
| rows = [] |
| |
| # header |
| headerCol = '<td class="Header">Test case</td>\n' |
| for batchResult in batchResults: |
| headerCol += '<td class="Header">%s</td>\n' % batchResult.name |
| rows.append(TableRow(None, headerCol)) |
| |
| # results |
| for result in diffResults: |
| col = '<td class="CasePath">%s</td>\n' % result.name |
| for statusCode in result.statusCodes: |
| col += '<td class="%s">%s</td>\n' % (statusCode, statusCode) |
| |
| rows.append(TableRow(result.name, col)) |
| |
| tableStr = "" |
| for row in rows: |
| if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases): |
| tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML |
| else: |
| tableStr += '<tr>\n%s</tr>\n' % row.innerHTML |
| |
| html = COMPARE_REPORT_TMPL |
| html = html.replace("${TITLE}", title) |
| html = html.replace("${RESULTS}", tableStr) |
| |
| return html |
| |
| def matchesAnyPattern (name, patterns): |
| for pattern in patterns: |
| if fnmatch.fnmatch(name, pattern): |
| return True |
| return False |
| |
| def statusCodesMatch (refResult, resResult): |
| return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult |
| |
| def compareBatchResults (referenceBatch, resultBatch, ignoreCases): |
| unifiedResults = computeUnifiedResults([referenceBatch, resultBatch]) |
| failedCases = [] |
| |
| for result in unifiedResults: |
| if not matchesAnyPattern(result.name, ignoreCases): |
| refResult = result.statusCodes[0] |
| resResult = result.statusCodes[1] |
| |
| if not statusCodesMatch(refResult, resResult): |
| failedCases.append(result) |
| |
| return failedCases |
| |
| def getUnusedPort (): |
| # \note Not 100%-proof method as other apps may grab this port before we launch execserver |
| s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) |
| s.bind(('localhost', 0)) |
| addr, port = s.getsockname() |
| s.close() |
| return port |
| |
| def runNightly (config): |
| build(config.buildConfig, config.generator) |
| |
| # Run parameters |
| timestamp = getResultTimestamp() |
| logDir = os.path.join(BASE_LOGS_DIR, config.name) |
| testLogPath = os.path.join(logDir, timestamp + ".qpa") |
| infoLogPath = os.path.join(logDir, timestamp + ".txt") |
| csvLogPath = os.path.join(logDir, timestamp + ".csv") |
| compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv") |
| port = getUnusedPort() |
| |
| if not os.path.exists(logDir): |
| os.makedirs(logDir) |
| |
| if os.path.exists(testLogPath) or os.path.exists(infoLogPath): |
| raise Exception("Result '%s' already exists", timestamp) |
| |
| # Paths, etc. |
| binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName)) |
| workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName)) |
| |
| execArgs = [ |
| config.getBinaryPath(EXECUTOR_PATH), |
| '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH), |
| '--port=%d' % port, |
| '--binaryname=%s' % binaryName, |
| '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "), |
| '--workdir=%s' % workingDir, |
| '--caselistdir=%s' % CASELIST_PATH, |
| '--testset=%s' % string.join(config.testset, ","), |
| '--out=%s' % testLogPath, |
| '--info=%s' % infoLogPath, |
| '--summary=no' |
| ] |
| |
| if len(config.exclude) > 0: |
| execArgs += ['--exclude=%s' % string.join(config.exclude, ",")] |
| |
| execute(execArgs) |
| |
| # Translate to CSV for comparison purposes |
| lastResultCsv = parseAsCSV(testLogPath, config) |
| writeFile(csvLogPath, lastResultCsv) |
| |
| if os.path.exists(compareLogPath): |
| refBatchResult = readBatchResultFromCSV(compareLogPath, "reference") |
| else: |
| refBatchResult = None |
| |
| # Generate comparison report |
| compareFilenames = getCompareFilenames(logDir) |
| batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames] |
| |
| if refBatchResult != None: |
| batchResults = [refBatchResult] + batchResults |
| |
| writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore)) |
| print "Comparison report written to %s" % COMPARE_REPORT_NAME |
| |
| # Compare to reference |
| if refBatchResult != None: |
| curBatchResult = BatchResult("current") |
| curBatchResult.results = parseResultCsv(lastResultCsv) |
| failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore) |
| |
| print "" |
| for result in failedCases: |
| print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1]) |
| |
| print "" |
| print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed") |
| |
| if len(failedCases) > 0: |
| return False |
| |
| return True |
| |
| # Configurations |
| |
| DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR |
| DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR |
| |
| WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl']) |
| ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu']) |
| |
| BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable'] |
| |
| CONFIGS = [ |
| NightlyRunConfig( |
| name = "wgl_x64_release_gles2", |
| buildConfig = WGL_X64_RELEASE_BUILD_CFG, |
| generator = DEFAULT_WIN64_GENERATOR, |
| binaryName = "modules/gles2/deqp-gles2", |
| args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, |
| testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"], |
| exclude = [ |
| "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*", |
| "dEQP-GLES2.functional.shaders.loops.*while*only_continue*", |
| "dEQP-GLES2.functional.shaders.loops.*while*double_continue*", |
| ], |
| ignore = [] |
| ), |
| NightlyRunConfig( |
| name = "wgl_x64_release_gles3", |
| buildConfig = WGL_X64_RELEASE_BUILD_CFG, |
| generator = DEFAULT_WIN64_GENERATOR, |
| binaryName = "modules/gles3/deqp-gles3", |
| args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, |
| testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"], |
| exclude = [ |
| "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*", |
| "dEQP-GLES3.functional.shaders.loops.*while*only_continue*", |
| "dEQP-GLES3.functional.shaders.loops.*while*double_continue*", |
| ], |
| ignore = [ |
| "dEQP-GLES3.functional.transform_feedback.*", |
| "dEQP-GLES3.functional.occlusion_query.*", |
| "dEQP-GLES3.functional.lifetime.*", |
| "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops", |
| ] |
| ), |
| NightlyRunConfig( |
| name = "wgl_x64_release_gles31", |
| buildConfig = WGL_X64_RELEASE_BUILD_CFG, |
| generator = DEFAULT_WIN64_GENERATOR, |
| binaryName = "modules/gles31/deqp-gles31", |
| args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, |
| testset = ["dEQP-GLES31.*"], |
| exclude = [], |
| ignore = [ |
| "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3", |
| "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer", |
| "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset", |
| "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint", |
| "dEQP-GLES31.functional.blend_equation_advanced.basic.*", |
| "dEQP-GLES31.functional.blend_equation_advanced.srgb.*", |
| "dEQP-GLES31.functional.blend_equation_advanced.barrier.*", |
| "dEQP-GLES31.functional.uniform_location.*", |
| "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv", |
| "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv", |
| "dEQP-GLES31.functional.debug.error_filters.case_0", |
| "dEQP-GLES31.functional.debug.error_filters.case_2", |
| ] |
| ), |
| NightlyRunConfig( |
| name = "wgl_x64_release_gl3", |
| buildConfig = WGL_X64_RELEASE_BUILD_CFG, |
| generator = DEFAULT_WIN64_GENERATOR, |
| binaryName = "modules/gl3/deqp-gl3", |
| args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS, |
| testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"], |
| exclude = [ |
| "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*", |
| "dEQP-GL3.functional.shaders.loops.*while*only_continue*", |
| "dEQP-GL3.functional.shaders.loops.*while*double_continue*", |
| ], |
| ignore = [ |
| "dEQP-GL3.functional.transform_feedback.*" |
| ] |
| ), |
| NightlyRunConfig( |
| name = "arm_gles3_emu_x32_egl", |
| buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG, |
| generator = DEFAULT_WIN32_GENERATOR, |
| binaryName = "modules/egl/deqp-egl", |
| args = BASE_ARGS, |
| testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"], |
| exclude = [ |
| "dEQP-EGL.functional.sharing.gles2.multithread.*", |
| "dEQP-EGL.functional.multithread.*", |
| ], |
| ignore = [] |
| ), |
| NightlyRunConfig( |
| name = "opencl_x64_release", |
| buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']), |
| generator = DEFAULT_WIN64_GENERATOR, |
| binaryName = "modules/opencl/deqp-opencl", |
| args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS, |
| testset = ["dEQP-CL.*"], |
| exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"], |
| ignore = [ |
| "dEQP-CL.scheduler.random.*", |
| "dEQP-CL.language.set_kernel_arg.random_structs.*", |
| "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset", |
| "dEQP-CL.language.call_function.arguments.random_structs.*", |
| "dEQP-CL.language.call_kernel.random_structs.*", |
| "dEQP-CL.language.inf_nan.nan.frexp.float", |
| "dEQP-CL.language.inf_nan.nan.lgamma_r.float", |
| "dEQP-CL.language.inf_nan.nan.modf.float", |
| "dEQP-CL.language.inf_nan.nan.sqrt.float", |
| "dEQP-CL.api.multithread.*", |
| "dEQP-CL.api.callback.random.nested.*", |
| "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb", |
| "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb", |
| "dEQP-CL.image.addressing_filtering12.1d_array.*", |
| "dEQP-CL.image.addressing_filtering12.2d_array.*" |
| ] |
| ) |
| ] |
| |
| if __name__ == "__main__": |
| config = None |
| |
| if len(sys.argv) == 2: |
| cfgName = sys.argv[1] |
| for curCfg in CONFIGS: |
| if curCfg.name == cfgName: |
| config = curCfg |
| break |
| |
| if config != None: |
| isOk = runNightly(config) |
| if not isOk: |
| sys.exit(-1) |
| else: |
| print "%s: [config]" % sys.argv[0] |
| print "" |
| print " Available configs:" |
| for config in CONFIGS: |
| print " %s" % config.name |
| sys.exit(-1) |