Import dEQP.
Import drawElements Quality Program from an internal repository.
Bug: 17388917
Change-Id: Ic109fe4a57e31b2a816113d90fbdf51a43e7abeb
diff --git a/scripts/run_nightly.py b/scripts/run_nightly.py
new file mode 100644
index 0000000..f8cb63d
--- /dev/null
+++ b/scripts/run_nightly.py
@@ -0,0 +1,476 @@
+# -*- coding: utf-8 -*-
+
+from build.common import *
+from build.config import *
+from build.build import *
+
+import os
+import sys
+import string
+import socket
+import fnmatch
+from datetime import datetime
+
+BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly"))
+BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build")
+BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs")
+BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs")
+
+EXECUTOR_PATH = "executor/executor"
+LOG_TO_CSV_PATH = "executor/testlog-to-csv"
+EXECSERVER_PATH = "execserver/execserver"
+
+CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data")
+
+COMPARE_NUM_RESULTS = 4
+COMPARE_REPORT_NAME = "nightly-report.html"
+
+COMPARE_REPORT_TMPL = '''
+<html>
+<head>
+<title>${TITLE}</title>
+<style type="text/css">
+<!--
+body { font: serif; font-size: 1em; }
+table { border-spacing: 0; border-collapse: collapse; }
+td { border-width: 1px; border-style: solid; border-color: #808080; }
+.Header { font-weight: bold; font-size: 1em; border-style: none; }
+.CasePath { }
+.Pass { background: #80ff80; }
+.Fail { background: #ff4040; }
+.QualityWarning { background: #ffff00; }
+.CompabilityWarning { background: #ffff00; }
+.Pending { background: #808080; }
+.Running { background: #d3d3d3; }
+.NotSupported { background: #ff69b4; }
+.ResourceError { background: #ff4040; }
+.InternalError { background: #ff1493; }
+.Canceled { background: #808080; }
+.Crash { background: #ffa500; }
+.Timeout { background: #ffa500; }
+.Disabled { background: #808080; }
+.Missing { background: #808080; }
+.Ignored { opacity: 0.5; }
+-->
+</style>
+</head>
+<body>
+<h1>${TITLE}</h1>
+<table>
+${RESULTS}
+</table>
+</body>
+</html>
+'''
+
+class NightlyRunConfig:
+ def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []):
+ self.name = name
+ self.buildConfig = buildConfig
+ self.generator = generator
+ self.binaryName = binaryName
+ self.testset = testset
+ self.args = args
+ self.exclude = exclude
+ self.ignore = ignore
+
+ def getBinaryPath(self, basePath):
+ return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath))
+
+class NightlyBuildConfig(BuildConfig):
+ def __init__(self, name, buildType, args):
+ BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args)
+
+class TestCaseResult:
+ def __init__ (self, name, statusCode):
+ self.name = name
+ self.statusCode = statusCode
+
+class MultiResult:
+ def __init__ (self, name, statusCodes):
+ self.name = name
+ self.statusCodes = statusCodes
+
+class BatchResult:
+ def __init__ (self, name):
+ self.name = name
+ self.results = []
+
+def parseResultCsv (data):
+ lines = data.splitlines()[1:]
+ results = []
+
+ for line in lines:
+ items = line.split(",")
+ results.append(TestCaseResult(items[0], items[1]))
+
+ return results
+
+def readTestCaseResultsFromCSV (filename):
+ return parseResultCsv(readFile(filename))
+
+def readBatchResultFromCSV (filename, batchResultName = None):
+ batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename))
+ batchResult.results = readTestCaseResultsFromCSV(filename)
+ return batchResult
+
+def getResultTimestamp ():
+ return datetime.now().strftime("%Y-%m-%d-%H-%M")
+
+def getCompareFilenames (logsDir):
+ files = []
+ for file in os.listdir(logsDir):
+ fullPath = os.path.join(logsDir, file)
+ if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"):
+ files.append(fullPath)
+ files.sort()
+
+ return files[-COMPARE_NUM_RESULTS:]
+
+def parseAsCSV (logPath, config):
+ args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath]
+ proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ out, err = proc.communicate()
+ return out
+
+def computeUnifiedTestCaseList (batchResults):
+ caseList = []
+ caseSet = set()
+
+ for batchResult in batchResults:
+ for result in batchResult.results:
+ if not result.name in caseSet:
+ caseList.append(result.name)
+ caseSet.add(result.name)
+
+ return caseList
+
+def computeUnifiedResults (batchResults):
+
+ def genResultMap (batchResult):
+ resMap = {}
+ for result in batchResult.results:
+ resMap[result.name] = result
+ return resMap
+
+ resultMap = [genResultMap(r) for r in batchResults]
+ caseList = computeUnifiedTestCaseList(batchResults)
+ results = []
+
+ for caseName in caseList:
+ statusCodes = []
+
+ for i in range(0, len(batchResults)):
+ result = resultMap[i][caseName] if caseName in resultMap[i] else None
+ statusCode = result.statusCode if result != None else 'Missing'
+ statusCodes.append(statusCode)
+
+ results.append(MultiResult(caseName, statusCodes))
+
+ return results
+
+def allStatusCodesEqual (result):
+ firstCode = result.statusCodes[0]
+ for i in range(1, len(result.statusCodes)):
+ if result.statusCodes[i] != firstCode:
+ return False
+ return True
+
+def computeDiffResults (unifiedResults):
+ diff = []
+ for result in unifiedResults:
+ if not allStatusCodesEqual(result):
+ diff.append(result)
+ return diff
+
+def genCompareReport (batchResults, title, ignoreCases):
+ class TableRow:
+ def __init__ (self, testCaseName, innerHTML):
+ self.testCaseName = testCaseName
+ self.innerHTML = innerHTML
+
+ unifiedResults = computeUnifiedResults(batchResults)
+ diffResults = computeDiffResults(unifiedResults)
+ rows = []
+
+ # header
+ headerCol = '<td class="Header">Test case</td>\n'
+ for batchResult in batchResults:
+ headerCol += '<td class="Header">%s</td>\n' % batchResult.name
+ rows.append(TableRow(None, headerCol))
+
+ # results
+ for result in diffResults:
+ col = '<td class="CasePath">%s</td>\n' % result.name
+ for statusCode in result.statusCodes:
+ col += '<td class="%s">%s</td>\n' % (statusCode, statusCode)
+
+ rows.append(TableRow(result.name, col))
+
+ tableStr = ""
+ for row in rows:
+ if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases):
+ tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML
+ else:
+ tableStr += '<tr>\n%s</tr>\n' % row.innerHTML
+
+ html = COMPARE_REPORT_TMPL
+ html = html.replace("${TITLE}", title)
+ html = html.replace("${RESULTS}", tableStr)
+
+ return html
+
+def matchesAnyPattern (name, patterns):
+ for pattern in patterns:
+ if fnmatch.fnmatch(name, pattern):
+ return True
+ return False
+
+def statusCodesMatch (refResult, resResult):
+ return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult
+
+def compareBatchResults (referenceBatch, resultBatch, ignoreCases):
+ unifiedResults = computeUnifiedResults([referenceBatch, resultBatch])
+ failedCases = []
+
+ for result in unifiedResults:
+ if not matchesAnyPattern(result.name, ignoreCases):
+ refResult = result.statusCodes[0]
+ resResult = result.statusCodes[1]
+
+ if not statusCodesMatch(refResult, resResult):
+ failedCases.append(result)
+
+ return failedCases
+
+def getUnusedPort ():
+ # \note Not 100%-proof method as other apps may grab this port before we launch execserver
+ s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ s.bind(('localhost', 0))
+ addr, port = s.getsockname()
+ s.close()
+ return port
+
+def runNightly (config):
+ build(config.buildConfig, config.generator)
+
+ # Run parameters
+ timestamp = getResultTimestamp()
+ logDir = os.path.join(BASE_LOGS_DIR, config.name)
+ testLogPath = os.path.join(logDir, timestamp + ".qpa")
+ infoLogPath = os.path.join(logDir, timestamp + ".txt")
+ csvLogPath = os.path.join(logDir, timestamp + ".csv")
+ compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv")
+ port = getUnusedPort()
+
+ if not os.path.exists(logDir):
+ os.makedirs(logDir)
+
+ if os.path.exists(testLogPath) or os.path.exists(infoLogPath):
+ raise Exception("Result '%s' already exists", timestamp)
+
+ # Paths, etc.
+ binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName))
+ workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName))
+
+ execArgs = [
+ config.getBinaryPath(EXECUTOR_PATH),
+ '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH),
+ '--port=%d' % port,
+ '--binaryname=%s' % binaryName,
+ '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "),
+ '--workdir=%s' % workingDir,
+ '--caselistdir=%s' % CASELIST_PATH,
+ '--testset=%s' % string.join(config.testset, ","),
+ '--out=%s' % testLogPath,
+ '--info=%s' % infoLogPath,
+ '--summary=no'
+ ]
+
+ if len(config.exclude) > 0:
+ execArgs += ['--exclude=%s' % string.join(config.exclude, ",")]
+
+ execute(execArgs)
+
+ # Translate to CSV for comparison purposes
+ lastResultCsv = parseAsCSV(testLogPath, config)
+ writeFile(csvLogPath, lastResultCsv)
+
+ if os.path.exists(compareLogPath):
+ refBatchResult = readBatchResultFromCSV(compareLogPath, "reference")
+ else:
+ refBatchResult = None
+
+ # Generate comparison report
+ compareFilenames = getCompareFilenames(logDir)
+ batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames]
+
+ if refBatchResult != None:
+ batchResults = [refBatchResult] + batchResults
+
+ writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore))
+ print "Comparison report written to %s" % COMPARE_REPORT_NAME
+
+ # Compare to reference
+ if refBatchResult != None:
+ curBatchResult = BatchResult("current")
+ curBatchResult.results = parseResultCsv(lastResultCsv)
+ failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore)
+
+ print ""
+ for result in failedCases:
+ print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])
+
+ print ""
+ print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")
+
+ if len(failedCases) > 0:
+ return False
+
+ return True
+
+# Configurations
+
+DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR
+DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR
+
+WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl'])
+ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu'])
+
+BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable']
+
+CONFIGS = [
+ NightlyRunConfig(
+ name = "wgl_x64_release_gles2",
+ buildConfig = WGL_X64_RELEASE_BUILD_CFG,
+ generator = DEFAULT_WIN64_GENERATOR,
+ binaryName = "modules/gles2/deqp-gles2",
+ args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
+ testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"],
+ exclude = [
+ "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*",
+ "dEQP-GLES2.functional.shaders.loops.*while*only_continue*",
+ "dEQP-GLES2.functional.shaders.loops.*while*double_continue*",
+ ],
+ ignore = []
+ ),
+ NightlyRunConfig(
+ name = "wgl_x64_release_gles3",
+ buildConfig = WGL_X64_RELEASE_BUILD_CFG,
+ generator = DEFAULT_WIN64_GENERATOR,
+ binaryName = "modules/gles3/deqp-gles3",
+ args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
+ testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"],
+ exclude = [
+ "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*",
+ "dEQP-GLES3.functional.shaders.loops.*while*only_continue*",
+ "dEQP-GLES3.functional.shaders.loops.*while*double_continue*",
+ ],
+ ignore = [
+ "dEQP-GLES3.functional.transform_feedback.*",
+ "dEQP-GLES3.functional.occlusion_query.*",
+ "dEQP-GLES3.functional.lifetime.*",
+ "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops",
+ ]
+ ),
+ NightlyRunConfig(
+ name = "wgl_x64_release_gles31",
+ buildConfig = WGL_X64_RELEASE_BUILD_CFG,
+ generator = DEFAULT_WIN64_GENERATOR,
+ binaryName = "modules/gles31/deqp-gles31",
+ args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
+ testset = ["dEQP-GLES31.*"],
+ exclude = [],
+ ignore = [
+ "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3",
+ "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer",
+ "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset",
+ "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint",
+ "dEQP-GLES31.functional.blend_equation_advanced.basic.*",
+ "dEQP-GLES31.functional.blend_equation_advanced.srgb.*",
+ "dEQP-GLES31.functional.blend_equation_advanced.barrier.*",
+ "dEQP-GLES31.functional.uniform_location.*",
+ "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv",
+ "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv",
+ "dEQP-GLES31.functional.debug.error_filters.case_0",
+ "dEQP-GLES31.functional.debug.error_filters.case_2",
+ ]
+ ),
+ NightlyRunConfig(
+ name = "wgl_x64_release_gl3",
+ buildConfig = WGL_X64_RELEASE_BUILD_CFG,
+ generator = DEFAULT_WIN64_GENERATOR,
+ binaryName = "modules/gl3/deqp-gl3",
+ args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
+ testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"],
+ exclude = [
+ "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*",
+ "dEQP-GL3.functional.shaders.loops.*while*only_continue*",
+ "dEQP-GL3.functional.shaders.loops.*while*double_continue*",
+ ],
+ ignore = [
+ "dEQP-GL3.functional.transform_feedback.*"
+ ]
+ ),
+ NightlyRunConfig(
+ name = "arm_gles3_emu_x32_egl",
+ buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG,
+ generator = DEFAULT_WIN32_GENERATOR,
+ binaryName = "modules/egl/deqp-egl",
+ args = BASE_ARGS,
+ testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"],
+ exclude = [
+ "dEQP-EGL.functional.sharing.gles2.multithread.*",
+ "dEQP-EGL.functional.multithread.*",
+ ],
+ ignore = []
+ ),
+ NightlyRunConfig(
+ name = "opencl_x64_release",
+ buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']),
+ generator = DEFAULT_WIN64_GENERATOR,
+ binaryName = "modules/opencl/deqp-opencl",
+ args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS,
+ testset = ["dEQP-CL.*"],
+ exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"],
+ ignore = [
+ "dEQP-CL.scheduler.random.*",
+ "dEQP-CL.language.set_kernel_arg.random_structs.*",
+ "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset",
+ "dEQP-CL.language.call_function.arguments.random_structs.*",
+ "dEQP-CL.language.call_kernel.random_structs.*",
+ "dEQP-CL.language.inf_nan.nan.frexp.float",
+ "dEQP-CL.language.inf_nan.nan.lgamma_r.float",
+ "dEQP-CL.language.inf_nan.nan.modf.float",
+ "dEQP-CL.language.inf_nan.nan.sqrt.float",
+ "dEQP-CL.api.multithread.*",
+ "dEQP-CL.api.callback.random.nested.*",
+ "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb",
+ "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb",
+ "dEQP-CL.image.addressing_filtering12.1d_array.*",
+ "dEQP-CL.image.addressing_filtering12.2d_array.*"
+ ]
+ )
+]
+
+if __name__ == "__main__":
+ config = None
+
+ if len(sys.argv) == 2:
+ cfgName = sys.argv[1]
+ for curCfg in CONFIGS:
+ if curCfg.name == cfgName:
+ config = curCfg
+ break
+
+ if config != None:
+ isOk = runNightly(config)
+ if not isOk:
+ sys.exit(-1)
+ else:
+ print "%s: [config]" % sys.argv[0]
+ print ""
+ print " Available configs:"
+ for config in CONFIGS:
+ print " %s" % config.name
+ sys.exit(-1)