Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / deqp / b12acd0bdf872d288e7670bfcfc458b7d424849d / . / scripts / run_nightly.py
blob: a4f9e18d75b0f73ff53c646e9ea29ded5950b63f [file] [log] [blame]
Jarkko Poyry3c827362014-09-02 11:48:52 +0300[diff] [blame]1# -*- coding: utf-8 -*-
2
Jarkko Pöyry3c77ed42015-01-06 12:54:34 -0800[diff] [blame]3#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13# http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
Jarkko Poyry3c827362014-09-02 11:48:52 +0300[diff] [blame]23from build.common import *
24from build.config import *
25from build.build import *
26
27import os
28import sys
29import string
30import socket
31import fnmatch
32from datetime import datetime
33
34BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly"))
35BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build")
36BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs")
37BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs")
38
39EXECUTOR_PATH = "executor/executor"
40LOG_TO_CSV_PATH = "executor/testlog-to-csv"
41EXECSERVER_PATH = "execserver/execserver"
42
43CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data")
44
45COMPARE_NUM_RESULTS = 4
46COMPARE_REPORT_NAME = "nightly-report.html"
47
48COMPARE_REPORT_TMPL = '''
49<html>
50<head>
51<title>${TITLE}</title>
52<style type="text/css">
53<!--
54body { font: serif; font-size: 1em; }
55table { border-spacing: 0; border-collapse: collapse; }
56td { border-width: 1px; border-style: solid; border-color: #808080; }
57.Header { font-weight: bold; font-size: 1em; border-style: none; }
58.CasePath { }
59.Pass { background: #80ff80; }
60.Fail { background: #ff4040; }
61.QualityWarning { background: #ffff00; }
62.CompabilityWarning { background: #ffff00; }
63.Pending { background: #808080; }
64.Running { background: #d3d3d3; }
65.NotSupported { background: #ff69b4; }
66.ResourceError { background: #ff4040; }
67.InternalError { background: #ff1493; }
68.Canceled { background: #808080; }
69.Crash { background: #ffa500; }
70.Timeout { background: #ffa500; }
71.Disabled { background: #808080; }
72.Missing { background: #808080; }
73.Ignored { opacity: 0.5; }
74-->
75</style>
76</head>
77<body>
78<h1>${TITLE}</h1>
79<table>
80${RESULTS}
81</table>
82</body>
83</html>
84'''
85
86class NightlyRunConfig:
87 def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []):
88 self.name = name
89 self.buildConfig = buildConfig
90 self.generator = generator
91 self.binaryName = binaryName
92 self.testset = testset
93 self.args = args
94 self.exclude = exclude
95 self.ignore = ignore
96
97 def getBinaryPath(self, basePath):
98 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath))
99
100class NightlyBuildConfig(BuildConfig):
101 def __init__(self, name, buildType, args):
102 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args)
103
104class TestCaseResult:
105 def __init__ (self, name, statusCode):
106 self.name = name
107 self.statusCode = statusCode
108
109class MultiResult:
110 def __init__ (self, name, statusCodes):
111 self.name = name
112 self.statusCodes = statusCodes
113
114class BatchResult:
115 def __init__ (self, name):
116 self.name = name
117 self.results = []
118
119def parseResultCsv (data):
120 lines = data.splitlines()[1:]
121 results = []
122
123 for line in lines:
124 items = line.split(",")
125 results.append(TestCaseResult(items[0], items[1]))
126
127 return results
128
129def readTestCaseResultsFromCSV (filename):
130 return parseResultCsv(readFile(filename))
131
132def readBatchResultFromCSV (filename, batchResultName = None):
133 batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename))
134 batchResult.results = readTestCaseResultsFromCSV(filename)
135 return batchResult
136
137def getResultTimestamp ():
138 return datetime.now().strftime("%Y-%m-%d-%H-%M")
139
140def getCompareFilenames (logsDir):
141 files = []
142 for file in os.listdir(logsDir):
143 fullPath = os.path.join(logsDir, file)
144 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"):
145 files.append(fullPath)
146 files.sort()
147
148 return files[-COMPARE_NUM_RESULTS:]
149
150def parseAsCSV (logPath, config):
151 args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath]
152 proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
153 out, err = proc.communicate()
154 return out
155
156def computeUnifiedTestCaseList (batchResults):
157 caseList = []
158 caseSet = set()
159
160 for batchResult in batchResults:
161 for result in batchResult.results:
162 if not result.name in caseSet:
163 caseList.append(result.name)
164 caseSet.add(result.name)
165
166 return caseList
167
168def computeUnifiedResults (batchResults):
169
170 def genResultMap (batchResult):
171 resMap = {}
172 for result in batchResult.results:
173 resMap[result.name] = result
174 return resMap
175
176 resultMap = [genResultMap(r) for r in batchResults]
177 caseList = computeUnifiedTestCaseList(batchResults)
178 results = []
179
180 for caseName in caseList:
181 statusCodes = []
182
183 for i in range(0, len(batchResults)):
184 result = resultMap[i][caseName] if caseName in resultMap[i] else None
185 statusCode = result.statusCode if result != None else 'Missing'
186 statusCodes.append(statusCode)
187
188 results.append(MultiResult(caseName, statusCodes))
189
190 return results
191
192def allStatusCodesEqual (result):
193 firstCode = result.statusCodes[0]
194 for i in range(1, len(result.statusCodes)):
195 if result.statusCodes[i] != firstCode:
196 return False
197 return True
198
199def computeDiffResults (unifiedResults):
200 diff = []
201 for result in unifiedResults:
202 if not allStatusCodesEqual(result):
203 diff.append(result)
204 return diff
205
206def genCompareReport (batchResults, title, ignoreCases):
207 class TableRow:
208 def __init__ (self, testCaseName, innerHTML):
209 self.testCaseName = testCaseName
210 self.innerHTML = innerHTML
211
212 unifiedResults = computeUnifiedResults(batchResults)
213 diffResults = computeDiffResults(unifiedResults)
214 rows = []
215
216 # header
217 headerCol = '<td class="Header">Test case</td>\n'
218 for batchResult in batchResults:
219 headerCol += '<td class="Header">%s</td>\n' % batchResult.name
220 rows.append(TableRow(None, headerCol))
221
222 # results
223 for result in diffResults:
224 col = '<td class="CasePath">%s</td>\n' % result.name
225 for statusCode in result.statusCodes:
226 col += '<td class="%s">%s</td>\n' % (statusCode, statusCode)
227
228 rows.append(TableRow(result.name, col))
229
230 tableStr = ""
231 for row in rows:
232 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases):
233 tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML
234 else:
235 tableStr += '<tr>\n%s</tr>\n' % row.innerHTML
236
237 html = COMPARE_REPORT_TMPL
238 html = html.replace("${TITLE}", title)
239 html = html.replace("${RESULTS}", tableStr)
240
241 return html
242
243def matchesAnyPattern (name, patterns):
244 for pattern in patterns:
245 if fnmatch.fnmatch(name, pattern):
246 return True
247 return False
248
249def statusCodesMatch (refResult, resResult):
250 return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult
251
252def compareBatchResults (referenceBatch, resultBatch, ignoreCases):
253 unifiedResults = computeUnifiedResults([referenceBatch, resultBatch])
254 failedCases = []
255
256 for result in unifiedResults:
257 if not matchesAnyPattern(result.name, ignoreCases):
258 refResult = result.statusCodes[0]
259 resResult = result.statusCodes[1]
260
261 if not statusCodesMatch(refResult, resResult):
262 failedCases.append(result)
263
264 return failedCases
265
266def getUnusedPort ():
267 # \note Not 100%-proof method as other apps may grab this port before we launch execserver
268 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
269 s.bind(('localhost', 0))
270 addr, port = s.getsockname()
271 s.close()
272 return port
273
274def runNightly (config):
275 build(config.buildConfig, config.generator)
276
277 # Run parameters
278 timestamp = getResultTimestamp()
279 logDir = os.path.join(BASE_LOGS_DIR, config.name)
280 testLogPath = os.path.join(logDir, timestamp + ".qpa")
281 infoLogPath = os.path.join(logDir, timestamp + ".txt")
282 csvLogPath = os.path.join(logDir, timestamp + ".csv")
283 compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv")
284 port = getUnusedPort()
285
286 if not os.path.exists(logDir):
287 os.makedirs(logDir)
288
289 if os.path.exists(testLogPath) or os.path.exists(infoLogPath):
290 raise Exception("Result '%s' already exists", timestamp)
291
292 # Paths, etc.
293 binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName))
294 workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName))
295
296 execArgs = [
297 config.getBinaryPath(EXECUTOR_PATH),
298 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH),
299 '--port=%d' % port,
300 '--binaryname=%s' % binaryName,
301 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "),
302 '--workdir=%s' % workingDir,
303 '--caselistdir=%s' % CASELIST_PATH,
304 '--testset=%s' % string.join(config.testset, ","),
305 '--out=%s' % testLogPath,
306 '--info=%s' % infoLogPath,
307 '--summary=no'
308 ]
309
310 if len(config.exclude) > 0:
311 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")]
312
313 execute(execArgs)
314
315 # Translate to CSV for comparison purposes
316 lastResultCsv = parseAsCSV(testLogPath, config)
317 writeFile(csvLogPath, lastResultCsv)
318
319 if os.path.exists(compareLogPath):
320 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference")
321 else:
322 refBatchResult = None
323
324 # Generate comparison report
325 compareFilenames = getCompareFilenames(logDir)
326 batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames]
327
328 if refBatchResult != None:
329 batchResults = [refBatchResult] + batchResults
330
331 writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore))
332 print "Comparison report written to %s" % COMPARE_REPORT_NAME
333
334 # Compare to reference
335 if refBatchResult != None:
336 curBatchResult = BatchResult("current")
337 curBatchResult.results = parseResultCsv(lastResultCsv)
338 failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore)
339
340 print ""
341 for result in failedCases:
342 print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])
343
344 print ""
345 print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")
346
347 if len(failedCases) > 0:
348 return False
349
350 return True
351
352# Configurations
353
354DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR
355DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR
356
357WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl'])
358ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu'])
359
360BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable']
361
362CONFIGS = [
363 NightlyRunConfig(
364 name = "wgl_x64_release_gles2",
365 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
366 generator = DEFAULT_WIN64_GENERATOR,
367 binaryName = "modules/gles2/deqp-gles2",
368 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
369 testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"],
370 exclude = [
371 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*",
372 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*",
373 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*",
374 ],
375 ignore = []
376 ),
377 NightlyRunConfig(
378 name = "wgl_x64_release_gles3",
379 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
380 generator = DEFAULT_WIN64_GENERATOR,
381 binaryName = "modules/gles3/deqp-gles3",
382 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
383 testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"],
384 exclude = [
385 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*",
386 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*",
387 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*",
388 ],
389 ignore = [
390 "dEQP-GLES3.functional.transform_feedback.*",
391 "dEQP-GLES3.functional.occlusion_query.*",
392 "dEQP-GLES3.functional.lifetime.*",
393 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops",
394 ]
395 ),
396 NightlyRunConfig(
397 name = "wgl_x64_release_gles31",
398 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
399 generator = DEFAULT_WIN64_GENERATOR,
400 binaryName = "modules/gles31/deqp-gles31",
401 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
402 testset = ["dEQP-GLES31.*"],
403 exclude = [],
404 ignore = [
405 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3",
406 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer",
407 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset",
408 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint",
409 "dEQP-GLES31.functional.blend_equation_advanced.basic.*",
410 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*",
411 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*",
412 "dEQP-GLES31.functional.uniform_location.*",
413 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv",
414 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv",
415 "dEQP-GLES31.functional.debug.error_filters.case_0",
416 "dEQP-GLES31.functional.debug.error_filters.case_2",
417 ]
418 ),
419 NightlyRunConfig(
420 name = "wgl_x64_release_gl3",
421 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
422 generator = DEFAULT_WIN64_GENERATOR,
423 binaryName = "modules/gl3/deqp-gl3",
424 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
425 testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"],
426 exclude = [
427 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*",
428 "dEQP-GL3.functional.shaders.loops.*while*only_continue*",
429 "dEQP-GL3.functional.shaders.loops.*while*double_continue*",
430 ],
431 ignore = [
432 "dEQP-GL3.functional.transform_feedback.*"
433 ]
434 ),
435 NightlyRunConfig(
436 name = "arm_gles3_emu_x32_egl",
437 buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG,
438 generator = DEFAULT_WIN32_GENERATOR,
439 binaryName = "modules/egl/deqp-egl",
440 args = BASE_ARGS,
441 testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"],
442 exclude = [
443 "dEQP-EGL.functional.sharing.gles2.multithread.*",
444 "dEQP-EGL.functional.multithread.*",
445 ],
446 ignore = []
447 ),
448 NightlyRunConfig(
449 name = "opencl_x64_release",
450 buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']),
451 generator = DEFAULT_WIN64_GENERATOR,
452 binaryName = "modules/opencl/deqp-opencl",
453 args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS,
454 testset = ["dEQP-CL.*"],
455 exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"],
456 ignore = [
457 "dEQP-CL.scheduler.random.*",
458 "dEQP-CL.language.set_kernel_arg.random_structs.*",
459 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset",
460 "dEQP-CL.language.call_function.arguments.random_structs.*",
461 "dEQP-CL.language.call_kernel.random_structs.*",
462 "dEQP-CL.language.inf_nan.nan.frexp.float",
463 "dEQP-CL.language.inf_nan.nan.lgamma_r.float",
464 "dEQP-CL.language.inf_nan.nan.modf.float",
465 "dEQP-CL.language.inf_nan.nan.sqrt.float",
466 "dEQP-CL.api.multithread.*",
467 "dEQP-CL.api.callback.random.nested.*",
468 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb",
469 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb",
470 "dEQP-CL.image.addressing_filtering12.1d_array.*",
471 "dEQP-CL.image.addressing_filtering12.2d_array.*"
472 ]
473 )
474]
475
476if __name__ == "__main__":
477 config = None
478
479 if len(sys.argv) == 2:
480 cfgName = sys.argv[1]
481 for curCfg in CONFIGS:
482 if curCfg.name == cfgName:
483 config = curCfg
484 break
485
486 if config != None:
487 isOk = runNightly(config)
488 if not isOk:
489 sys.exit(-1)
490 else:
491 print "%s: [config]" % sys.argv[0]
492 print ""
493 print " Available configs:"
494 for config in CONFIGS:
495 print " %s" % config.name
496 sys.exit(-1)