Gitiles
Code Review Sign In
gerrit-public.fairphone.software / platform / external / deqp / 3c827367444ee418f129b2c238299f49d3264554 / . / scripts / run_nightly.py
blob: f8cb63d0ffecad3385f9566f13dc34403b658696 [file] [log] [blame]
Jarkko Poyry3c827362014-09-02 11:48:52 +0300[diff] [blame^]1# -*- coding: utf-8 -*-
2
3from build.common import *
4from build.config import *
5from build.build import *
6
7import os
8import sys
9import string
10import socket
11import fnmatch
12from datetime import datetime
13
14BASE_NIGHTLY_DIR = os.path.normpath(os.path.join(DEQP_DIR, "..", "deqp-nightly"))
15BASE_BUILD_DIR = os.path.join(BASE_NIGHTLY_DIR, "build")
16BASE_LOGS_DIR = os.path.join(BASE_NIGHTLY_DIR, "logs")
17BASE_REFS_DIR = os.path.join(BASE_NIGHTLY_DIR, "refs")
18
19EXECUTOR_PATH = "executor/executor"
20LOG_TO_CSV_PATH = "executor/testlog-to-csv"
21EXECSERVER_PATH = "execserver/execserver"
22
23CASELIST_PATH = os.path.join(DEQP_DIR, "Candy", "Data")
24
25COMPARE_NUM_RESULTS = 4
26COMPARE_REPORT_NAME = "nightly-report.html"
27
28COMPARE_REPORT_TMPL = '''
29<html>
30<head>
31<title>${TITLE}</title>
32<style type="text/css">
33<!--
34body { font: serif; font-size: 1em; }
35table { border-spacing: 0; border-collapse: collapse; }
36td { border-width: 1px; border-style: solid; border-color: #808080; }
37.Header { font-weight: bold; font-size: 1em; border-style: none; }
38.CasePath { }
39.Pass { background: #80ff80; }
40.Fail { background: #ff4040; }
41.QualityWarning { background: #ffff00; }
42.CompabilityWarning { background: #ffff00; }
43.Pending { background: #808080; }
44.Running { background: #d3d3d3; }
45.NotSupported { background: #ff69b4; }
46.ResourceError { background: #ff4040; }
47.InternalError { background: #ff1493; }
48.Canceled { background: #808080; }
49.Crash { background: #ffa500; }
50.Timeout { background: #ffa500; }
51.Disabled { background: #808080; }
52.Missing { background: #808080; }
53.Ignored { opacity: 0.5; }
54-->
55</style>
56</head>
57<body>
58<h1>${TITLE}</h1>
59<table>
60${RESULTS}
61</table>
62</body>
63</html>
64'''
65
66class NightlyRunConfig:
67 def __init__(self, name, buildConfig, generator, binaryName, testset, args = [], exclude = [], ignore = []):
68 self.name = name
69 self.buildConfig = buildConfig
70 self.generator = generator
71 self.binaryName = binaryName
72 self.testset = testset
73 self.args = args
74 self.exclude = exclude
75 self.ignore = ignore
76
77 def getBinaryPath(self, basePath):
78 return os.path.join(self.buildConfig.getBuildDir(), self.generator.getBinaryPath(self.buildConfig.getBuildType(), basePath))
79
80class NightlyBuildConfig(BuildConfig):
81 def __init__(self, name, buildType, args):
82 BuildConfig.__init__(self, os.path.join(BASE_BUILD_DIR, name), buildType, args)
83
84class TestCaseResult:
85 def __init__ (self, name, statusCode):
86 self.name = name
87 self.statusCode = statusCode
88
89class MultiResult:
90 def __init__ (self, name, statusCodes):
91 self.name = name
92 self.statusCodes = statusCodes
93
94class BatchResult:
95 def __init__ (self, name):
96 self.name = name
97 self.results = []
98
99def parseResultCsv (data):
100 lines = data.splitlines()[1:]
101 results = []
102
103 for line in lines:
104 items = line.split(",")
105 results.append(TestCaseResult(items[0], items[1]))
106
107 return results
108
109def readTestCaseResultsFromCSV (filename):
110 return parseResultCsv(readFile(filename))
111
112def readBatchResultFromCSV (filename, batchResultName = None):
113 batchResult = BatchResult(batchResultName if batchResultName != None else os.path.basename(filename))
114 batchResult.results = readTestCaseResultsFromCSV(filename)
115 return batchResult
116
117def getResultTimestamp ():
118 return datetime.now().strftime("%Y-%m-%d-%H-%M")
119
120def getCompareFilenames (logsDir):
121 files = []
122 for file in os.listdir(logsDir):
123 fullPath = os.path.join(logsDir, file)
124 if os.path.isfile(fullPath) and fnmatch.fnmatch(file, "*.csv"):
125 files.append(fullPath)
126 files.sort()
127
128 return files[-COMPARE_NUM_RESULTS:]
129
130def parseAsCSV (logPath, config):
131 args = [config.getBinaryPath(LOG_TO_CSV_PATH), "--mode=all", "--format=csv", logPath]
132 proc = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
133 out, err = proc.communicate()
134 return out
135
136def computeUnifiedTestCaseList (batchResults):
137 caseList = []
138 caseSet = set()
139
140 for batchResult in batchResults:
141 for result in batchResult.results:
142 if not result.name in caseSet:
143 caseList.append(result.name)
144 caseSet.add(result.name)
145
146 return caseList
147
148def computeUnifiedResults (batchResults):
149
150 def genResultMap (batchResult):
151 resMap = {}
152 for result in batchResult.results:
153 resMap[result.name] = result
154 return resMap
155
156 resultMap = [genResultMap(r) for r in batchResults]
157 caseList = computeUnifiedTestCaseList(batchResults)
158 results = []
159
160 for caseName in caseList:
161 statusCodes = []
162
163 for i in range(0, len(batchResults)):
164 result = resultMap[i][caseName] if caseName in resultMap[i] else None
165 statusCode = result.statusCode if result != None else 'Missing'
166 statusCodes.append(statusCode)
167
168 results.append(MultiResult(caseName, statusCodes))
169
170 return results
171
172def allStatusCodesEqual (result):
173 firstCode = result.statusCodes[0]
174 for i in range(1, len(result.statusCodes)):
175 if result.statusCodes[i] != firstCode:
176 return False
177 return True
178
179def computeDiffResults (unifiedResults):
180 diff = []
181 for result in unifiedResults:
182 if not allStatusCodesEqual(result):
183 diff.append(result)
184 return diff
185
186def genCompareReport (batchResults, title, ignoreCases):
187 class TableRow:
188 def __init__ (self, testCaseName, innerHTML):
189 self.testCaseName = testCaseName
190 self.innerHTML = innerHTML
191
192 unifiedResults = computeUnifiedResults(batchResults)
193 diffResults = computeDiffResults(unifiedResults)
194 rows = []
195
196 # header
197 headerCol = '<td class="Header">Test case</td>\n'
198 for batchResult in batchResults:
199 headerCol += '<td class="Header">%s</td>\n' % batchResult.name
200 rows.append(TableRow(None, headerCol))
201
202 # results
203 for result in diffResults:
204 col = '<td class="CasePath">%s</td>\n' % result.name
205 for statusCode in result.statusCodes:
206 col += '<td class="%s">%s</td>\n' % (statusCode, statusCode)
207
208 rows.append(TableRow(result.name, col))
209
210 tableStr = ""
211 for row in rows:
212 if row.testCaseName is not None and matchesAnyPattern(row.testCaseName, ignoreCases):
213 tableStr += '<tr class="Ignored">\n%s</tr>\n' % row.innerHTML
214 else:
215 tableStr += '<tr>\n%s</tr>\n' % row.innerHTML
216
217 html = COMPARE_REPORT_TMPL
218 html = html.replace("${TITLE}", title)
219 html = html.replace("${RESULTS}", tableStr)
220
221 return html
222
223def matchesAnyPattern (name, patterns):
224 for pattern in patterns:
225 if fnmatch.fnmatch(name, pattern):
226 return True
227 return False
228
229def statusCodesMatch (refResult, resResult):
230 return refResult == 'Missing' or resResult == 'Missing' or refResult == resResult
231
232def compareBatchResults (referenceBatch, resultBatch, ignoreCases):
233 unifiedResults = computeUnifiedResults([referenceBatch, resultBatch])
234 failedCases = []
235
236 for result in unifiedResults:
237 if not matchesAnyPattern(result.name, ignoreCases):
238 refResult = result.statusCodes[0]
239 resResult = result.statusCodes[1]
240
241 if not statusCodesMatch(refResult, resResult):
242 failedCases.append(result)
243
244 return failedCases
245
246def getUnusedPort ():
247 # \note Not 100%-proof method as other apps may grab this port before we launch execserver
248 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
249 s.bind(('localhost', 0))
250 addr, port = s.getsockname()
251 s.close()
252 return port
253
254def runNightly (config):
255 build(config.buildConfig, config.generator)
256
257 # Run parameters
258 timestamp = getResultTimestamp()
259 logDir = os.path.join(BASE_LOGS_DIR, config.name)
260 testLogPath = os.path.join(logDir, timestamp + ".qpa")
261 infoLogPath = os.path.join(logDir, timestamp + ".txt")
262 csvLogPath = os.path.join(logDir, timestamp + ".csv")
263 compareLogPath = os.path.join(BASE_REFS_DIR, config.name + ".csv")
264 port = getUnusedPort()
265
266 if not os.path.exists(logDir):
267 os.makedirs(logDir)
268
269 if os.path.exists(testLogPath) or os.path.exists(infoLogPath):
270 raise Exception("Result '%s' already exists", timestamp)
271
272 # Paths, etc.
273 binaryName = config.generator.getBinaryPath(config.buildConfig.getBuildType(), os.path.basename(config.binaryName))
274 workingDir = os.path.join(config.buildConfig.getBuildDir(), os.path.dirname(config.binaryName))
275
276 execArgs = [
277 config.getBinaryPath(EXECUTOR_PATH),
278 '--start-server=%s' % config.getBinaryPath(EXECSERVER_PATH),
279 '--port=%d' % port,
280 '--binaryname=%s' % binaryName,
281 '--cmdline=%s' % string.join([shellquote(arg) for arg in config.args], " "),
282 '--workdir=%s' % workingDir,
283 '--caselistdir=%s' % CASELIST_PATH,
284 '--testset=%s' % string.join(config.testset, ","),
285 '--out=%s' % testLogPath,
286 '--info=%s' % infoLogPath,
287 '--summary=no'
288 ]
289
290 if len(config.exclude) > 0:
291 execArgs += ['--exclude=%s' % string.join(config.exclude, ",")]
292
293 execute(execArgs)
294
295 # Translate to CSV for comparison purposes
296 lastResultCsv = parseAsCSV(testLogPath, config)
297 writeFile(csvLogPath, lastResultCsv)
298
299 if os.path.exists(compareLogPath):
300 refBatchResult = readBatchResultFromCSV(compareLogPath, "reference")
301 else:
302 refBatchResult = None
303
304 # Generate comparison report
305 compareFilenames = getCompareFilenames(logDir)
306 batchResults = [readBatchResultFromCSV(filename) for filename in compareFilenames]
307
308 if refBatchResult != None:
309 batchResults = [refBatchResult] + batchResults
310
311 writeFile(COMPARE_REPORT_NAME, genCompareReport(batchResults, config.name, config.ignore))
312 print "Comparison report written to %s" % COMPARE_REPORT_NAME
313
314 # Compare to reference
315 if refBatchResult != None:
316 curBatchResult = BatchResult("current")
317 curBatchResult.results = parseResultCsv(lastResultCsv)
318 failedCases = compareBatchResults(refBatchResult, curBatchResult, config.ignore)
319
320 print ""
321 for result in failedCases:
322 print "MISMATCH: %s: expected %s, got %s" % (result.name, result.statusCodes[0], result.statusCodes[1])
323
324 print ""
325 print "%d / %d cases passed, run %s" % (len(curBatchResult.results)-len(failedCases), len(curBatchResult.results), "FAILED" if len(failedCases) > 0 else "passed")
326
327 if len(failedCases) > 0:
328 return False
329
330 return True
331
332# Configurations
333
334DEFAULT_WIN32_GENERATOR = ANY_VS_X32_GENERATOR
335DEFAULT_WIN64_GENERATOR = ANY_VS_X64_GENERATOR
336
337WGL_X64_RELEASE_BUILD_CFG = NightlyBuildConfig("wgl_x64_release", "Release", ['-DDEQP_TARGET=win32_wgl'])
338ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG = NightlyBuildConfig("arm_gles3_emu_release", "Release", ['-DDEQP_TARGET=arm_gles3_emu'])
339
340BASE_ARGS = ['--deqp-visibility=hidden', '--deqp-watchdog=enable', '--deqp-crashhandler=enable']
341
342CONFIGS = [
343 NightlyRunConfig(
344 name = "wgl_x64_release_gles2",
345 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
346 generator = DEFAULT_WIN64_GENERATOR,
347 binaryName = "modules/gles2/deqp-gles2",
348 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
349 testset = ["dEQP-GLES2.info.*", "dEQP-GLES2.functional.*", "dEQP-GLES2.usecases.*"],
350 exclude = [
351 "dEQP-GLES2.functional.shaders.loops.*while*unconditional_continue*",
352 "dEQP-GLES2.functional.shaders.loops.*while*only_continue*",
353 "dEQP-GLES2.functional.shaders.loops.*while*double_continue*",
354 ],
355 ignore = []
356 ),
357 NightlyRunConfig(
358 name = "wgl_x64_release_gles3",
359 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
360 generator = DEFAULT_WIN64_GENERATOR,
361 binaryName = "modules/gles3/deqp-gles3",
362 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
363 testset = ["dEQP-GLES3.info.*", "dEQP-GLES3.functional.*", "dEQP-GLES3.usecases.*"],
364 exclude = [
365 "dEQP-GLES3.functional.shaders.loops.*while*unconditional_continue*",
366 "dEQP-GLES3.functional.shaders.loops.*while*only_continue*",
367 "dEQP-GLES3.functional.shaders.loops.*while*double_continue*",
368 ],
369 ignore = [
370 "dEQP-GLES3.functional.transform_feedback.*",
371 "dEQP-GLES3.functional.occlusion_query.*",
372 "dEQP-GLES3.functional.lifetime.*",
373 "dEQP-GLES3.functional.fragment_ops.depth_stencil.stencil_ops",
374 ]
375 ),
376 NightlyRunConfig(
377 name = "wgl_x64_release_gles31",
378 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
379 generator = DEFAULT_WIN64_GENERATOR,
380 binaryName = "modules/gles31/deqp-gles31",
381 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
382 testset = ["dEQP-GLES31.*"],
383 exclude = [],
384 ignore = [
385 "dEQP-GLES31.functional.draw_indirect.negative.command_bad_alignment_3",
386 "dEQP-GLES31.functional.draw_indirect.negative.command_offset_not_in_buffer",
387 "dEQP-GLES31.functional.vertex_attribute_binding.negative.bind_vertex_buffer_negative_offset",
388 "dEQP-GLES31.functional.ssbo.layout.single_basic_type.packed.mediump_uint",
389 "dEQP-GLES31.functional.blend_equation_advanced.basic.*",
390 "dEQP-GLES31.functional.blend_equation_advanced.srgb.*",
391 "dEQP-GLES31.functional.blend_equation_advanced.barrier.*",
392 "dEQP-GLES31.functional.uniform_location.*",
393 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_framebuffer_attachment_parameteriv",
394 "dEQP-GLES31.functional.debug.negative_coverage.log.state.get_renderbuffer_parameteriv",
395 "dEQP-GLES31.functional.debug.error_filters.case_0",
396 "dEQP-GLES31.functional.debug.error_filters.case_2",
397 ]
398 ),
399 NightlyRunConfig(
400 name = "wgl_x64_release_gl3",
401 buildConfig = WGL_X64_RELEASE_BUILD_CFG,
402 generator = DEFAULT_WIN64_GENERATOR,
403 binaryName = "modules/gl3/deqp-gl3",
404 args = ['--deqp-gl-config-name=rgba8888d24s8ms0'] + BASE_ARGS,
405 testset = ["dEQP-GL3.info.*", "dEQP-GL3.functional.*"],
406 exclude = [
407 "dEQP-GL3.functional.shaders.loops.*while*unconditional_continue*",
408 "dEQP-GL3.functional.shaders.loops.*while*only_continue*",
409 "dEQP-GL3.functional.shaders.loops.*while*double_continue*",
410 ],
411 ignore = [
412 "dEQP-GL3.functional.transform_feedback.*"
413 ]
414 ),
415 NightlyRunConfig(
416 name = "arm_gles3_emu_x32_egl",
417 buildConfig = ARM_GLES3_EMU_X32_RELEASE_BUILD_CFG,
418 generator = DEFAULT_WIN32_GENERATOR,
419 binaryName = "modules/egl/deqp-egl",
420 args = BASE_ARGS,
421 testset = ["dEQP-EGL.info.*", "dEQP-EGL.functional.*"],
422 exclude = [
423 "dEQP-EGL.functional.sharing.gles2.multithread.*",
424 "dEQP-EGL.functional.multithread.*",
425 ],
426 ignore = []
427 ),
428 NightlyRunConfig(
429 name = "opencl_x64_release",
430 buildConfig = NightlyBuildConfig("opencl_x64_release", "Release", ['-DDEQP_TARGET=opencl_icd']),
431 generator = DEFAULT_WIN64_GENERATOR,
432 binaryName = "modules/opencl/deqp-opencl",
433 args = ['--deqp-cl-platform-id=2 --deqp-cl-device-ids=1'] + BASE_ARGS,
434 testset = ["dEQP-CL.*"],
435 exclude = ["dEQP-CL.performance.*", "dEQP-CL.robustness.*", "dEQP-CL.stress.memory.*"],
436 ignore = [
437 "dEQP-CL.scheduler.random.*",
438 "dEQP-CL.language.set_kernel_arg.random_structs.*",
439 "dEQP-CL.language.builtin_function.work_item.invalid_get_global_offset",
440 "dEQP-CL.language.call_function.arguments.random_structs.*",
441 "dEQP-CL.language.call_kernel.random_structs.*",
442 "dEQP-CL.language.inf_nan.nan.frexp.float",
443 "dEQP-CL.language.inf_nan.nan.lgamma_r.float",
444 "dEQP-CL.language.inf_nan.nan.modf.float",
445 "dEQP-CL.language.inf_nan.nan.sqrt.float",
446 "dEQP-CL.api.multithread.*",
447 "dEQP-CL.api.callback.random.nested.*",
448 "dEQP-CL.api.memory_migration.out_of_order_host.image2d.single_device_kernel_migrate_validate_abb",
449 "dEQP-CL.api.memory_migration.out_of_order.image2d.single_device_kernel_migrate_kernel_validate_abbb",
450 "dEQP-CL.image.addressing_filtering12.1d_array.*",
451 "dEQP-CL.image.addressing_filtering12.2d_array.*"
452 ]
453 )
454]
455
456if __name__ == "__main__":
457 config = None
458
459 if len(sys.argv) == 2:
460 cfgName = sys.argv[1]
461 for curCfg in CONFIGS:
462 if curCfg.name == cfgName:
463 config = curCfg
464 break
465
466 if config != None:
467 isOk = runNightly(config)
468 if not isOk:
469 sys.exit(-1)
470 else:
471 print "%s: [config]" % sys.argv[0]
472 print ""
473 print " Available configs:"
474 for config in CONFIGS:
475 print " %s" % config.name
476 sys.exit(-1)