Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 1 | # -*- coding: utf-8 -*- |
| 2 | |
| 3 | #------------------------------------------------------------------------- |
| 4 | # drawElements Quality Program utilities |
| 5 | # -------------------------------------- |
| 6 | # |
| 7 | # Copyright 2016 The Android Open Source Project |
| 8 | # |
| 9 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 10 | # you may not use this file except in compliance with the License. |
| 11 | # You may obtain a copy of the License at |
| 12 | # |
| 13 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 14 | # |
| 15 | # Unless required by applicable law or agreed to in writing, software |
| 16 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 18 | # See the License for the specific language governing permissions and |
| 19 | # limitations under the License. |
| 20 | # |
| 21 | #------------------------------------------------------------------------- |
| 22 | |
| 23 | from build.common import * |
| 24 | from build.config import ANY_GENERATOR |
| 25 | from build.build import build |
| 26 | from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET |
| 27 | from fnmatch import fnmatch |
| 28 | from copy import copy |
| 29 | |
| 30 | import xml.etree.cElementTree as ElementTree |
| 31 | import xml.dom.minidom as minidom |
| 32 | |
Pyry Haulos | f189365 | 2016-08-16 12:35:51 +0200 | [diff] [blame] | 33 | APK_NAME = "com.drawelements.deqp.apk" |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 34 | |
| 35 | GENERATED_FILE_WARNING = """ |
| 36 | This file has been automatically generated. Edit with caution. |
| 37 | """ |
| 38 | |
| 39 | class Project: |
| 40 | def __init__ (self, path, copyright = None): |
| 41 | self.path = path |
| 42 | self.copyright = copyright |
| 43 | |
| 44 | class Configuration: |
Pyry Haulos | e232a6e | 2016-08-23 15:37:44 -0700 | [diff] [blame^] | 45 | def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, required = False, runtime = None): |
| 46 | self.name = name |
| 47 | self.glconfig = glconfig |
| 48 | self.rotation = rotation |
| 49 | self.surfacetype = surfacetype |
| 50 | self.required = required |
| 51 | self.filters = filters |
| 52 | self.expectedRuntime = runtime |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 53 | |
| 54 | class Package: |
| 55 | def __init__ (self, module, configurations): |
| 56 | self.module = module |
| 57 | self.configurations = configurations |
| 58 | |
| 59 | class Mustpass: |
| 60 | def __init__ (self, project, version, packages): |
| 61 | self.project = project |
| 62 | self.version = version |
| 63 | self.packages = packages |
| 64 | |
| 65 | class Filter: |
| 66 | TYPE_INCLUDE = 0 |
| 67 | TYPE_EXCLUDE = 1 |
| 68 | |
| 69 | def __init__ (self, type, filename): |
| 70 | self.type = type |
| 71 | self.filename = filename |
| 72 | |
| 73 | class TestRoot: |
| 74 | def __init__ (self): |
| 75 | self.children = [] |
| 76 | |
| 77 | class TestGroup: |
| 78 | def __init__ (self, name): |
| 79 | self.name = name |
| 80 | self.children = [] |
| 81 | |
| 82 | class TestCase: |
| 83 | def __init__ (self, name): |
| 84 | self.name = name |
| 85 | self.configurations = [] |
| 86 | |
| 87 | class GLESVersion: |
| 88 | def __init__(self, major, minor): |
| 89 | self.major = major |
| 90 | self.minor = minor |
| 91 | |
| 92 | def encode (self): |
| 93 | return (self.major << 16) | (self.minor) |
| 94 | |
| 95 | def getModuleGLESVersion (module): |
| 96 | versions = { |
| 97 | 'dEQP-EGL': GLESVersion(2,0), |
| 98 | 'dEQP-GLES2': GLESVersion(2,0), |
| 99 | 'dEQP-GLES3': GLESVersion(3,0), |
| 100 | 'dEQP-GLES31': GLESVersion(3,1) |
| 101 | } |
| 102 | return versions[module.name] if module.name in versions else None |
| 103 | |
| 104 | def getSrcDir (mustpass): |
| 105 | return os.path.join(mustpass.project.path, mustpass.version, "src") |
| 106 | |
| 107 | def getTmpDir (mustpass): |
| 108 | return os.path.join(mustpass.project.path, mustpass.version, "tmp") |
| 109 | |
| 110 | def getModuleShorthand (module): |
| 111 | assert module.name[:5] == "dEQP-" |
| 112 | return module.name[5:].lower() |
| 113 | |
| 114 | def getCaseListFileName (package, configuration): |
| 115 | return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) |
| 116 | |
| 117 | def getDstCaseListPath (mustpass, package, configuration): |
| 118 | return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration)) |
| 119 | |
| 120 | def getCTSPackageName (package): |
| 121 | return "com.drawelements.deqp." + getModuleShorthand(package.module) |
| 122 | |
| 123 | def getCommandLine (config): |
| 124 | cmdLine = "" |
| 125 | |
| 126 | if config.glconfig != None: |
| 127 | cmdLine += "--deqp-gl-config-name=%s " % config.glconfig |
| 128 | |
| 129 | if config.rotation != None: |
| 130 | cmdLine += "--deqp-screen-rotation=%s " % config.rotation |
| 131 | |
| 132 | if config.surfacetype != None: |
| 133 | cmdLine += "--deqp-surface-type=%s " % config.surfacetype |
| 134 | |
| 135 | cmdLine += "--deqp-watchdog=enable" |
| 136 | |
| 137 | return cmdLine |
| 138 | |
| 139 | def readCaseList (filename): |
| 140 | cases = [] |
| 141 | with open(filename, 'rb') as f: |
| 142 | for line in f: |
| 143 | if line[:6] == "TEST: ": |
| 144 | cases.append(line[6:].strip()) |
| 145 | return cases |
| 146 | |
| 147 | def getCaseList (buildCfg, generator, module): |
| 148 | build(buildCfg, generator, [module.binName]) |
| 149 | genCaseList(buildCfg, generator, module, "txt") |
| 150 | return readCaseList(getCaseListPath(buildCfg, module, "txt")) |
| 151 | |
| 152 | def readPatternList (filename): |
| 153 | ptrns = [] |
| 154 | with open(filename, 'rb') as f: |
| 155 | for line in f: |
| 156 | line = line.strip() |
| 157 | if len(line) > 0 and line[0] != '#': |
| 158 | ptrns.append(line) |
| 159 | return ptrns |
| 160 | |
| 161 | def applyPatterns (caseList, patterns, filename, op): |
| 162 | matched = set() |
| 163 | errors = [] |
| 164 | curList = copy(caseList) |
| 165 | trivialPtrns = [p for p in patterns if p.find('*') < 0] |
| 166 | regularPtrns = [p for p in patterns if p.find('*') >= 0] |
| 167 | |
| 168 | # Apply trivial (just case paths) |
| 169 | allCasesSet = set(caseList) |
| 170 | for path in trivialPtrns: |
| 171 | if path in allCasesSet: |
| 172 | if path in matched: |
| 173 | errors.append((path, "Same case specified more than once")) |
| 174 | matched.add(path) |
| 175 | else: |
| 176 | errors.append((path, "Test case not found")) |
| 177 | |
| 178 | curList = [c for c in curList if c not in matched] |
| 179 | |
| 180 | for pattern in regularPtrns: |
| 181 | matchedThisPtrn = set() |
| 182 | |
| 183 | for case in curList: |
| 184 | if fnmatch(case, pattern): |
| 185 | matchedThisPtrn.add(case) |
| 186 | |
| 187 | if len(matchedThisPtrn) == 0: |
| 188 | errors.append((pattern, "Pattern didn't match any cases")) |
| 189 | |
| 190 | matched = matched | matchedThisPtrn |
| 191 | curList = [c for c in curList if c not in matched] |
| 192 | |
| 193 | for pattern, reason in errors: |
| 194 | print "ERROR: %s: %s" % (reason, pattern) |
| 195 | |
| 196 | if len(errors) > 0: |
| 197 | die("Found %s invalid patterns while processing file %s" % (len(errors), filename)) |
| 198 | |
| 199 | return [c for c in caseList if op(c in matched)] |
| 200 | |
| 201 | def applyInclude (caseList, patterns, filename): |
| 202 | return applyPatterns(caseList, patterns, filename, lambda b: b) |
| 203 | |
| 204 | def applyExclude (caseList, patterns, filename): |
| 205 | return applyPatterns(caseList, patterns, filename, lambda b: not b) |
| 206 | |
| 207 | def readPatternLists (mustpass): |
| 208 | lists = {} |
| 209 | for package in mustpass.packages: |
| 210 | for cfg in package.configurations: |
| 211 | for filter in cfg.filters: |
| 212 | if not filter.filename in lists: |
| 213 | lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) |
| 214 | return lists |
| 215 | |
| 216 | def applyFilters (caseList, patternLists, filters): |
| 217 | res = copy(caseList) |
| 218 | for filter in filters: |
| 219 | ptrnList = patternLists[filter.filename] |
| 220 | if filter.type == Filter.TYPE_INCLUDE: |
| 221 | res = applyInclude(res, ptrnList, filter.filename) |
| 222 | else: |
| 223 | assert filter.type == Filter.TYPE_EXCLUDE |
| 224 | res = applyExclude(res, ptrnList, filter.filename) |
| 225 | return res |
| 226 | |
| 227 | def appendToHierarchy (root, casePath): |
| 228 | def findChild (node, name): |
| 229 | for child in node.children: |
| 230 | if child.name == name: |
| 231 | return child |
| 232 | return None |
| 233 | |
| 234 | curNode = root |
| 235 | components = casePath.split('.') |
| 236 | |
| 237 | for component in components[:-1]: |
| 238 | nextNode = findChild(curNode, component) |
| 239 | if not nextNode: |
| 240 | nextNode = TestGroup(component) |
| 241 | curNode.children.append(nextNode) |
| 242 | curNode = nextNode |
| 243 | |
| 244 | if not findChild(curNode, components[-1]): |
| 245 | curNode.children.append(TestCase(components[-1])) |
| 246 | |
| 247 | def buildTestHierachy (caseList): |
| 248 | root = TestRoot() |
| 249 | for case in caseList: |
| 250 | appendToHierarchy(root, case) |
| 251 | return root |
| 252 | |
| 253 | def buildTestCaseMap (root): |
| 254 | caseMap = {} |
| 255 | |
| 256 | def recursiveBuild (curNode, prefix): |
| 257 | curPath = prefix + curNode.name |
| 258 | if isinstance(curNode, TestCase): |
| 259 | caseMap[curPath] = curNode |
| 260 | else: |
| 261 | for child in curNode.children: |
| 262 | recursiveBuild(child, curPath + '.') |
| 263 | |
| 264 | for child in root.children: |
| 265 | recursiveBuild(child, '') |
| 266 | |
| 267 | return caseMap |
| 268 | |
| 269 | def include (filename): |
| 270 | return Filter(Filter.TYPE_INCLUDE, filename) |
| 271 | |
| 272 | def exclude (filename): |
| 273 | return Filter(Filter.TYPE_EXCLUDE, filename) |
| 274 | |
| 275 | def insertXMLHeaders (mustpass, doc): |
| 276 | if mustpass.project.copyright != None: |
| 277 | doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) |
| 278 | doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) |
| 279 | |
| 280 | def prettifyXML (doc): |
| 281 | uglyString = ElementTree.tostring(doc, 'utf-8') |
| 282 | reparsed = minidom.parseString(uglyString) |
| 283 | return reparsed.toprettyxml(indent='\t', encoding='utf-8') |
| 284 | |
| 285 | def genCTSPackageXML (mustpass, package, root): |
| 286 | def isLeafGroup (testGroup): |
| 287 | numGroups = 0 |
| 288 | numTests = 0 |
| 289 | |
| 290 | for child in testGroup.children: |
| 291 | if isinstance(child, TestCase): |
| 292 | numTests += 1 |
| 293 | else: |
| 294 | numGroups += 1 |
| 295 | |
| 296 | assert numGroups + numTests > 0 |
| 297 | |
| 298 | if numGroups > 0 and numTests > 0: |
| 299 | die("Mixed groups and cases in %s" % testGroup.name) |
| 300 | |
| 301 | return numGroups == 0 |
| 302 | |
| 303 | def makeConfiguration (parentElem, config): |
| 304 | attributes = {} |
| 305 | |
| 306 | if config.glconfig != None: |
| 307 | attributes['glconfig'] = config.glconfig |
| 308 | |
| 309 | if config.rotation != None: |
| 310 | attributes['rotation'] = config.rotation |
| 311 | |
| 312 | if config.surfacetype != None: |
| 313 | attributes['surfacetype'] = config.surfacetype |
| 314 | |
| 315 | return ElementTree.SubElement(parentElem, "TestInstance", attributes) |
| 316 | |
| 317 | def makeTestCase (parentElem, testCase): |
| 318 | caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name) |
| 319 | for config in testCase.configurations: |
| 320 | makeConfiguration(caseElem, config) |
| 321 | return caseElem |
| 322 | |
| 323 | def makeTestGroup (parentElem, testGroup): |
| 324 | groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name) |
| 325 | for child in testGroup.children: |
| 326 | if isinstance(child, TestCase): |
| 327 | makeTestCase(groupElem, child) |
| 328 | else: |
| 329 | makeTestGroup(groupElem, child) |
| 330 | return groupElem |
| 331 | |
| 332 | pkgElem = ElementTree.Element("TestPackage", |
| 333 | name = package.module.name, |
| 334 | appPackageName = getCTSPackageName(package), |
| 335 | testType = "deqpTest") |
| 336 | |
| 337 | pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp") |
| 338 | insertXMLHeaders(mustpass, pkgElem) |
| 339 | |
| 340 | glesVersion = getModuleGLESVersion(package.module) |
| 341 | |
| 342 | if glesVersion != None: |
| 343 | pkgElem.set("deqp:glesVersion", str(glesVersion.encode())) |
| 344 | |
| 345 | for child in root.children: |
| 346 | makeTestGroup(pkgElem, child) |
| 347 | |
| 348 | return pkgElem |
| 349 | |
| 350 | def genSpecXML (mustpass): |
| 351 | mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) |
| 352 | insertXMLHeaders(mustpass, mustpassElem) |
| 353 | |
| 354 | for package in mustpass.packages: |
| 355 | packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) |
| 356 | |
| 357 | for config in package.configurations: |
| 358 | configElem = ElementTree.SubElement(packageElem, "Configuration", |
| 359 | name = config.name, |
| 360 | caseListFile = getCaseListFileName(package, config), |
| 361 | commandLine = getCommandLine(config)) |
| 362 | |
| 363 | return mustpassElem |
| 364 | |
| 365 | def addOptionElement (parent, optionName, optionValue): |
| 366 | ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) |
| 367 | |
| 368 | def genAndroidTestXml (mustpass): |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 369 | RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" |
| 370 | configElement = ElementTree.Element("configuration") |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 371 | |
| 372 | for package in mustpass.packages: |
| 373 | for config in package.configurations: |
| 374 | testElement = ElementTree.SubElement(configElement, "test") |
| 375 | testElement.set("class", RUNNER_CLASS) |
| 376 | addOptionElement(testElement, "deqp-package", package.module.name) |
| 377 | addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config)) |
| 378 | # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. |
| 379 | if config.glconfig != None: |
| 380 | addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) |
| 381 | |
| 382 | if config.surfacetype != None: |
| 383 | addOptionElement(testElement, "deqp-surface-type", config.surfacetype) |
| 384 | |
| 385 | if config.rotation != None: |
| 386 | addOptionElement(testElement, "deqp-screen-rotation", config.rotation) |
| 387 | |
Kalle Raita | 2519661 | 2016-04-04 17:09:44 -0700 | [diff] [blame] | 388 | if config.expectedRuntime != None: |
| 389 | addOptionElement(testElement, "runtime-hint", config.expectedRuntime) |
| 390 | |
Pyry Haulos | e232a6e | 2016-08-23 15:37:44 -0700 | [diff] [blame^] | 391 | if config.required: |
| 392 | addOptionElement(testElement, "deqp-config-required", "true") |
| 393 | |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 394 | insertXMLHeaders(mustpass, configElement) |
| 395 | |
| 396 | return configElement |
| 397 | |
| 398 | def genMustpass (mustpass, moduleCaseLists): |
| 399 | print "Generating mustpass '%s'" % mustpass.version |
| 400 | |
| 401 | patternLists = readPatternLists(mustpass) |
| 402 | |
| 403 | for package in mustpass.packages: |
| 404 | allCasesInPkg = moduleCaseLists[package.module] |
| 405 | matchingByConfig = {} |
| 406 | allMatchingSet = set() |
| 407 | |
| 408 | for config in package.configurations: |
| 409 | filtered = applyFilters(allCasesInPkg, patternLists, config.filters) |
| 410 | dstFile = getDstCaseListPath(mustpass, package, config) |
| 411 | |
| 412 | print " Writing deqp caselist: " + dstFile |
| 413 | writeFile(dstFile, "\n".join(filtered) + "\n") |
| 414 | |
| 415 | matchingByConfig[config] = filtered |
| 416 | allMatchingSet = allMatchingSet | set(filtered) |
| 417 | |
| 418 | allMatchingCases = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering |
| 419 | root = buildTestHierachy(allMatchingCases) |
| 420 | testCaseMap = buildTestCaseMap(root) |
| 421 | |
| 422 | for config in package.configurations: |
| 423 | for case in matchingByConfig[config]: |
| 424 | testCaseMap[case].configurations.append(config) |
| 425 | |
| 426 | # NOTE: CTS v2 does not need package XML files. Remove when transition is complete. |
| 427 | packageXml = genCTSPackageXML(mustpass, package, root) |
| 428 | xmlFilename = os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml") |
| 429 | |
| 430 | print " Writing CTS caselist: " + xmlFilename |
| 431 | writeFile(xmlFilename, prettifyXML(packageXml)) |
| 432 | |
| 433 | specXML = genSpecXML(mustpass) |
| 434 | specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") |
| 435 | |
| 436 | print " Writing spec: " + specFilename |
| 437 | writeFile(specFilename, prettifyXML(specXML)) |
| 438 | |
| 439 | # TODO: Which is the best selector mechanism? |
Mika Isojärvi | fb2d85c | 2016-02-04 15:35:09 -0800 | [diff] [blame] | 440 | if (mustpass.version == "master"): |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 441 | androidTestXML = genAndroidTestXml(mustpass) |
| 442 | androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") |
| 443 | |
| 444 | print " Writing AndroidTest.xml: " + androidTestFilename |
| 445 | writeFile(androidTestFilename, prettifyXML(androidTestXML)) |
| 446 | |
| 447 | print "Done!" |
| 448 | |
| 449 | def genMustpassLists (mustpassLists, generator, buildCfg): |
| 450 | moduleCaseLists = {} |
| 451 | |
| 452 | # Getting case lists involves invoking build, so we want to cache the results |
| 453 | for mustpass in mustpassLists: |
| 454 | for package in mustpass.packages: |
| 455 | if not package.module in moduleCaseLists: |
| 456 | moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module) |
| 457 | |
| 458 | for mustpass in mustpassLists: |
| 459 | genMustpass(mustpass, moduleCaseLists) |