Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 1 | # -*- coding: utf-8 -*- |
| 2 | |
| 3 | #------------------------------------------------------------------------- |
| 4 | # drawElements Quality Program utilities |
| 5 | # -------------------------------------- |
| 6 | # |
| 7 | # Copyright 2016 The Android Open Source Project |
| 8 | # |
| 9 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 10 | # you may not use this file except in compliance with the License. |
| 11 | # You may obtain a copy of the License at |
| 12 | # |
| 13 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 14 | # |
| 15 | # Unless required by applicable law or agreed to in writing, software |
| 16 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 17 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 18 | # See the License for the specific language governing permissions and |
| 19 | # limitations under the License. |
| 20 | # |
| 21 | #------------------------------------------------------------------------- |
| 22 | |
| 23 | from build.common import * |
| 24 | from build.config import ANY_GENERATOR |
| 25 | from build.build import build |
| 26 | from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET |
| 27 | from fnmatch import fnmatch |
| 28 | from copy import copy |
| 29 | |
| 30 | import xml.etree.cElementTree as ElementTree |
| 31 | import xml.dom.minidom as minidom |
| 32 | |
| 33 | APK_NAME = "com.drawelements.deqp.apk" |
| 34 | |
| 35 | GENERATED_FILE_WARNING = """ |
| 36 | This file has been automatically generated. Edit with caution. |
| 37 | """ |
| 38 | |
| 39 | class Project: |
| 40 | def __init__ (self, path, copyright = None): |
| 41 | self.path = path |
| 42 | self.copyright = copyright |
| 43 | |
| 44 | class Configuration: |
Kalle Raita | 2519661 | 2016-04-04 17:09:44 -0700 | [diff] [blame] | 45 | def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, runtime = None): |
| 46 | self.name = name |
| 47 | self.glconfig = glconfig |
| 48 | self.rotation = rotation |
| 49 | self.surfacetype = surfacetype |
| 50 | self.filters = filters |
| 51 | self.expectedRuntime = runtime |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 52 | |
| 53 | class Package: |
| 54 | def __init__ (self, module, configurations): |
| 55 | self.module = module |
| 56 | self.configurations = configurations |
| 57 | |
| 58 | class Mustpass: |
| 59 | def __init__ (self, project, version, packages): |
| 60 | self.project = project |
| 61 | self.version = version |
| 62 | self.packages = packages |
| 63 | |
| 64 | class Filter: |
| 65 | TYPE_INCLUDE = 0 |
| 66 | TYPE_EXCLUDE = 1 |
| 67 | |
| 68 | def __init__ (self, type, filename): |
| 69 | self.type = type |
| 70 | self.filename = filename |
| 71 | |
| 72 | class TestRoot: |
| 73 | def __init__ (self): |
| 74 | self.children = [] |
| 75 | |
| 76 | class TestGroup: |
| 77 | def __init__ (self, name): |
| 78 | self.name = name |
| 79 | self.children = [] |
| 80 | |
| 81 | class TestCase: |
| 82 | def __init__ (self, name): |
| 83 | self.name = name |
| 84 | self.configurations = [] |
| 85 | |
| 86 | class GLESVersion: |
| 87 | def __init__(self, major, minor): |
| 88 | self.major = major |
| 89 | self.minor = minor |
| 90 | |
| 91 | def encode (self): |
| 92 | return (self.major << 16) | (self.minor) |
| 93 | |
| 94 | def getModuleGLESVersion (module): |
| 95 | versions = { |
| 96 | 'dEQP-EGL': GLESVersion(2,0), |
| 97 | 'dEQP-GLES2': GLESVersion(2,0), |
| 98 | 'dEQP-GLES3': GLESVersion(3,0), |
| 99 | 'dEQP-GLES31': GLESVersion(3,1) |
| 100 | } |
| 101 | return versions[module.name] if module.name in versions else None |
| 102 | |
| 103 | def getSrcDir (mustpass): |
| 104 | return os.path.join(mustpass.project.path, mustpass.version, "src") |
| 105 | |
| 106 | def getTmpDir (mustpass): |
| 107 | return os.path.join(mustpass.project.path, mustpass.version, "tmp") |
| 108 | |
| 109 | def getModuleShorthand (module): |
| 110 | assert module.name[:5] == "dEQP-" |
| 111 | return module.name[5:].lower() |
| 112 | |
| 113 | def getCaseListFileName (package, configuration): |
| 114 | return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name) |
| 115 | |
| 116 | def getDstCaseListPath (mustpass, package, configuration): |
| 117 | return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration)) |
| 118 | |
| 119 | def getCTSPackageName (package): |
| 120 | return "com.drawelements.deqp." + getModuleShorthand(package.module) |
| 121 | |
| 122 | def getCommandLine (config): |
| 123 | cmdLine = "" |
| 124 | |
| 125 | if config.glconfig != None: |
| 126 | cmdLine += "--deqp-gl-config-name=%s " % config.glconfig |
| 127 | |
| 128 | if config.rotation != None: |
| 129 | cmdLine += "--deqp-screen-rotation=%s " % config.rotation |
| 130 | |
| 131 | if config.surfacetype != None: |
| 132 | cmdLine += "--deqp-surface-type=%s " % config.surfacetype |
| 133 | |
| 134 | cmdLine += "--deqp-watchdog=enable" |
| 135 | |
| 136 | return cmdLine |
| 137 | |
| 138 | def readCaseList (filename): |
| 139 | cases = [] |
| 140 | with open(filename, 'rb') as f: |
| 141 | for line in f: |
| 142 | if line[:6] == "TEST: ": |
| 143 | cases.append(line[6:].strip()) |
| 144 | return cases |
| 145 | |
| 146 | def getCaseList (buildCfg, generator, module): |
| 147 | build(buildCfg, generator, [module.binName]) |
| 148 | genCaseList(buildCfg, generator, module, "txt") |
| 149 | return readCaseList(getCaseListPath(buildCfg, module, "txt")) |
| 150 | |
| 151 | def readPatternList (filename): |
| 152 | ptrns = [] |
| 153 | with open(filename, 'rb') as f: |
| 154 | for line in f: |
| 155 | line = line.strip() |
| 156 | if len(line) > 0 and line[0] != '#': |
| 157 | ptrns.append(line) |
| 158 | return ptrns |
| 159 | |
| 160 | def applyPatterns (caseList, patterns, filename, op): |
| 161 | matched = set() |
| 162 | errors = [] |
| 163 | curList = copy(caseList) |
| 164 | trivialPtrns = [p for p in patterns if p.find('*') < 0] |
| 165 | regularPtrns = [p for p in patterns if p.find('*') >= 0] |
| 166 | |
| 167 | # Apply trivial (just case paths) |
| 168 | allCasesSet = set(caseList) |
| 169 | for path in trivialPtrns: |
| 170 | if path in allCasesSet: |
| 171 | if path in matched: |
| 172 | errors.append((path, "Same case specified more than once")) |
| 173 | matched.add(path) |
| 174 | else: |
| 175 | errors.append((path, "Test case not found")) |
| 176 | |
| 177 | curList = [c for c in curList if c not in matched] |
| 178 | |
| 179 | for pattern in regularPtrns: |
| 180 | matchedThisPtrn = set() |
| 181 | |
| 182 | for case in curList: |
| 183 | if fnmatch(case, pattern): |
| 184 | matchedThisPtrn.add(case) |
| 185 | |
| 186 | if len(matchedThisPtrn) == 0: |
| 187 | errors.append((pattern, "Pattern didn't match any cases")) |
| 188 | |
| 189 | matched = matched | matchedThisPtrn |
| 190 | curList = [c for c in curList if c not in matched] |
| 191 | |
| 192 | for pattern, reason in errors: |
| 193 | print "ERROR: %s: %s" % (reason, pattern) |
| 194 | |
| 195 | if len(errors) > 0: |
| 196 | die("Found %s invalid patterns while processing file %s" % (len(errors), filename)) |
| 197 | |
| 198 | return [c for c in caseList if op(c in matched)] |
| 199 | |
| 200 | def applyInclude (caseList, patterns, filename): |
| 201 | return applyPatterns(caseList, patterns, filename, lambda b: b) |
| 202 | |
| 203 | def applyExclude (caseList, patterns, filename): |
| 204 | return applyPatterns(caseList, patterns, filename, lambda b: not b) |
| 205 | |
| 206 | def readPatternLists (mustpass): |
| 207 | lists = {} |
| 208 | for package in mustpass.packages: |
| 209 | for cfg in package.configurations: |
| 210 | for filter in cfg.filters: |
| 211 | if not filter.filename in lists: |
| 212 | lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename)) |
| 213 | return lists |
| 214 | |
| 215 | def applyFilters (caseList, patternLists, filters): |
| 216 | res = copy(caseList) |
| 217 | for filter in filters: |
| 218 | ptrnList = patternLists[filter.filename] |
| 219 | if filter.type == Filter.TYPE_INCLUDE: |
| 220 | res = applyInclude(res, ptrnList, filter.filename) |
| 221 | else: |
| 222 | assert filter.type == Filter.TYPE_EXCLUDE |
| 223 | res = applyExclude(res, ptrnList, filter.filename) |
| 224 | return res |
| 225 | |
| 226 | def appendToHierarchy (root, casePath): |
| 227 | def findChild (node, name): |
| 228 | for child in node.children: |
| 229 | if child.name == name: |
| 230 | return child |
| 231 | return None |
| 232 | |
| 233 | curNode = root |
| 234 | components = casePath.split('.') |
| 235 | |
| 236 | for component in components[:-1]: |
| 237 | nextNode = findChild(curNode, component) |
| 238 | if not nextNode: |
| 239 | nextNode = TestGroup(component) |
| 240 | curNode.children.append(nextNode) |
| 241 | curNode = nextNode |
| 242 | |
| 243 | if not findChild(curNode, components[-1]): |
| 244 | curNode.children.append(TestCase(components[-1])) |
| 245 | |
| 246 | def buildTestHierachy (caseList): |
| 247 | root = TestRoot() |
| 248 | for case in caseList: |
| 249 | appendToHierarchy(root, case) |
| 250 | return root |
| 251 | |
| 252 | def buildTestCaseMap (root): |
| 253 | caseMap = {} |
| 254 | |
| 255 | def recursiveBuild (curNode, prefix): |
| 256 | curPath = prefix + curNode.name |
| 257 | if isinstance(curNode, TestCase): |
| 258 | caseMap[curPath] = curNode |
| 259 | else: |
| 260 | for child in curNode.children: |
| 261 | recursiveBuild(child, curPath + '.') |
| 262 | |
| 263 | for child in root.children: |
| 264 | recursiveBuild(child, '') |
| 265 | |
| 266 | return caseMap |
| 267 | |
| 268 | def include (filename): |
| 269 | return Filter(Filter.TYPE_INCLUDE, filename) |
| 270 | |
| 271 | def exclude (filename): |
| 272 | return Filter(Filter.TYPE_EXCLUDE, filename) |
| 273 | |
| 274 | def insertXMLHeaders (mustpass, doc): |
| 275 | if mustpass.project.copyright != None: |
| 276 | doc.insert(0, ElementTree.Comment(mustpass.project.copyright)) |
| 277 | doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING)) |
| 278 | |
| 279 | def prettifyXML (doc): |
| 280 | uglyString = ElementTree.tostring(doc, 'utf-8') |
| 281 | reparsed = minidom.parseString(uglyString) |
| 282 | return reparsed.toprettyxml(indent='\t', encoding='utf-8') |
| 283 | |
| 284 | def genCTSPackageXML (mustpass, package, root): |
| 285 | def isLeafGroup (testGroup): |
| 286 | numGroups = 0 |
| 287 | numTests = 0 |
| 288 | |
| 289 | for child in testGroup.children: |
| 290 | if isinstance(child, TestCase): |
| 291 | numTests += 1 |
| 292 | else: |
| 293 | numGroups += 1 |
| 294 | |
| 295 | assert numGroups + numTests > 0 |
| 296 | |
| 297 | if numGroups > 0 and numTests > 0: |
| 298 | die("Mixed groups and cases in %s" % testGroup.name) |
| 299 | |
| 300 | return numGroups == 0 |
| 301 | |
| 302 | def makeConfiguration (parentElem, config): |
| 303 | attributes = {} |
| 304 | |
| 305 | if config.glconfig != None: |
| 306 | attributes['glconfig'] = config.glconfig |
| 307 | |
| 308 | if config.rotation != None: |
| 309 | attributes['rotation'] = config.rotation |
| 310 | |
| 311 | if config.surfacetype != None: |
| 312 | attributes['surfacetype'] = config.surfacetype |
| 313 | |
| 314 | return ElementTree.SubElement(parentElem, "TestInstance", attributes) |
| 315 | |
| 316 | def makeTestCase (parentElem, testCase): |
| 317 | caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name) |
| 318 | for config in testCase.configurations: |
| 319 | makeConfiguration(caseElem, config) |
| 320 | return caseElem |
| 321 | |
| 322 | def makeTestGroup (parentElem, testGroup): |
| 323 | groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name) |
| 324 | for child in testGroup.children: |
| 325 | if isinstance(child, TestCase): |
| 326 | makeTestCase(groupElem, child) |
| 327 | else: |
| 328 | makeTestGroup(groupElem, child) |
| 329 | return groupElem |
| 330 | |
| 331 | pkgElem = ElementTree.Element("TestPackage", |
| 332 | name = package.module.name, |
| 333 | appPackageName = getCTSPackageName(package), |
| 334 | testType = "deqpTest") |
| 335 | |
| 336 | pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp") |
| 337 | insertXMLHeaders(mustpass, pkgElem) |
| 338 | |
| 339 | glesVersion = getModuleGLESVersion(package.module) |
| 340 | |
| 341 | if glesVersion != None: |
| 342 | pkgElem.set("deqp:glesVersion", str(glesVersion.encode())) |
| 343 | |
| 344 | for child in root.children: |
| 345 | makeTestGroup(pkgElem, child) |
| 346 | |
| 347 | return pkgElem |
| 348 | |
| 349 | def genSpecXML (mustpass): |
| 350 | mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version) |
| 351 | insertXMLHeaders(mustpass, mustpassElem) |
| 352 | |
| 353 | for package in mustpass.packages: |
| 354 | packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name) |
| 355 | |
| 356 | for config in package.configurations: |
| 357 | configElem = ElementTree.SubElement(packageElem, "Configuration", |
| 358 | name = config.name, |
| 359 | caseListFile = getCaseListFileName(package, config), |
| 360 | commandLine = getCommandLine(config)) |
| 361 | |
| 362 | return mustpassElem |
| 363 | |
| 364 | def addOptionElement (parent, optionName, optionValue): |
| 365 | ElementTree.SubElement(parent, "option", name=optionName, value=optionValue) |
| 366 | |
| 367 | def genAndroidTestXml (mustpass): |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 368 | RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner" |
| 369 | configElement = ElementTree.Element("configuration") |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 370 | |
| 371 | for package in mustpass.packages: |
| 372 | for config in package.configurations: |
| 373 | testElement = ElementTree.SubElement(configElement, "test") |
| 374 | testElement.set("class", RUNNER_CLASS) |
| 375 | addOptionElement(testElement, "deqp-package", package.module.name) |
| 376 | addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config)) |
| 377 | # \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well. |
| 378 | if config.glconfig != None: |
| 379 | addOptionElement(testElement, "deqp-gl-config-name", config.glconfig) |
| 380 | |
| 381 | if config.surfacetype != None: |
| 382 | addOptionElement(testElement, "deqp-surface-type", config.surfacetype) |
| 383 | |
| 384 | if config.rotation != None: |
| 385 | addOptionElement(testElement, "deqp-screen-rotation", config.rotation) |
| 386 | |
Kalle Raita | 2519661 | 2016-04-04 17:09:44 -0700 | [diff] [blame] | 387 | if config.expectedRuntime != None: |
| 388 | addOptionElement(testElement, "runtime-hint", config.expectedRuntime) |
| 389 | |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 390 | insertXMLHeaders(mustpass, configElement) |
| 391 | |
| 392 | return configElement |
| 393 | |
| 394 | def genMustpass (mustpass, moduleCaseLists): |
| 395 | print "Generating mustpass '%s'" % mustpass.version |
| 396 | |
| 397 | patternLists = readPatternLists(mustpass) |
| 398 | |
| 399 | for package in mustpass.packages: |
| 400 | allCasesInPkg = moduleCaseLists[package.module] |
| 401 | matchingByConfig = {} |
| 402 | allMatchingSet = set() |
| 403 | |
| 404 | for config in package.configurations: |
| 405 | filtered = applyFilters(allCasesInPkg, patternLists, config.filters) |
| 406 | dstFile = getDstCaseListPath(mustpass, package, config) |
| 407 | |
| 408 | print " Writing deqp caselist: " + dstFile |
| 409 | writeFile(dstFile, "\n".join(filtered) + "\n") |
| 410 | |
| 411 | matchingByConfig[config] = filtered |
| 412 | allMatchingSet = allMatchingSet | set(filtered) |
| 413 | |
| 414 | allMatchingCases = [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering |
| 415 | root = buildTestHierachy(allMatchingCases) |
| 416 | testCaseMap = buildTestCaseMap(root) |
| 417 | |
| 418 | for config in package.configurations: |
| 419 | for case in matchingByConfig[config]: |
| 420 | testCaseMap[case].configurations.append(config) |
| 421 | |
| 422 | # NOTE: CTS v2 does not need package XML files. Remove when transition is complete. |
| 423 | packageXml = genCTSPackageXML(mustpass, package, root) |
| 424 | xmlFilename = os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml") |
| 425 | |
| 426 | print " Writing CTS caselist: " + xmlFilename |
| 427 | writeFile(xmlFilename, prettifyXML(packageXml)) |
| 428 | |
| 429 | specXML = genSpecXML(mustpass) |
| 430 | specFilename = os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml") |
| 431 | |
| 432 | print " Writing spec: " + specFilename |
| 433 | writeFile(specFilename, prettifyXML(specXML)) |
| 434 | |
| 435 | # TODO: Which is the best selector mechanism? |
Mika Isojärvi | fb2d85c | 2016-02-04 15:35:09 -0800 | [diff] [blame] | 436 | if (mustpass.version == "master"): |
Pyry Haulos | 2bbb9d2 | 2016-01-14 13:48:08 -0800 | [diff] [blame] | 437 | androidTestXML = genAndroidTestXml(mustpass) |
| 438 | androidTestFilename = os.path.join(mustpass.project.path, "AndroidTest.xml") |
| 439 | |
| 440 | print " Writing AndroidTest.xml: " + androidTestFilename |
| 441 | writeFile(androidTestFilename, prettifyXML(androidTestXML)) |
| 442 | |
| 443 | print "Done!" |
| 444 | |
| 445 | def genMustpassLists (mustpassLists, generator, buildCfg): |
| 446 | moduleCaseLists = {} |
| 447 | |
| 448 | # Getting case lists involves invoking build, so we want to cache the results |
| 449 | for mustpass in mustpassLists: |
| 450 | for package in mustpass.packages: |
| 451 | if not package.module in moduleCaseLists: |
| 452 | moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module) |
| 453 | |
| 454 | for mustpass in mustpassLists: |
| 455 | genMustpass(mustpass, moduleCaseLists) |