scripts/mustpass.py

# -*- coding: utf-8 -*-

#-------------------------------------------------------------------------
# drawElements Quality Program utilities
# --------------------------------------
#
# Copyright 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#-------------------------------------------------------------------------

from build.common import *
from build.config import ANY_GENERATOR
from build.build import build
from build_caselists import Module, getModuleByName, getBuildConfig, genCaseList, getCaseListPath, DEFAULT_BUILD_DIR, DEFAULT_TARGET
from fnmatch import fnmatch
from copy import copy

import xml.etree.cElementTree as ElementTree
import xml.dom.minidom as minidom

APK_NAME 		= "com.drawelements.deqp.apk"

GENERATED_FILE_WARNING = """
     This file has been automatically generated. Edit with caution.
     """

class Project:
	def __init__ (self, path, copyright = None):
		self.path		= path
		self.copyright	= copyright

class Configuration:
	def __init__ (self, name, filters, glconfig = None, rotation = None, surfacetype = None, runtime = None):
		self.name			 = name
		self.glconfig		 = glconfig
		self.rotation		 = rotation
		self.surfacetype	 = surfacetype
		self.filters		 = filters
		self.expectedRuntime = runtime

class Package:
	def __init__ (self, module, configurations):
		self.module			= module
		self.configurations	= configurations

class Mustpass:
	def __init__ (self, project, version, packages):
		self.project	= project
		self.version	= version
		self.packages	= packages

class Filter:
	TYPE_INCLUDE = 0
	TYPE_EXCLUDE = 1

	def __init__ (self, type, filename):
		self.type		= type
		self.filename	= filename

class TestRoot:
	def __init__ (self):
		self.children	= []

class TestGroup:
	def __init__ (self, name):
		self.name		= name
		self.children	= []

class TestCase:
	def __init__ (self, name):
		self.name			= name
		self.configurations	= []

class GLESVersion:
	def __init__(self, major, minor):
		self.major = major
		self.minor = minor

	def encode (self):
		return (self.major << 16) | (self.minor)

def getModuleGLESVersion (module):
	versions = {
		'dEQP-EGL':		GLESVersion(2,0),
		'dEQP-GLES2':	GLESVersion(2,0),
		'dEQP-GLES3':	GLESVersion(3,0),
		'dEQP-GLES31':	GLESVersion(3,1)
	}
	return versions[module.name] if module.name in versions else None

def getSrcDir (mustpass):
	return os.path.join(mustpass.project.path, mustpass.version, "src")

def getTmpDir (mustpass):
	return os.path.join(mustpass.project.path, mustpass.version, "tmp")

def getModuleShorthand (module):
	assert module.name[:5] == "dEQP-"
	return module.name[5:].lower()

def getCaseListFileName (package, configuration):
	return "%s-%s.txt" % (getModuleShorthand(package.module), configuration.name)

def getDstCaseListPath (mustpass, package, configuration):
	return os.path.join(mustpass.project.path, mustpass.version, getCaseListFileName(package, configuration))

def getCTSPackageName (package):
	return "com.drawelements.deqp." + getModuleShorthand(package.module)

def getCommandLine (config):
	cmdLine = ""

	if config.glconfig != None:
		cmdLine += "--deqp-gl-config-name=%s " % config.glconfig

	if config.rotation != None:
		cmdLine += "--deqp-screen-rotation=%s " % config.rotation

	if config.surfacetype != None:
		cmdLine += "--deqp-surface-type=%s " % config.surfacetype

	cmdLine += "--deqp-watchdog=enable"

	return cmdLine

def readCaseList (filename):
	cases = []
	with open(filename, 'rb') as f:
		for line in f:
			if line[:6] == "TEST: ":
				cases.append(line[6:].strip())
	return cases

def getCaseList (buildCfg, generator, module):
	build(buildCfg, generator, [module.binName])
	genCaseList(buildCfg, generator, module, "txt")
	return readCaseList(getCaseListPath(buildCfg, module, "txt"))

def readPatternList (filename):
	ptrns = []
	with open(filename, 'rb') as f:
		for line in f:
			line = line.strip()
			if len(line) > 0 and line[0] != '#':
				ptrns.append(line)
	return ptrns

def applyPatterns (caseList, patterns, filename, op):
	matched			= set()
	errors			= []
	curList			= copy(caseList)
	trivialPtrns	= [p for p in patterns if p.find('*') < 0]
	regularPtrns	= [p for p in patterns if p.find('*') >= 0]

	# Apply trivial (just case paths)
	allCasesSet		= set(caseList)
	for path in trivialPtrns:
		if path in allCasesSet:
			if path in matched:
				errors.append((path, "Same case specified more than once"))
			matched.add(path)
		else:
			errors.append((path, "Test case not found"))

	curList = [c for c in curList if c not in matched]

	for pattern in regularPtrns:
		matchedThisPtrn = set()

		for case in curList:
			if fnmatch(case, pattern):
				matchedThisPtrn.add(case)

		if len(matchedThisPtrn) == 0:
			errors.append((pattern, "Pattern didn't match any cases"))

		matched	= matched | matchedThisPtrn
		curList = [c for c in curList if c not in matched]

	for pattern, reason in errors:
		print "ERROR: %s: %s" % (reason, pattern)

	if len(errors) > 0:
		die("Found %s invalid patterns while processing file %s" % (len(errors), filename))

	return [c for c in caseList if op(c in matched)]

def applyInclude (caseList, patterns, filename):
	return applyPatterns(caseList, patterns, filename, lambda b: b)

def applyExclude (caseList, patterns, filename):
	return applyPatterns(caseList, patterns, filename, lambda b: not b)

def readPatternLists (mustpass):
	lists = {}
	for package in mustpass.packages:
		for cfg in package.configurations:
			for filter in cfg.filters:
				if not filter.filename in lists:
					lists[filter.filename] = readPatternList(os.path.join(getSrcDir(mustpass), filter.filename))
	return lists

def applyFilters (caseList, patternLists, filters):
	res = copy(caseList)
	for filter in filters:
		ptrnList = patternLists[filter.filename]
		if filter.type == Filter.TYPE_INCLUDE:
			res = applyInclude(res, ptrnList, filter.filename)
		else:
			assert filter.type == Filter.TYPE_EXCLUDE
			res = applyExclude(res, ptrnList, filter.filename)
	return res

def appendToHierarchy (root, casePath):
	def findChild (node, name):
		for child in node.children:
			if child.name == name:
				return child
		return None

	curNode		= root
	components	= casePath.split('.')

	for component in components[:-1]:
		nextNode = findChild(curNode, component)
		if not nextNode:
			nextNode = TestGroup(component)
			curNode.children.append(nextNode)
		curNode = nextNode

	if not findChild(curNode, components[-1]):
		curNode.children.append(TestCase(components[-1]))

def buildTestHierachy (caseList):
	root = TestRoot()
	for case in caseList:
		appendToHierarchy(root, case)
	return root

def buildTestCaseMap (root):
	caseMap = {}

	def recursiveBuild (curNode, prefix):
		curPath = prefix + curNode.name
		if isinstance(curNode, TestCase):
			caseMap[curPath] = curNode
		else:
			for child in curNode.children:
				recursiveBuild(child, curPath + '.')

	for child in root.children:
		recursiveBuild(child, '')

	return caseMap

def include (filename):
	return Filter(Filter.TYPE_INCLUDE, filename)

def exclude (filename):
	return Filter(Filter.TYPE_EXCLUDE, filename)

def insertXMLHeaders (mustpass, doc):
	if mustpass.project.copyright != None:
		doc.insert(0, ElementTree.Comment(mustpass.project.copyright))
	doc.insert(1, ElementTree.Comment(GENERATED_FILE_WARNING))

def prettifyXML (doc):
	uglyString	= ElementTree.tostring(doc, 'utf-8')
	reparsed	= minidom.parseString(uglyString)
	return reparsed.toprettyxml(indent='\t', encoding='utf-8')

def genCTSPackageXML (mustpass, package, root):
	def isLeafGroup (testGroup):
		numGroups	= 0
		numTests	= 0

		for child in testGroup.children:
			if isinstance(child, TestCase):
				numTests += 1
			else:
				numGroups += 1

		assert numGroups + numTests > 0

		if numGroups > 0 and numTests > 0:
			die("Mixed groups and cases in %s" % testGroup.name)

		return numGroups == 0

	def makeConfiguration (parentElem, config):
		attributes = {}

		if config.glconfig != None:
			attributes['glconfig'] = config.glconfig

		if config.rotation != None:
			attributes['rotation'] = config.rotation

		if config.surfacetype != None:
			attributes['surfacetype'] = config.surfacetype

		return ElementTree.SubElement(parentElem, "TestInstance", attributes)

	def makeTestCase (parentElem, testCase):
		caseElem = ElementTree.SubElement(parentElem, "Test", name=testCase.name)
		for config in testCase.configurations:
			makeConfiguration(caseElem, config)
		return caseElem

	def makeTestGroup (parentElem, testGroup):
		groupElem = ElementTree.SubElement(parentElem, "TestCase" if isLeafGroup(testGroup) else "TestSuite", name=testGroup.name)
		for child in testGroup.children:
			if isinstance(child, TestCase):
				makeTestCase(groupElem, child)
			else:
				makeTestGroup(groupElem, child)
		return groupElem

	pkgElem = ElementTree.Element("TestPackage",
								  name				= package.module.name,
								  appPackageName	= getCTSPackageName(package),
								  testType			= "deqpTest")

	pkgElem.set("xmlns:deqp", "http://drawelements.com/deqp")
	insertXMLHeaders(mustpass, pkgElem)

	glesVersion = getModuleGLESVersion(package.module)

	if glesVersion != None:
		pkgElem.set("deqp:glesVersion", str(glesVersion.encode()))

	for child in root.children:
		makeTestGroup(pkgElem, child)

	return pkgElem

def genSpecXML (mustpass):
	mustpassElem = ElementTree.Element("Mustpass", version = mustpass.version)
	insertXMLHeaders(mustpass, mustpassElem)

	for package in mustpass.packages:
		packageElem = ElementTree.SubElement(mustpassElem, "TestPackage", name = package.module.name)

		for config in package.configurations:
			configElem = ElementTree.SubElement(packageElem, "Configuration",
												name			= config.name,
												caseListFile	= getCaseListFileName(package, config),
												commandLine		= getCommandLine(config))

	return mustpassElem

def addOptionElement (parent, optionName, optionValue):
	ElementTree.SubElement(parent, "option", name=optionName, value=optionValue)

def genAndroidTestXml (mustpass):
	INSTALLER_CLASS = "com.android.compatibility.common.tradefed.targetprep.ApkInstaller"
	RUNNER_CLASS = "com.drawelements.deqp.runner.DeqpTestRunner"
	configElement = ElementTree.Element("configuration")
	preparerElement = ElementTree.SubElement(configElement, "target_preparer")
	preparerElement.set("class", INSTALLER_CLASS)
	addOptionElement(preparerElement, "cleanup-apks", "true")
	addOptionElement(preparerElement, "test-file-name", APK_NAME)

	for package in mustpass.packages:
		for config in package.configurations:
			testElement = ElementTree.SubElement(configElement, "test")
			testElement.set("class", RUNNER_CLASS)
			addOptionElement(testElement, "deqp-package", package.module.name)
			addOptionElement(testElement, "deqp-caselist-file", getCaseListFileName(package,config))
			# \todo [2015-10-16 kalle]: Replace with just command line? - requires simplifications in the runner/tests as well.
			if config.glconfig != None:
				addOptionElement(testElement, "deqp-gl-config-name", config.glconfig)

			if config.surfacetype != None:
				addOptionElement(testElement, "deqp-surface-type", config.surfacetype)

			if config.rotation != None:
				addOptionElement(testElement, "deqp-screen-rotation", config.rotation)

			if config.expectedRuntime != None:
				addOptionElement(testElement, "runtime-hint", config.expectedRuntime)

	insertXMLHeaders(mustpass, configElement)

	return configElement

def genMustpass (mustpass, moduleCaseLists):
	print "Generating mustpass '%s'" % mustpass.version

	patternLists = readPatternLists(mustpass)

	for package in mustpass.packages:
		allCasesInPkg		= moduleCaseLists[package.module]
		matchingByConfig	= {}
		allMatchingSet		= set()

		for config in package.configurations:
			filtered	= applyFilters(allCasesInPkg, patternLists, config.filters)
			dstFile		= getDstCaseListPath(mustpass, package, config)

			print "  Writing deqp caselist: " + dstFile
			writeFile(dstFile, "\n".join(filtered) + "\n")

			matchingByConfig[config]	= filtered
			allMatchingSet				= allMatchingSet | set(filtered)

		allMatchingCases	= [c for c in allCasesInPkg if c in allMatchingSet] # To preserve ordering
		root				= buildTestHierachy(allMatchingCases)
		testCaseMap			= buildTestCaseMap(root)

		for config in package.configurations:
			for case in matchingByConfig[config]:
				testCaseMap[case].configurations.append(config)

		# NOTE: CTS v2 does not need package XML files. Remove when transition is complete.
		packageXml	= genCTSPackageXML(mustpass, package, root)
		xmlFilename	= os.path.join(mustpass.project.path, mustpass.version, getCTSPackageName(package) + ".xml")

		print "  Writing CTS caselist: " + xmlFilename
		writeFile(xmlFilename, prettifyXML(packageXml))

	specXML			= genSpecXML(mustpass)
	specFilename	= os.path.join(mustpass.project.path, mustpass.version, "mustpass.xml")

	print "  Writing spec: " + specFilename
	writeFile(specFilename, prettifyXML(specXML))

	# TODO: Which is the best selector mechanism?
	if (mustpass.version == "master"):
		androidTestXML		= genAndroidTestXml(mustpass)
		androidTestFilename	= os.path.join(mustpass.project.path, "AndroidTest.xml")

		print "  Writing AndroidTest.xml: " + androidTestFilename
		writeFile(androidTestFilename, prettifyXML(androidTestXML))

	print "Done!"

def genMustpassLists (mustpassLists, generator, buildCfg):
	moduleCaseLists = {}

	# Getting case lists involves invoking build, so we want to cache the results
	for mustpass in mustpassLists:
		for package in mustpass.packages:
			if not package.module in moduleCaseLists:
				moduleCaseLists[package.module] = getCaseList(buildCfg, generator, package.module)

	for mustpass in mustpassLists:
		genMustpass(mustpass, moduleCaseLists)