blob: 9c75bbcb9c3120311f46a777c1d2c2b3942aad8a [file] [log] [blame]
Jarkko Poyry3c827362014-09-02 11:48:52 +03001# -*- coding: utf-8 -*-
2
Jarkko Pöyry3c77ed42015-01-06 12:54:34 -08003#-------------------------------------------------------------------------
4# drawElements Quality Program utilities
5# --------------------------------------
6#
7# Copyright 2015 The Android Open Source Project
8#
9# Licensed under the Apache License, Version 2.0 (the "License");
10# you may not use this file except in compliance with the License.
11# You may obtain a copy of the License at
12#
13# http://www.apache.org/licenses/LICENSE-2.0
14#
15# Unless required by applicable law or agreed to in writing, software
16# distributed under the License is distributed on an "AS IS" BASIS,
17# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
18# See the License for the specific language governing permissions and
19# limitations under the License.
20#
21#-------------------------------------------------------------------------
22
Jarkko Poyry3c827362014-09-02 11:48:52 +030023import sys
24import random
25import string
26import subprocess
27from optparse import OptionParser
28
29def all (results, predicate):
30 for result in results:
31 if not predicate(result):
32 return False
33 return True
34
35def any (results, predicate):
36 for result in results:
37 if predicate(result):
38 return True
39 return False
40
41class FilterRule:
42 def __init__ (self, name, description, filters):
43 self.name = name
44 self.description = description
45 self.filters = filters
46
47class TestCaseResult:
48 def __init__ (self, name, results):
49 self.name = name
50 self.results = results
51
52class Group:
53 def __init__ (self, name):
54 self.name = name
55 self.cases = []
56
57def readCaseList (filename):
58 f = open(filename, 'rb')
59 cases = []
60 for line in f:
61 if line[:6] == "TEST: ":
62 case = line[6:].strip()
63 if len(case) > 0:
64 cases.append(case)
65 return cases
66
67def toResultList (caselist):
68 results = []
69 for case in caselist:
70 results.append(TestCaseResult(case, []))
71 return results
72
73def addResultsToCaseList (caselist, results):
74 resultMap = {}
75 caseListRes = toResultList(caselist)
76
77 for res in caseListRes:
78 resultMap[res.name] = res
79
80 for result in results:
81 if result.name in resultMap:
82 resultMap[result.name].results += result.results
83
84 return caseListRes
85
86def readTestResults (filename):
87 f = open(filename, 'rb')
88 csvData = f.read()
89 csvLines = csvData.splitlines()
90 results = []
91
92 f.close()
93
94 for line in csvLines[1:]:
95 args = line.split(',')
96 if len(args) == 1:
97 continue # Ignore
98
99 results.append(TestCaseResult(args[0], args[1:]))
100
101 if len(results) == 0:
102 raise Exception("Empty result list")
103
104 # Sanity check for results
105 numResultItems = len(results[0].results)
106 seenResults = set()
107 for result in results:
108 if result.name in seenResults:
109 raise Exception("Duplicate result row for test case '%s'" % result.name)
110 if len(result.results) != numResultItems:
111 raise Exception("Found %d results for test case '%s', expected %d" % (len(result.results), result.name, numResultItems))
112 seenResults.add(result.name)
113
114 return results
115
116def readGroupList (filename):
117 f = open(filename, 'rb')
118 groups = []
119 for line in f:
120 group = line.strip()
121 if group != "":
122 groups.append(group)
123 return groups
124
125def createGroups (results, groupNames):
126 groups = []
127 matched = set()
128
129 for groupName in groupNames:
130 group = Group(groupName)
131 groups.append(group)
132
133 prefix = groupName + "."
134 prefixLen = len(prefix)
135 for case in results:
136 if case.name[:prefixLen] == prefix:
137 if case in matched:
138 die("Case '%s' matched by multiple groups (when processing '%s')" % (case.name, group.name))
139 group.cases.append(case)
140 matched.add(case)
141
142 return groups
143
144def createLeafGroups (results):
145 groups = []
146 groupMap = {}
147
148 for case in results:
149 parts = case.name.split('.')
150 groupName = string.join(parts[:-1], ".")
151
152 if not groupName in groupMap:
153 group = Group(groupName)
154 groups.append(group)
155 groupMap[groupName] = group
156 else:
157 group = groupMap[groupName]
158
159 group.cases.append(case)
160
161 return groups
162
163def filterList (results, condition):
164 filtered = []
165 for case in results:
166 if condition(case.results):
167 filtered.append(case)
168 return filtered
169
170def getFilter (list, name):
171 for filter in list:
172 if filter.name == name:
173 return filter
174 return None
175
176def getNumCasesInGroups (groups):
177 numCases = 0
178 for group in groups:
179 numCases += len(group.cases)
180 return numCases
181
182def getCasesInSet (results, caseSet):
183 filtered = []
184 for case in results:
185 if case in caseSet:
186 filtered.append(case)
187 return filtered
188
189def selectCasesInGroups (results, groups):
190 casesInGroups = set()
191 for group in groups:
192 for case in group.cases:
193 casesInGroups.add(case)
194 return getCasesInSet(results, casesInGroups)
195
196def selectRandomSubset (results, groups, limit, seed):
197 selectedCases = set()
198 numSelect = min(limit, getNumCasesInGroups(groups))
199
200 random.seed(seed)
201 random.shuffle(groups)
202
203 groupNdx = 0
204 while len(selectedCases) < numSelect:
205 group = groups[groupNdx]
206 if len(group.cases) == 0:
207 del groups[groupNdx]
208 if groupNdx == len(groups):
209 groupNdx -= 1
210 continue # Try next
211
212 selected = random.choice(group.cases)
213 selectedCases.add(selected)
214 group.cases.remove(selected)
215
216 groupNdx = (groupNdx + 1) % len(groups)
217
218 return getCasesInSet(results, selectedCases)
219
220def die (msg):
221 print msg
222 sys.exit(-1)
223
224# Named filter lists
225FILTER_RULES = [
226 FilterRule("all", "No filtering", []),
Pyry Haulosf1893652016-08-16 12:35:51 +0200227 FilterRule("all-pass", "All results must be 'Pass'", [lambda l: all(l, lambda r: r == 'Pass')]),
Jarkko Poyry3c827362014-09-02 11:48:52 +0300228 FilterRule("any-pass", "Any of results is 'Pass'", [lambda l: any(l, lambda r: r == 'Pass')]),
229 FilterRule("any-fail", "Any of results is not 'Pass' or 'NotSupported'", [lambda l: not all(l, lambda r: r == 'Pass' or r == 'NotSupported')]),
230 FilterRule("prev-failing", "Any except last result is failure", [lambda l: l[-1] == 'Pass' and not all(l[:-1], lambda r: r == 'Pass')]),
231 FilterRule("prev-passing", "Any except last result is 'Pass'", [lambda l: l[-1] != 'Pass' and any(l[:-1], lambda r: r == 'Pass')])
232]
233
234if __name__ == "__main__":
235 parser = OptionParser(usage = "usage: %prog [options] [caselist] [result csv file]")
236 parser.add_option("-f", "--filter", dest="filter", default="all", help="filter rule name")
237 parser.add_option("-l", "--list", action="store_true", dest="list", default=False, help="list available rules")
238 parser.add_option("-n", "--num", dest="limit", default=0, help="limit number of cases")
239 parser.add_option("-s", "--seed", dest="seed", default=0, help="use selected seed for random selection")
240 parser.add_option("-g", "--groups", dest="groups_file", default=None, help="select cases based on group list file")
241
242 (options, args) = parser.parse_args()
243
244 if options.list:
245 print "Available filter rules:"
246 for filter in FILTER_RULES:
247 print " %s: %s" % (filter.name, filter.description)
248 sys.exit(0)
249
250 if len(args) == 0:
251 die("No input files specified")
252 elif len(args) > 2:
253 die("Too many arguments")
254
255 # Fetch filter
256 filter = getFilter(FILTER_RULES, options.filter)
257 if filter == None:
258 die("Unknown filter '%s'" % options.filter)
259
260 # Read case list
261 caselist = readCaseList(args[0])
262 if len(args) > 1:
263 results = readTestResults(args[1])
264 results = addResultsToCaseList(caselist, results)
265 else:
266 results = toResultList(caselist)
267
268 # Execute filters for results
269 for rule in filter.filters:
270 results = filterList(results, rule)
271
272 if options.limit != 0:
273 if options.groups_file != None:
274 groups = createGroups(results, readGroupList(options.groups_file))
275 else:
276 groups = createLeafGroups(results)
277 results = selectRandomSubset(results, groups, int(options.limit), int(options.seed))
278 elif options.groups_file != None:
279 groups = createGroups(results, readGroupList(options.groups_file))
280 results = selectCasesInGroups(results, groups)
281
282 # Print test set
283 for result in results:
284 print result.name