blob: ed0eaa2071b2320f7a337aae3be7ae552a0bd1e9 [file] [log] [blame]
Daniel Veillardbb7ddb32002-02-17 21:26:33 +00001#!/usr/bin/python
2import sys
Daniel Veillard878eab02002-02-19 13:46:09 +00003import time
Daniel Veillardbb7ddb32002-02-17 21:26:33 +00004import os
5import string
6sys.path.append("python")
7import libxml2
8
Daniel Veillard1a993962003-10-11 20:58:06 +00009test_nr = 0
10test_succeed = 0
11test_failed = 0
12test_error = 0
13
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000014#
15# the testsuite description
16#
17CONF="xml-test-suite/xmlconf/xmlconf.xml"
18LOG="check-xml-test-suite.log"
19
20log = open(LOG, "w")
21
22#
23# Error and warning handlers
24#
25error_nr = 0
Daniel Veillardc7612992002-02-17 22:47:37 +000026error_msg = ''
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000027def errorHandler(ctx, str):
28 global error_nr
Daniel Veillardc7612992002-02-17 22:47:37 +000029 global error_msg
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000030
Daniel Veillard8f597c32003-10-06 08:19:27 +000031 error_nr = error_nr + 1
Daniel Veillardc7612992002-02-17 22:47:37 +000032 if len(error_msg) < 300:
33 if len(error_msg) == 0 or error_msg[-1] == '\n':
34 error_msg = error_msg + " >>" + str
35 else:
36 error_msg = error_msg + str
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000037
38libxml2.registerErrorHandler(errorHandler, None)
39
40#warning_nr = 0
41#warning = ''
42#def warningHandler(ctx, str):
43# global warning_nr
44# global warning
45#
46# warning_nr = warning_nr + 1
47# warning = warning + str
48#
49#libxml2.registerWarningHandler(warningHandler, None)
50
51#
52# Used to load the XML testsuite description
53#
54def loadNoentDoc(filename):
55 ctxt = libxml2.createFileParserCtxt(filename)
56 if ctxt == None:
57 return None
58 ctxt.replaceEntities(1)
59 ctxt.parseDocument()
Daniel Veillardfa49d872002-03-09 10:20:00 +000060 try:
61 doc = ctxt.doc()
62 except:
63 doc = None
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000064 if ctxt.wellFormed() != 1:
65 doc.freeDoc()
66 return None
67 return doc
68
69#
70# The conformance testing routines
71#
72
73def testNotWf(filename, id):
74 global error_nr
Daniel Veillardc7612992002-02-17 22:47:37 +000075 global error_msg
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000076 global log
77
78 error_nr = 0
Daniel Veillardc7612992002-02-17 22:47:37 +000079 error_msg = ''
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000080
81 ctxt = libxml2.createFileParserCtxt(filename)
82 if ctxt == None:
83 return -1
Daniel Veillard8f597c32003-10-06 08:19:27 +000084 ret = ctxt.parseDocument()
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000085
Daniel Veillardfa49d872002-03-09 10:20:00 +000086 try:
87 doc = ctxt.doc()
88 except:
89 doc = None
Daniel Veillard8f597c32003-10-06 08:19:27 +000090 if doc != None:
91 doc.freeDoc()
92 if ret == 0 or ctxt.wellFormed() != 0:
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000093 print "%s: error: Well Formedness error not detected" % (id)
94 log.write("%s: error: Well Formedness error not detected\n" % (id))
Daniel Veillardbb7ddb32002-02-17 21:26:33 +000095 return 0
96 return 1
97
98def testNotWfEnt(filename, id):
99 global error_nr
Daniel Veillardc7612992002-02-17 22:47:37 +0000100 global error_msg
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000101 global log
102
103 error_nr = 0
Daniel Veillardc7612992002-02-17 22:47:37 +0000104 error_msg = ''
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000105
106 ctxt = libxml2.createFileParserCtxt(filename)
107 if ctxt == None:
108 return -1
109 ctxt.replaceEntities(1)
Daniel Veillard8f597c32003-10-06 08:19:27 +0000110 ret = ctxt.parseDocument()
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000111
Daniel Veillardfa49d872002-03-09 10:20:00 +0000112 try:
113 doc = ctxt.doc()
114 except:
115 doc = None
Daniel Veillard8f597c32003-10-06 08:19:27 +0000116 if doc != None:
117 doc.freeDoc()
118 if ret == 0 or ctxt.wellFormed() != 0:
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000119 print "%s: error: Well Formedness error not detected" % (id)
120 log.write("%s: error: Well Formedness error not detected\n" % (id))
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000121 return 0
122 return 1
123
124def testNotWfEntDtd(filename, id):
125 global error_nr
Daniel Veillardc7612992002-02-17 22:47:37 +0000126 global error_msg
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000127 global log
128
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000129 error_nr = 0
Daniel Veillardc7612992002-02-17 22:47:37 +0000130 error_msg = ''
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000131
132 ctxt = libxml2.createFileParserCtxt(filename)
133 if ctxt == None:
134 return -1
135 ctxt.replaceEntities(1)
136 ctxt.loadSubset(1)
Daniel Veillard8f597c32003-10-06 08:19:27 +0000137 ret = ctxt.parseDocument()
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000138
Daniel Veillardfa49d872002-03-09 10:20:00 +0000139 try:
140 doc = ctxt.doc()
141 except:
142 doc = None
Daniel Veillard8f597c32003-10-06 08:19:27 +0000143 if doc != None:
144 doc.freeDoc()
145 if ret == 0 or ctxt.wellFormed() != 0:
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000146 print "%s: error: Well Formedness error not detected" % (id)
147 log.write("%s: error: Well Formedness error not detected\n" % (id))
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000148 return 0
149 return 1
150
151def testWfEntDtd(filename, id):
152 global error_nr
Daniel Veillardc7612992002-02-17 22:47:37 +0000153 global error_msg
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000154 global log
155
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000156 error_nr = 0
Daniel Veillardc7612992002-02-17 22:47:37 +0000157 error_msg = ''
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000158
159 ctxt = libxml2.createFileParserCtxt(filename)
160 if ctxt == None:
161 return -1
162 ctxt.replaceEntities(1)
163 ctxt.loadSubset(1)
Daniel Veillard8f597c32003-10-06 08:19:27 +0000164 ret = ctxt.parseDocument()
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000165
Daniel Veillardfa49d872002-03-09 10:20:00 +0000166 try:
167 doc = ctxt.doc()
168 except:
169 doc = None
Daniel Veillard8f597c32003-10-06 08:19:27 +0000170 if doc == None or ret != 0 or ctxt.wellFormed() == 0:
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000171 print "%s: error: wrongly failed to parse the document" % (id)
172 log.write("%s: error: wrongly failed to parse the document\n" % (id))
Daniel Veillard8f597c32003-10-06 08:19:27 +0000173 if doc != None:
174 doc.freeDoc()
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000175 return 0
176 if error_nr != 0:
177 print "%s: warning: WF document generated an error msg" % (id)
178 log.write("%s: error: WF document generated an error msg\n" % (id))
179 doc.freeDoc()
180 return 2
181 doc.freeDoc()
182 return 1
183
Daniel Veillard55253e22002-02-18 14:32:39 +0000184def testError(filename, id):
185 global error_nr
186 global error_msg
187 global log
188
189 error_nr = 0
190 error_msg = ''
191
192 ctxt = libxml2.createFileParserCtxt(filename)
193 if ctxt == None:
194 return -1
195 ctxt.replaceEntities(1)
196 ctxt.loadSubset(1)
Daniel Veillard8f597c32003-10-06 08:19:27 +0000197 ret = ctxt.parseDocument()
Daniel Veillard55253e22002-02-18 14:32:39 +0000198
Daniel Veillardfa49d872002-03-09 10:20:00 +0000199 try:
200 doc = ctxt.doc()
201 except:
202 doc = None
Daniel Veillard8f597c32003-10-06 08:19:27 +0000203 if doc != None:
204 doc.freeDoc()
Daniel Veillard55253e22002-02-18 14:32:39 +0000205 if ctxt.wellFormed() == 0:
206 print "%s: warning: failed to parse the document but accepted" % (id)
207 log.write("%s: warning: failed to parse the document but accepte\n" % (id))
208 return 2
209 if error_nr != 0:
210 print "%s: warning: WF document generated an error msg" % (id)
211 log.write("%s: error: WF document generated an error msg\n" % (id))
Daniel Veillard55253e22002-02-18 14:32:39 +0000212 return 2
Daniel Veillard55253e22002-02-18 14:32:39 +0000213 return 1
214
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000215def testInvalid(filename, id):
216 global error_nr
Daniel Veillardc7612992002-02-17 22:47:37 +0000217 global error_msg
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000218 global log
219
220 error_nr = 0
Daniel Veillardc7612992002-02-17 22:47:37 +0000221 error_msg = ''
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000222
223 ctxt = libxml2.createFileParserCtxt(filename)
224 if ctxt == None:
225 return -1
226 ctxt.validate(1)
Daniel Veillard8f597c32003-10-06 08:19:27 +0000227 ret = ctxt.parseDocument()
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000228
Daniel Veillardfa49d872002-03-09 10:20:00 +0000229 try:
230 doc = ctxt.doc()
231 except:
232 doc = None
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000233 valid = ctxt.isValid()
234 if doc == None:
235 print "%s: error: wrongly failed to parse the document" % (id)
236 log.write("%s: error: wrongly failed to parse the document\n" % (id))
237 return 0
238 if valid == 1:
239 print "%s: error: Validity error not detected" % (id)
240 log.write("%s: error: Validity error not detected\n" % (id))
241 doc.freeDoc()
242 return 0
243 if error_nr == 0:
244 print "%s: warning: Validity error not reported" % (id)
245 log.write("%s: warning: Validity error not reported\n" % (id))
246 doc.freeDoc()
247 return 2
248
249 doc.freeDoc()
250 return 1
251
252def testValid(filename, id):
253 global error_nr
Daniel Veillardc7612992002-02-17 22:47:37 +0000254 global error_msg
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000255
256 error_nr = 0
Daniel Veillardc7612992002-02-17 22:47:37 +0000257 error_msg = ''
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000258
259 ctxt = libxml2.createFileParserCtxt(filename)
260 if ctxt == None:
261 return -1
262 ctxt.validate(1)
263 ctxt.parseDocument()
264
Daniel Veillardfa49d872002-03-09 10:20:00 +0000265 try:
266 doc = ctxt.doc()
267 except:
268 doc = None
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000269 valid = ctxt.isValid()
270 if doc == None:
271 print "%s: error: wrongly failed to parse the document" % (id)
272 log.write("%s: error: wrongly failed to parse the document\n" % (id))
273 return 0
274 if valid != 1:
275 print "%s: error: Validity check failed" % (id)
276 log.write("%s: error: Validity check failed\n" % (id))
277 doc.freeDoc()
278 return 0
279 if error_nr != 0 or valid != 1:
280 print "%s: warning: valid document reported an error" % (id)
281 log.write("%s: warning: valid document reported an error\n" % (id))
282 doc.freeDoc()
283 return 2
284 doc.freeDoc()
285 return 1
286
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000287def runTest(test):
288 global test_nr
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000289 global test_succeed
Daniel Veillard1a993962003-10-11 20:58:06 +0000290 global test_failed
Daniel Veillardc7612992002-02-17 22:47:37 +0000291 global error_msg
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000292 global log
293
294 uri = test.prop('URI')
295 id = test.prop('ID')
296 if uri == None:
297 print "Test without ID:", uri
298 return -1
299 if id == None:
300 print "Test without URI:", id
301 return -1
302 base = test.getBase(None)
303 URI = libxml2.buildURI(uri, base)
304 if os.access(URI, os.R_OK) == 0:
305 print "Test %s missing: base %s uri %s" % (URI, base, uri)
306 return -1
307 type = test.prop('TYPE')
308 if type == None:
309 print "Test %s missing TYPE" % (id)
310 return -1
311
312 extra = None
313 if type == "invalid":
314 res = testInvalid(URI, id)
315 elif type == "valid":
316 res = testValid(URI, id)
317 elif type == "not-wf":
318 extra = test.prop('ENTITIES')
319 # print URI
320 #if extra == None:
321 # res = testNotWfEntDtd(URI, id)
322 #elif extra == 'none':
323 # res = testNotWf(URI, id)
324 #elif extra == 'general':
325 # res = testNotWfEnt(URI, id)
326 #elif extra == 'both' or extra == 'parameter':
327 res = testNotWfEntDtd(URI, id)
328 #else:
329 # print "Unknow value %s for an ENTITIES test value" % (extra)
330 # return -1
331 elif type == "error":
Daniel Veillard55253e22002-02-18 14:32:39 +0000332 res = testError(URI, id)
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000333 else:
334 # TODO skipped for now
335 return -1
336
337 test_nr = test_nr + 1
338 if res > 0:
339 test_succeed = test_succeed + 1
340 elif res == 0:
341 test_failed = test_failed + 1
342 elif res < 0:
343 test_error = test_error + 1
344
345 # Log the ontext
346 if res != 1:
347 log.write(" File: %s\n" % (URI))
Daniel Veillardc7612992002-02-17 22:47:37 +0000348 content = string.strip(test.content)
349 while content[-1] == '\n':
350 content = content[0:-1]
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000351 if extra != None:
Daniel Veillardc7612992002-02-17 22:47:37 +0000352 log.write(" %s:%s:%s\n" % (type, extra, content))
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000353 else:
354 log.write(" %s:%s\n\n" % (type, content))
Daniel Veillardc7612992002-02-17 22:47:37 +0000355 if error_msg != '':
356 log.write(" ----\n%s ----\n" % (error_msg))
357 error_msg = ''
358 log.write("\n")
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000359
360 return 0
361
362
363def runTestCases(case):
364 profile = case.prop('PROFILE')
365 if profile != None and \
366 string.find(profile, "IBM XML Conformance Test Suite - Production") < 0:
367 print "=>", profile
368 test = case.children
369 while test != None:
370 if test.name == 'TEST':
371 runTest(test)
372 if test.name == 'TESTCASES':
373 runTestCases(test)
374 test = test.next
375
376conf = loadNoentDoc(CONF)
377if conf == None:
378 print "Unable to load %s" % CONF
379 sys.exit(1)
380
381testsuite = conf.getRootElement()
382if testsuite.name != 'TESTSUITE':
383 print "Expecting TESTSUITE root element: aborting"
384 sys.exit(1)
385
386profile = testsuite.prop('PROFILE')
387if profile != None:
388 print profile
389
Daniel Veillard878eab02002-02-19 13:46:09 +0000390start = time.time()
391
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000392case = testsuite.children
393while case != None:
Daniel Veillardbb7ddb32002-02-17 21:26:33 +0000394 if case.name == 'TESTCASES':
395 old_test_nr = test_nr
396 old_test_succeed = test_succeed
397 old_test_failed = test_failed
398 old_test_error = test_error
399 runTestCases(case)
400 print " Ran %d tests: %d suceeded, %d failed and %d generated an error" % (
401 test_nr - old_test_nr, test_succeed - old_test_succeed,
402 test_failed - old_test_failed, test_error - old_test_error)
403 case = case.next
404
405conf.freeDoc()
406log.close()
407
Daniel Veillard878eab02002-02-19 13:46:09 +0000408print "Ran %d tests: %d suceeded, %d failed and %d generated an error in %.2f s." % (
409 test_nr, test_succeed, test_failed, test_error, time.time() - start)