Daniel Veillard | bb7ddb3 | 2002-02-17 21:26:33 +0000 | [diff] [blame^] | 1 | #!/usr/bin/python |
| 2 | import sys |
| 3 | import os |
| 4 | import string |
| 5 | sys.path.append("python") |
| 6 | import libxml2 |
| 7 | |
| 8 | # |
| 9 | # the testsuite description |
| 10 | # |
| 11 | CONF="xml-test-suite/xmlconf/xmlconf.xml" |
| 12 | LOG="check-xml-test-suite.log" |
| 13 | |
| 14 | log = open(LOG, "w") |
| 15 | |
| 16 | # |
| 17 | # Error and warning handlers |
| 18 | # |
| 19 | error_nr = 0 |
| 20 | def errorHandler(ctx, str): |
| 21 | global error_nr |
| 22 | |
| 23 | error_nr = error_nr + 1 |
| 24 | |
| 25 | libxml2.registerErrorHandler(errorHandler, None) |
| 26 | |
| 27 | #warning_nr = 0 |
| 28 | #warning = '' |
| 29 | #def warningHandler(ctx, str): |
| 30 | # global warning_nr |
| 31 | # global warning |
| 32 | # |
| 33 | # warning_nr = warning_nr + 1 |
| 34 | # warning = warning + str |
| 35 | # |
| 36 | #libxml2.registerWarningHandler(warningHandler, None) |
| 37 | |
| 38 | # |
| 39 | # Used to load the XML testsuite description |
| 40 | # |
| 41 | def loadNoentDoc(filename): |
| 42 | ctxt = libxml2.createFileParserCtxt(filename) |
| 43 | if ctxt == None: |
| 44 | return None |
| 45 | ctxt.replaceEntities(1) |
| 46 | ctxt.parseDocument() |
| 47 | doc = ctxt.doc() |
| 48 | if ctxt.wellFormed() != 1: |
| 49 | doc.freeDoc() |
| 50 | return None |
| 51 | return doc |
| 52 | |
| 53 | # |
| 54 | # The conformance testing routines |
| 55 | # |
| 56 | |
| 57 | def testNotWf(filename, id): |
| 58 | global error_nr |
| 59 | global log |
| 60 | |
| 61 | error_nr = 0 |
| 62 | |
| 63 | ctxt = libxml2.createFileParserCtxt(filename) |
| 64 | if ctxt == None: |
| 65 | return -1 |
| 66 | ctxt.parseDocument() |
| 67 | |
| 68 | doc = ctxt.doc() |
| 69 | if error_nr == 0 or ctxt.wellFormed() != 0: |
| 70 | print "%s: error: Well Formedness error not detected" % (id) |
| 71 | log.write("%s: error: Well Formedness error not detected\n" % (id)) |
| 72 | doc.freeDoc() |
| 73 | return 0 |
| 74 | return 1 |
| 75 | |
| 76 | def testNotWfEnt(filename, id): |
| 77 | global error_nr |
| 78 | global log |
| 79 | |
| 80 | error_nr = 0 |
| 81 | |
| 82 | ctxt = libxml2.createFileParserCtxt(filename) |
| 83 | if ctxt == None: |
| 84 | return -1 |
| 85 | ctxt.replaceEntities(1) |
| 86 | ctxt.parseDocument() |
| 87 | |
| 88 | doc = ctxt.doc() |
| 89 | if error_nr == 0 or ctxt.wellFormed() != 0: |
| 90 | print "%s: error: Well Formedness error not detected" % (id) |
| 91 | log.write("%s: error: Well Formedness error not detected\n" % (id)) |
| 92 | doc.freeDoc() |
| 93 | return 0 |
| 94 | return 1 |
| 95 | |
| 96 | def testNotWfEntDtd(filename, id): |
| 97 | global error_nr |
| 98 | global log |
| 99 | |
| 100 | error = '' |
| 101 | error_nr = 0 |
| 102 | |
| 103 | ctxt = libxml2.createFileParserCtxt(filename) |
| 104 | if ctxt == None: |
| 105 | return -1 |
| 106 | ctxt.replaceEntities(1) |
| 107 | ctxt.loadSubset(1) |
| 108 | ctxt.parseDocument() |
| 109 | |
| 110 | doc = ctxt.doc() |
| 111 | if error_nr == 0 or ctxt.wellFormed() != 0: |
| 112 | print "%s: error: Well Formedness error not detected" % (id) |
| 113 | log.write("%s: error: Well Formedness error not detected\n" % (id)) |
| 114 | doc.freeDoc() |
| 115 | return 0 |
| 116 | return 1 |
| 117 | |
| 118 | def testWfEntDtd(filename, id): |
| 119 | global error_nr |
| 120 | global log |
| 121 | |
| 122 | error = '' |
| 123 | error_nr = 0 |
| 124 | |
| 125 | ctxt = libxml2.createFileParserCtxt(filename) |
| 126 | if ctxt == None: |
| 127 | return -1 |
| 128 | ctxt.replaceEntities(1) |
| 129 | ctxt.loadSubset(1) |
| 130 | ctxt.parseDocument() |
| 131 | |
| 132 | doc = ctxt.doc() |
| 133 | if ctxt.wellFormed() == 0: |
| 134 | print "%s: error: wrongly failed to parse the document" % (id) |
| 135 | log.write("%s: error: wrongly failed to parse the document\n" % (id)) |
| 136 | return 0 |
| 137 | if error_nr != 0: |
| 138 | print "%s: warning: WF document generated an error msg" % (id) |
| 139 | log.write("%s: error: WF document generated an error msg\n" % (id)) |
| 140 | doc.freeDoc() |
| 141 | return 2 |
| 142 | doc.freeDoc() |
| 143 | return 1 |
| 144 | |
| 145 | def testInvalid(filename, id): |
| 146 | global error_nr |
| 147 | global log |
| 148 | |
| 149 | error_nr = 0 |
| 150 | |
| 151 | ctxt = libxml2.createFileParserCtxt(filename) |
| 152 | if ctxt == None: |
| 153 | return -1 |
| 154 | ctxt.validate(1) |
| 155 | ctxt.parseDocument() |
| 156 | |
| 157 | doc = ctxt.doc() |
| 158 | valid = ctxt.isValid() |
| 159 | if doc == None: |
| 160 | print "%s: error: wrongly failed to parse the document" % (id) |
| 161 | log.write("%s: error: wrongly failed to parse the document\n" % (id)) |
| 162 | return 0 |
| 163 | if valid == 1: |
| 164 | print "%s: error: Validity error not detected" % (id) |
| 165 | log.write("%s: error: Validity error not detected\n" % (id)) |
| 166 | doc.freeDoc() |
| 167 | return 0 |
| 168 | if error_nr == 0: |
| 169 | print "%s: warning: Validity error not reported" % (id) |
| 170 | log.write("%s: warning: Validity error not reported\n" % (id)) |
| 171 | doc.freeDoc() |
| 172 | return 2 |
| 173 | |
| 174 | doc.freeDoc() |
| 175 | return 1 |
| 176 | |
| 177 | def testValid(filename, id): |
| 178 | global error_nr |
| 179 | |
| 180 | error_nr = 0 |
| 181 | |
| 182 | ctxt = libxml2.createFileParserCtxt(filename) |
| 183 | if ctxt == None: |
| 184 | return -1 |
| 185 | ctxt.validate(1) |
| 186 | ctxt.parseDocument() |
| 187 | |
| 188 | doc = ctxt.doc() |
| 189 | valid = ctxt.isValid() |
| 190 | if doc == None: |
| 191 | print "%s: error: wrongly failed to parse the document" % (id) |
| 192 | log.write("%s: error: wrongly failed to parse the document\n" % (id)) |
| 193 | return 0 |
| 194 | if valid != 1: |
| 195 | print "%s: error: Validity check failed" % (id) |
| 196 | log.write("%s: error: Validity check failed\n" % (id)) |
| 197 | doc.freeDoc() |
| 198 | return 0 |
| 199 | if error_nr != 0 or valid != 1: |
| 200 | print "%s: warning: valid document reported an error" % (id) |
| 201 | log.write("%s: warning: valid document reported an error\n" % (id)) |
| 202 | doc.freeDoc() |
| 203 | return 2 |
| 204 | doc.freeDoc() |
| 205 | return 1 |
| 206 | |
| 207 | test_nr = 0 |
| 208 | test_succeed = 0 |
| 209 | test_failed = 0 |
| 210 | test_error = 0 |
| 211 | def runTest(test): |
| 212 | global test_nr |
| 213 | global test_failed |
| 214 | global test_error |
| 215 | global test_succeed |
| 216 | global log |
| 217 | |
| 218 | uri = test.prop('URI') |
| 219 | id = test.prop('ID') |
| 220 | if uri == None: |
| 221 | print "Test without ID:", uri |
| 222 | return -1 |
| 223 | if id == None: |
| 224 | print "Test without URI:", id |
| 225 | return -1 |
| 226 | base = test.getBase(None) |
| 227 | URI = libxml2.buildURI(uri, base) |
| 228 | if os.access(URI, os.R_OK) == 0: |
| 229 | print "Test %s missing: base %s uri %s" % (URI, base, uri) |
| 230 | return -1 |
| 231 | type = test.prop('TYPE') |
| 232 | if type == None: |
| 233 | print "Test %s missing TYPE" % (id) |
| 234 | return -1 |
| 235 | |
| 236 | extra = None |
| 237 | if type == "invalid": |
| 238 | res = testInvalid(URI, id) |
| 239 | elif type == "valid": |
| 240 | res = testValid(URI, id) |
| 241 | elif type == "not-wf": |
| 242 | extra = test.prop('ENTITIES') |
| 243 | # print URI |
| 244 | #if extra == None: |
| 245 | # res = testNotWfEntDtd(URI, id) |
| 246 | #elif extra == 'none': |
| 247 | # res = testNotWf(URI, id) |
| 248 | #elif extra == 'general': |
| 249 | # res = testNotWfEnt(URI, id) |
| 250 | #elif extra == 'both' or extra == 'parameter': |
| 251 | res = testNotWfEntDtd(URI, id) |
| 252 | #else: |
| 253 | # print "Unknow value %s for an ENTITIES test value" % (extra) |
| 254 | # return -1 |
| 255 | elif type == "error": |
| 256 | res = testWfEntDtd(URI, id) |
| 257 | else: |
| 258 | # TODO skipped for now |
| 259 | return -1 |
| 260 | |
| 261 | test_nr = test_nr + 1 |
| 262 | if res > 0: |
| 263 | test_succeed = test_succeed + 1 |
| 264 | elif res == 0: |
| 265 | test_failed = test_failed + 1 |
| 266 | elif res < 0: |
| 267 | test_error = test_error + 1 |
| 268 | |
| 269 | # Log the ontext |
| 270 | if res != 1: |
| 271 | log.write(" File: %s\n" % (URI)) |
| 272 | content = test.content |
| 273 | if extra != None: |
| 274 | log.write(" %s:%s:%s\n\n" % (type, extra, content)) |
| 275 | else: |
| 276 | log.write(" %s:%s\n\n" % (type, content)) |
| 277 | |
| 278 | return 0 |
| 279 | |
| 280 | |
| 281 | def runTestCases(case): |
| 282 | profile = case.prop('PROFILE') |
| 283 | if profile != None and \ |
| 284 | string.find(profile, "IBM XML Conformance Test Suite - Production") < 0: |
| 285 | print "=>", profile |
| 286 | test = case.children |
| 287 | while test != None: |
| 288 | if test.name == 'TEST': |
| 289 | runTest(test) |
| 290 | if test.name == 'TESTCASES': |
| 291 | runTestCases(test) |
| 292 | test = test.next |
| 293 | |
| 294 | conf = loadNoentDoc(CONF) |
| 295 | if conf == None: |
| 296 | print "Unable to load %s" % CONF |
| 297 | sys.exit(1) |
| 298 | |
| 299 | testsuite = conf.getRootElement() |
| 300 | if testsuite.name != 'TESTSUITE': |
| 301 | print "Expecting TESTSUITE root element: aborting" |
| 302 | sys.exit(1) |
| 303 | |
| 304 | profile = testsuite.prop('PROFILE') |
| 305 | if profile != None: |
| 306 | print profile |
| 307 | |
| 308 | case = testsuite.children |
| 309 | while case != None: |
| 310 | global test_nr |
| 311 | global test_succeed |
| 312 | global test_failed |
| 313 | global test_error |
| 314 | |
| 315 | if case.name == 'TESTCASES': |
| 316 | old_test_nr = test_nr |
| 317 | old_test_succeed = test_succeed |
| 318 | old_test_failed = test_failed |
| 319 | old_test_error = test_error |
| 320 | runTestCases(case) |
| 321 | print " Ran %d tests: %d suceeded, %d failed and %d generated an error" % ( |
| 322 | test_nr - old_test_nr, test_succeed - old_test_succeed, |
| 323 | test_failed - old_test_failed, test_error - old_test_error) |
| 324 | case = case.next |
| 325 | |
| 326 | conf.freeDoc() |
| 327 | log.close() |
| 328 | |
| 329 | print "Ran %d tests: %d suceeded, %d failed and %d generated an error" % ( |
| 330 | test_nr, test_succeed, test_failed, test_error) |