harness.c

/*
 * Copyright 2001-2004 Unicode, Inc.
 * 
 * Disclaimer
 * 
 * This source code is provided as is by Unicode, Inc. No claims are
 * made as to fitness for any particular purpose. No warranties of any
 * kind are expressed or implied. The recipient agrees to determine
 * applicability of information provided. If this file has been
 * purchased on magnetic or optical media from Unicode, Inc., the
 * sole remedy for any claim will be exchange of defective media
 * within 90 days of receipt.
 * 
 * Limitations on Rights to Redistribute This Code
 * 
 * Unicode, Inc. hereby grants the right to freely use the information
 * supplied in this file in the creation of products supporting the
 * Unicode Standard, and to make copies of this file in any form
 * for internal or external distribution as long as this notice
 * remains attached.
 *
 * harness.c
 *
 * This is a test harness for "ConvertUTF.c".  Compile this
 * and run without arguments.  It will exhaustively test
 * the conversion routines, and print a few lines of diagnostic
 * output.  You don't need to compile ConvertUTF.c itself, 
 * since it gets #included here along with the header.
 * Example of a compile line:
 *
 *		$	gcc -g harness.c -o harness
 *
 * Rev History: Rick McGowan, new file April 2001.
 * Sept 19, 2002: Corrected error on line 234:  utf16_buf[2] becomes utf16_result[2]
 * 	per report from Iain Murray.
 * July 3, 2003: Updated printout message.
 * Oct 19, 2004: Updated isLegalUTF8 test data and corrected switch statements to catch
 *	illegal surrogate use in UTF-8, per report from Frank Tang.
 *
 */

#define CVTUTF_DEBUG 1

#include <stdio.h>
#include "ConvertUTF.c"

/* ---------------------------------------------------------------------
	test01 - Spot check a few legal & illegal UTF-8 values only.
	This is not an exhaustive test, just a brief one that was
	used to develop the "isLegalUTF8" routine.

	Legal UTF-8 sequences are:

	1st----	2nd----	3rd----	4th----	Codepoints---

	00-7F				  0000-  007F
	C2-DF	80-BF			  0080-  07FF
	E0	A0-BF	80-BF		  0800-  0FFF
	E1-EC   80-BF   80-BF             1000-  CFFF
	ED      80-9F   80-BF             D000-  D7FF
	EE-EF   80-BF   80-BF             E000-  FFFF
	F0	90-BF	80-BF	80-BF	 10000- 3FFFF
	F1-F3	80-BF	80-BF	80-BF	 40000- FFFFF
	F4	80-8F	80-BF	80-BF	100000-10FFFF

   --------------------------------------------------------------------- */


struct utf8_test {
    Boolean utf8_legal;	/* is legal sequence? */
    int utf8_len;	/* length of sequence */
    unsigned char utf8_seq[5];	/* the sequence */
};

struct utf8_test utf8_testData[] = {
    { 1,	1,	{ 0x7A, 0x00, 0x00, 0x00, 0x00 }},	/* 0 */
    { 1,	2,	{ 0xC2, 0xAC, 0x00, 0x00, 0x00 }},	/* 1 */
    { 1,	2,	{ 0xDF, 0xB2, 0x00, 0x00, 0x00 }},	/* 2 */
    { 1,	3,	{ 0xE0, 0xA1, 0x81, 0x00, 0x00 }},	/* 3 */
    { 1,	3,	{ 0xE1, 0xAC, 0x90, 0x00, 0x00 }},	/* 4 */
    { 1,	3,	{ 0xF0, 0x93, 0xB2, 0xA1, 0x00 }},	/* 5 */
    { 1,	4,	{ 0xF1, 0x87, 0x9A, 0xB0, 0x00 }},	/* 6 */
    { 1,	4,	{ 0xF3, 0x88, 0x9B, 0xAD, 0x00 }},	/* 7 */
    { 1,	4,	{ 0xF4, 0x82, 0x89, 0x8F, 0x00 }},	/* 8 */

    { 0,	3,	{ 0x82, 0x00, 0x00, 0x00, 0x00 }},	/* 9 */
    { 0,	2,	{ 0xF8, 0xAC, 0x00, 0x00, 0x00 }},	/* 10 */
    { 0,	2,	{ 0xE1, 0xFC, 0xFF, 0x00, 0x00 }},	/* 11 */
    { 0,	3,	{ 0xC2, 0xFC, 0x00, 0x00, 0x00 }},	/* 12 */
    { 0,	3,	{ 0xE1, 0xC2, 0x81, 0x00, 0x00 }},	/* 13 */
    { 0,	2,	{ 0xC2, 0xC1, 0x00, 0x00, 0x00 }},	/* 14 */
    { 0,	2,	{ 0xC0, 0xAF, 0x00, 0x00, 0x00 }},	/* 15 */
    { 0,	3,	{ 0xE0, 0x9F, 0x80, 0x00, 0x00 }},	/* 16 */
    { 0,	4,	{ 0xF0, 0x93, 0xB2, 0xC1, 0x00 }},	/* 17 */

    { 1,	3,	{ 0xED, 0x9F, 0xBF, 0x00, 0x00 }},	/* 18 */
    { 1,	3,	{ 0xEE, 0x80, 0x80, 0x00, 0x00 }},	/* 19 */
    { 0,	3,	{ 0xED, 0xA0, 0x80, 0x00, 0x00 }},	/* 20 */
    { 0,	3,	{ 0xED, 0xBF, 0xBF, 0x00, 0x00 }},	/* 21 */

/* for all > 21 use "short" buffer lengths to detect over-run */
    { 0,	4,	{ 0xF0, 0x93, 0xB2, 0xC3, 0x00 }},	/* 18 use short buflen */
    { 0,	0,	{ 0x00, 0x00, 0x00, 0x00, 0x00 }},

};

int test01() {
	int i;
	int rval, wantVal1, wantVal2, gotVal1, gotVal2, len2;

	printf("Begin Test01\n"); fflush(stdout);

	rval = 0;
	for (i = 0; utf8_testData[i].utf8_len; i++) {
		wantVal1 = wantVal2 = utf8_testData[i].utf8_legal;
		gotVal1 = isLegalUTF8(&(utf8_testData[i].utf8_seq[0]), utf8_testData[i].utf8_len);
		/* use truncated length for tests over 21 */
		if (i <= 21) { len2 = 4; } else { len2 = utf8_testData[i].utf8_len-1; wantVal2 = 0; }
		gotVal2 = isLegalUTF8Sequence(&(utf8_testData[i].utf8_seq[0]), &(utf8_testData[i].utf8_seq[0])+len2);
		if ((gotVal1 != wantVal1) || (gotVal2 != wantVal2)) {
			printf("Test01 error: seq %d is %d & %d (should be %d & %d) for bytes (%x,%x,%x,%x,%x,) & len %d\n",
			i, gotVal1, gotVal2, wantVal1, wantVal2, utf8_testData[i].utf8_seq[0],
			utf8_testData[i].utf8_seq[1], utf8_testData[i].utf8_seq[2],
			utf8_testData[i].utf8_seq[3], utf8_testData[i].utf8_seq[4],
			utf8_testData[i].utf8_len);
			++rval;
		}
	}

	return (rval ? 0 : 1);
}


/* ---------------------------------------------------------------------
	test02 - Test round trip UTF32 -> UTF16 -> UTF8 -> UTF16 -> UTF32

	This is an exhaustive test of values 0 through 0x10FFFF.  It
	takes each integer value and converts from UTC4 through the
	other encoding forms, and back to UTR32, checking the results
	along the way.

	It does not check the un-paired low surrogates, except for
	the first low surrogate.  It intends to get that one illegal
	result, prints a message, and continues with tests.

   --------------------------------------------------------------------- */

int test02() {
	int i, n;
	ConversionResult result;
	UTF32 utf32_buf[2], utf32_result[2];
	UTF16 utf16_buf[3], utf16_result[3];
	UTF8 utf8_buf[8];
	UTF32 *utf32SourceStart, *utf32TargetStart;
	UTF16 *utf16SourceStart, *utf16TargetStart;
	UTF8 *utf8SourceStart, *utf8TargetStart;

	printf("Begin Test02\n"); fflush(stdout);

	for (i = 0; i <= 0x10FFFF; i++) {
		utf32_buf[0] = i; utf32_buf[1] = 0;
		utf32_result[0] = utf32_result[1] = 0;
		utf16_buf[0] = utf16_buf[1] = utf16_buf[2] = 0;
		utf16_result[0] = utf16_result[1] = utf16_result[2] = 0;
		for (n = 0; n < 8; n++) utf8_buf[n] = 0;

		utf32SourceStart = utf32_buf; utf32TargetStart = utf32_result;
		utf16TargetStart = utf16SourceStart = utf16_buf;
		utf8TargetStart = utf8SourceStart = utf8_buf;

		/*
		 * Test UTF32 -> UTF16
		 */
		result = ConvertUTF32toUTF16((const UTF32 **) &utf32SourceStart, &(utf32_buf[1]), &utf16TargetStart, &(utf16_buf[2]), strictConversion);
		if (i < UNI_SUR_HIGH_START || i > UNI_SUR_LOW_END) {
			/* skip result checking for all but 0000d800, which we know to be illegal */
			switch (result) {
			default: fprintf(stderr, "Test02A fatal error: result %d for input %08x\n", result, utf32_buf[0]); exit(1);
			case conversionOK: break;
			case sourceExhausted: printf("sourceExhausted\t"); break;
			case targetExhausted: printf("targetExhausted\t"); break;
			case sourceIllegal: printf("sourceIllegal\t"); break;
		    }
		}
		if (result != conversionOK) {
			if (i <= UNI_SUR_HIGH_START || i > UNI_SUR_LOW_END) {
				printf("Test02A for %d, input %08x, output %04x,%04x, result %d\n",
				    i, utf32_buf[0], utf16_buf[0], utf16_buf[1], result);
				if ((i != UNI_SUR_HIGH_START) || (result != sourceIllegal)) {
					return 0;
				} else {
					printf("!!! Test02A: note expected illegal result for 0x0000D800\n");
				}
			}
		}
		if (i > UNI_SUR_HIGH_START && i <= UNI_SUR_LOW_END) continue;

		/*
		 * Test UTF16 -> UTF8, with legality check on.  We check for everything except
		 * for unpaired low surrogates.  We do make one check that the lowest low
		 * surrogate, when unpaired, is illegal.
		 */
		result = ConvertUTF16toUTF8((const UTF16 **) &utf16SourceStart, &(utf16_buf[2]), &utf8TargetStart, &(utf8_buf[7]), strictConversion);
		switch (result) {
		default: fprintf(stderr, "Test02B fatal error: result %d for input %08x\n", result, utf32_buf[0]); exit(1);
		case conversionOK: break;
		case sourceExhausted: printf("sourceExhausted\t"); break;
		case targetExhausted: printf("targetExhausted\t"); break;
		case sourceIllegal: printf("sourceIllegal\t"); break;
		}
		if (result != conversionOK) {
			printf("Test02B for %d (0x%x), input %04x,%04x; output %s; result %d\n",
				i, utf32_buf[0], utf16_buf[0], utf16_buf[1], utf8_buf, result);
			if ((i != UNI_SUR_LOW_START) && (i != UNI_SUR_HIGH_START)) {
				return 0;
			} else {
				/* Note: This illegal result only happens if we remove the surrogate
				    check in Test02A.  So it shouldn't be seen unless that check and
				    the "continue" are removed in the test above.
				*/
				if (i == UNI_SUR_LOW_START)
				    printf("!!! Test02B: note expected illegal result for 0xDC00,0000\n");
				else if (i == UNI_SUR_HIGH_START)
				    printf("!!! Test02B: note expected illegal result for 0xD800,0000\n");
			}
		}
		if ((i == UNI_SUR_LOW_START) && result != sourceIllegal) {
			printf("Test02B for %d (0x%x), input %04x,%04x; output %s; result %d\n",
				i, utf32_buf[0], utf16_buf[0], utf16_buf[1], utf8_buf, result);
				printf("Test02B: expected illegal result for 0xDC00,0000 was not flagged illegal.\n");
				return 0;
		}

		if ((i >= UNI_SUR_HIGH_START) & (i <= UNI_SUR_LOW_END)) continue;

		/*
		 * Reset some result buffer pointers for the trip back.
		 */
		utf32SourceStart = utf32_buf; utf32TargetStart = utf32_result;
		utf16TargetStart = utf16SourceStart = utf16_result;
		utf8TargetStart = utf8SourceStart = utf8_buf;

		/*
		 * Test UTF8 -> UTF16, with legality check on.
		 */
		result = ConvertUTF8toUTF16((const UTF8 **) &utf8SourceStart, &(utf8_buf[trailingBytesForUTF8[utf8_buf[0]]+1]), &utf16TargetStart, &(utf16_result[2]), strictConversion);
		switch (result) {
		default: fprintf(stderr, "Test02C fatal error: result %d for input %08x\n", result, utf32_buf[0]); exit(1);
		case conversionOK: break;
		case sourceExhausted: printf("sourceExhausted\t"); break;
		case targetExhausted: printf("targetExhausted\t"); break;
		case sourceIllegal: printf("sourceIllegal\t"); break;
		}
		if (result != conversionOK) {
			printf("Test02C for %d (0x%x), input %s; output %04x,%04x; result %d\n",
				i, utf32_buf[0], utf8_buf, utf16_buf[0], utf16_buf[1], result);
			return 0;
		}
		for (n = 0; n < 3; n++) { /* check that the utf16 result is the same as what went in. */
			if (utf16_buf[n] != utf16_result[n]) {
				printf("Test02C error: input = 0x%08x; utf16_buf = 0x%04x,0x%04x; utf16_result = 0x%04x,0x%04x\n",
					utf32_buf[0], utf16_buf[0], utf16_buf[1], utf16_result[0], utf16_result[1]);
				return 0;
			}
		}

		/*
		 * Test UTF16 -> UTF32, with legality check on.  If the result of our previous
		 * conversion gave us a "surrogate pair", then we need to convert 2 entities
		 * back to UTF32.
		 */
		if (utf16_result[0] >= UNI_SUR_HIGH_START && utf16_result[0] <= UNI_SUR_HIGH_END) {
			result = ConvertUTF16toUTF32((const UTF16 **) &utf16SourceStart, &(utf16_result[2]), &utf32TargetStart, &(utf32_result[1]), strictConversion);
		} else {
			result = ConvertUTF16toUTF32((const UTF16 **) &utf16SourceStart, &(utf16_result[1]), &utf32TargetStart, &(utf32_result[1]), strictConversion);
		}
		switch (result) {
		default: fprintf(stderr, "Test02D fatal error: result %d for input %08x\n", result, utf32_buf[0]); exit(1);
		case conversionOK: break;
		case sourceExhausted: printf("sourceExhausted\t"); break;
		case targetExhausted: printf("targetExhausted\t"); break;
		case sourceIllegal: printf("sourceIllegal\t"); break;
		}
		if (result != conversionOK) {
			printf("Test02D for %d (0x%x), input %04x,%04x; output %08x; result %d\n",
				i, utf32_buf[0], utf16_buf[0], utf16_buf[1], utf32_result[0], result);
			return 0;
		}

		/*
		 * Now, check the final round-trip value.
		 */
		if (utf32_buf[0] != utf32_result[0]) {
			printf("Test02E for %d: utf32 input %08x; trip output %08x (utf_16buf is %04x,%04x)\n", i, utf32_buf[0], utf32_result[0], utf16_buf[0], utf16_buf[1]);
			return 0;
		}
	}
	return 1;
}

/* ---------------------------------------------------------------------
	test03 - Test round trip UTF32 -> UTF8 -> UTF32

	This tests the functions that were not tested by test02 above.
	For each UTF32 value 0 through 0x10FFFF, it tests the conversion
	to UTF-8 and back.  The test is exhaustive.

   --------------------------------------------------------------------- */

int test03() {
	int i, n;
	ConversionResult result;
	UTF32 utf32_buf[2], utf32_result[2];
	UTF8 utf8_buf[8];
	UTF32 *utf32SourceStart, *utf32TargetStart;
	UTF8 *utf8SourceStart, *utf8TargetStart;

	printf("Begin Test03\n"); fflush(stdout);

	for (i = 0; i <= 0x10FFFF; i++) {
		/* Skip all surrogates except UNI_SUR_HIGH_START, which we test for illegality. */
		if (i > UNI_SUR_HIGH_START && i <= UNI_SUR_LOW_END) continue;

		utf32_buf[0] = i; utf32_buf[1] = 0;
		utf32_result[0] = utf32_result[1] = 0;
		for (n = 0; n < 8; n++) utf8_buf[n] = 0;

		utf32SourceStart = utf32_buf; utf32TargetStart = utf32_result;
		utf8TargetStart = utf8SourceStart = utf8_buf;

		/*
		 * Test UTF32 -> UTF8, with legality check on.
		 */
		result = ConvertUTF32toUTF8((const UTF32 **) &utf32SourceStart, &(utf32_buf[1]), & utf8TargetStart, &(utf8_buf[7]), strictConversion);
		switch (result) {
		default: fprintf(stderr, "Test03A fatal error: result %d for input %08x\n", result, utf32_buf[0]); exit(1);
		case conversionOK: break;
		case sourceExhausted: printf("sourceExhausted\t"); break;
		case targetExhausted: printf("targetExhausted\t"); break;
		case sourceIllegal: printf("sourceIllegal\t"); break;
		}
		if (result != conversionOK) {
			printf("Test03A for %d (0x%x); output %s; result %d\n",
				i, utf32_buf[0], utf8_buf, result);
			if (i != UNI_SUR_HIGH_START) {
				return 0;
			} else {
				printf("!!! Test03A: note expected illegal result for 0x0000D800\n");
			}
		}
		if ((i == UNI_SUR_HIGH_START) && result != sourceIllegal) {
			printf("Test03A for %d (0x%x); output %s; result %d\n",
				i, utf32_buf[0], utf8_buf, result);
				printf("Test03A: expected illegal result for 0x0000D800 was not flagged illegal.\n");
				return 0;
		}

		if ((i >= UNI_SUR_HIGH_START) & (i <= UNI_SUR_LOW_END)) continue;

		/*
		 * Reset some result buffer pointers for the trip back.
		 */
		utf32SourceStart = utf32_buf; utf32TargetStart = utf32_result;
		utf8TargetStart = utf8SourceStart = utf8_buf;

		/*
		 * Test UTF8 -> UTF32, with legality check on.
		 */
		result = ConvertUTF8toUTF32((const UTF8 **) &utf8SourceStart, &(utf8_buf[trailingBytesForUTF8[utf8_buf[0]]+1]), &utf32TargetStart, &(utf32_result[1]), strictConversion);
		switch (result) {
		default: fprintf(stderr, "Test03B fatal error: result %d for input %08x\n", result, utf32_buf[0]); exit(1);
		case conversionOK: break;
		case sourceExhausted: printf("sourceExhausted\t"); break;
		case targetExhausted: printf("targetExhausted\t"); break;
		case sourceIllegal: printf("sourceIllegal\t"); break;
		}
		if (result != conversionOK) {
			printf("Test03B for %d (0x%x), input %s; output 0x%08x; result %d\n",
				i, utf32_buf[0], utf8_buf, utf32_result[0], result);
			return 0;
		}

		/*
		 * Now, check the final round-trip value.
		 */
		if (utf32_buf[0] != utf32_result[0]) {
			printf("Test03C for %d: utf32 input %08x; utf8 buf %s; trip output %08x\n", i, utf32_buf[0], utf8_buf, utf32_result[0]);
			return 0;
		}
	}
	return 1;
}

/* ---------------------------------------------------------------------
	test04 - Test an illegal UTF-32 value > 10FFFF conversion to UTF-8.
	Expect it will be turned into UNI_REPLACEMENT_CHAR.

   --------------------------------------------------------------------- */

int test04() {
	int i, n;
	ConversionResult result;
	UTF32 utf32_buf[2];
	UTF8 utf8_buf[8];
	UTF32 *utf32SourceStart, *utf32TargetStart;
	UTF8 *utf8SourceStart, *utf8TargetStart;

	printf("Begin Test04\n"); fflush(stdout);

	i = 0x10FFFF + 21; /* an arbitrary value > legal */

	utf32_buf[0] = i; utf32_buf[1] = 0;
	for (n = 0; n < 8; n++) utf8_buf[n] = 0;

	utf32SourceStart = utf32_buf;
	utf8TargetStart = utf8_buf;

	/*
	 * Test UTF32 -> UTF8, with legality check on.
	 */
	result = ConvertUTF32toUTF8((const UTF32 **) &utf32SourceStart, &(utf32_buf[1]), & utf8TargetStart, &(utf8_buf[7]), strictConversion);
	if (result != sourceIllegal) {
		fprintf(stderr, "Test04A fatal error: result %d for input %08x\n", result, utf32_buf[0]); exit(1);
	}

	return 1;
}

/* --------------------------------------------------------------------- */

int main() {
	printf("Three tests of round-trip conversions will be performed.\n");
	printf("One test of illegal UTF-32 will be peroformed.\n");
	printf("Two illegal result messages are expected; one in test 02A; one in test 03A.\n");
	printf("These are for tests of Surrogate conversion.\n\n");
	fflush(stdout);
	if (test01()) {	printf("******** Test01 succeeded without error. ********\n\n"); }
	else { printf("-------- Test01 failed. --------\n\n"); }
	if (test02()) { printf("******** Test02 succeeded without error. ********\n\n"); }
	else { printf("-------- Test02 failed. --------\n\n"); }
	if (test03()) { printf("******** Test03 succeeded without error. ********\n\n"); }
	else { printf("-------- Test03 failed. --------\n\n"); }
	if (test04()) { printf("******** Test04 succeeded without error. ********\n\n"); }
	else { printf("-------- Test04 failed. --------\n\n"); }
	return 0;
}