Bring CJKCodecs 1.1 into trunk. This completely reorganizes source
and installed layouts to make maintenance simple and easy. And it
also adds four new codecs; big5hkscs, euc-jis-2004, shift-jis-2004
and iso2022-jp-2004.
diff --git a/Lib/test/cjkencodings_test.py b/Lib/test/cjkencodings_test.py
index 8bd0ee7..f5f9a377 100644
--- a/Lib/test/cjkencodings_test.py
+++ b/Lib/test/cjkencodings_test.py
@@ -63,6 +63,9 @@
"\x88\x91\xe5\x80\x91\xe6\x89\x80\x0a\xe8\xa6\x81\xe8\xa8\x8e\xe8"
"\xab\x96\xe7\x9a\x84\xe5\x95\x8f\xe9\xa1\x8c\xe5\xb0\xb1\xe6\x98"
"\xaf\x3a\x0a\x0a"),
+'big5hkscs': (
+"\x88\x45\x88\x5c\x8a\x73\x8b\xda\x8d\xd8\x0a",
+"\xf0\xa0\x84\x8c\xc4\x9a\xe9\xb5\xae\xe7\xbd\x93\xe6\xb4\x86\x0a"),
'cp949': (
"\x8c\x63\xb9\xe6\xb0\xa2\xc7\xcf\x20\xbc\x84\xbd\xc3\xc4\xdd\xb6"
"\xf3\x0a\x0a\xa8\xc0\xa8\xc0\xb3\xb3\x21\x21\x20\xec\xd7\xce\xfa"
diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py
index 269e00a..6d71218 100644
--- a/Lib/test/test_codecencodings_cn.py
+++ b/Lib/test/test_codecencodings_cn.py
@@ -3,7 +3,7 @@
# test_codecencodings_cn.py
# Codec encoding tests for PRC encodings.
#
-# $CJKCodecs: test_codecencodings_cn.py,v 1.1 2003/12/19 03:00:05 perky Exp $
+# $CJKCodecs: test_codecencodings_cn.py,v 1.2 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
@@ -27,8 +27,8 @@
tstring = test_multibytecodec_support.load_teststring('gbk')
codectests = (
# invalid bytes
- ("abc\x80\x80\xc1\xc4", "strict", None),
- ("abc\xc8", "strict", None),
+ ("abc\x80\x80\xc1\xc4", "strict", None),
+ ("abc\xc8", "strict", None),
("abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\u804a"),
("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u804a\ufffd"),
("abc\x80\x80\xc1\xc4", "ignore", u"abc\u804a"),
diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py
new file mode 100644
index 0000000..e7fad90
--- /dev/null
+++ b/Lib/test/test_codecencodings_hk.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+#
+# test_codecencodings_hk.py
+# Codec encoding tests for HongKong encodings.
+#
+# $CJKCodecs: test_codecencodings_hk.py,v 1.1 2004/07/10 17:35:20 perky Exp $
+
+from test import test_support
+from test import test_multibytecodec_support
+import unittest
+
+class Test_Big5HKSCS(test_multibytecodec_support.TestBase, unittest.TestCase):
+ encoding = 'big5hkscs'
+ tstring = test_multibytecodec_support.load_teststring('big5hkscs')
+ codectests = (
+ # invalid bytes
+ ("abc\x80\x80\xc1\xc4", "strict", None),
+ ("abc\xc8", "strict", None),
+ ("abc\x80\x80\xc1\xc4", "replace", u"abc\ufffd\u8b10"),
+ ("abc\x80\x80\xc1\xc4\xc8", "replace", u"abc\ufffd\u8b10\ufffd"),
+ ("abc\x80\x80\xc1\xc4", "ignore", u"abc\u8b10"),
+ )
+
+def test_main():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(Test_Big5HKSCS))
+ test_support.run_suite(suite)
+
+if __name__ == "__main__":
+ test_main()
diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py
index f574511..483b7db 100644
--- a/Lib/test/test_codecencodings_jp.py
+++ b/Lib/test/test_codecencodings_jp.py
@@ -3,7 +3,7 @@
# test_codecencodings_jp.py
# Codec encoding tests for Japanese encodings.
#
-# $CJKCodecs: test_codecencodings_jp.py,v 1.2 2004/01/06 09:25:37 perky Exp $
+# $CJKCodecs: test_codecencodings_jp.py,v 1.3 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
@@ -64,16 +64,6 @@
(u"\u203e", "strict", "\x7e"),
)
-class Test_EUC_JP_STRICT(test_multibytecodec_support.TestBase,
- unittest.TestCase):
- encoding = 'euc_jp_strict'
- tstring = test_multibytecodec_support.load_teststring('euc_jp')
- codectests = eucjp_commontests + (
- ("\xa1\xc0\\", "strict", u"\\\\"),
- (u"\xa5", "strict", None),
- (u"\u203e", "strict", None),
- )
-
shiftjis_commonenctests = (
("abc\x80\x80\x82\x84", "strict", None),
("abc\xf8", "strict", None),
@@ -90,14 +80,6 @@
("\x81\x5f\x81\x61\x81\x7c", "strict", u"\uff3c\u2016\u2212"),
)
-class Test_SJIS_STRICT(test_multibytecodec_support.TestBase, unittest.TestCase):
- encoding = 'shift_jis_strict'
- tstring = test_multibytecodec_support.load_teststring('shift_jis')
- codectests = shiftjis_commonenctests + (
- ("\\\x7e", "replace", u"\xa5\u203e"),
- ("\x81\x5f\x81\x61\x81\x7c", "replace", u"\x5c\u2016\u2212"),
- )
-
class Test_SJISX0213(test_multibytecodec_support.TestBase, unittest.TestCase):
encoding = 'shift_jisx0213'
tstring = test_multibytecodec_support.load_teststring('shift_jisx0213')
@@ -123,9 +105,6 @@
suite.addTest(unittest.makeSuite(Test_EUC_JISX0213))
suite.addTest(unittest.makeSuite(Test_EUC_JP_COMPAT))
suite.addTest(unittest.makeSuite(Test_SJIS_COMPAT))
- if test_multibytecodec_support.__cjkcodecs__:
- suite.addTest(unittest.makeSuite(Test_EUC_JP_STRICT))
- suite.addTest(unittest.makeSuite(Test_SJIS_STRICT))
suite.addTest(unittest.makeSuite(Test_SJISX0213))
test_support.run_suite(suite)
diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py
index 449237d..489c9f1 100644
--- a/Lib/test/test_codecencodings_kr.py
+++ b/Lib/test/test_codecencodings_kr.py
@@ -3,7 +3,7 @@
# test_codecencodings_kr.py
# Codec encoding tests for ROK encodings.
#
-# $CJKCodecs: test_codecencodings_kr.py,v 1.1 2003/12/19 03:00:06 perky Exp $
+# $CJKCodecs: test_codecencodings_kr.py,v 1.2 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py
index 897f060..fb8a4d0 100644
--- a/Lib/test/test_codecencodings_tw.py
+++ b/Lib/test/test_codecencodings_tw.py
@@ -3,7 +3,7 @@
# test_codecencodings_tw.py
# Codec encoding tests for ROC encodings.
#
-# $CJKCodecs: test_codecencodings_tw.py,v 1.1 2003/12/19 03:00:06 perky Exp $
+# $CJKCodecs: test_codecencodings_tw.py,v 1.2 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
diff --git a/Lib/test/test_codecmaps_cn.py b/Lib/test/test_codecmaps_cn.py
index 337ac91..31871e2 100644
--- a/Lib/test/test_codecmaps_cn.py
+++ b/Lib/test/test_codecmaps_cn.py
@@ -3,7 +3,7 @@
# test_codecmaps_cn.py
# Codec mapping tests for PRC encodings
#
-# $CJKCodecs: test_codecmaps_cn.py,v 1.2 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_codecmaps_cn.py,v 1.3 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
diff --git a/Lib/test/test_codecmaps_hk.py b/Lib/test/test_codecmaps_hk.py
new file mode 100644
index 0000000..43137b6
--- /dev/null
+++ b/Lib/test/test_codecmaps_hk.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+#
+# test_codecmaps_hk.py
+# Codec mapping tests for HongKong encodings
+#
+# $CJKCodecs: test_codecmaps_hk.py,v 1.1 2004/07/10 17:35:20 perky Exp $
+
+from test import test_support
+from test import test_multibytecodec_support
+import unittest
+
+class TestBig5HKSCSMap(test_multibytecodec_support.TestBase_Mapping,
+ unittest.TestCase):
+ encoding = 'big5hkscs'
+ mapfilename = 'BIG5HKSCS.TXT'
+ mapfileurl = 'http://people.freebsd.org/~perky/i18n/BIG5HKSCS.TXT'
+
+def test_main():
+ suite = unittest.TestSuite()
+ suite.addTest(unittest.makeSuite(TestBig5HKSCSMap))
+ test_support.run_suite(suite)
+
+test_multibytecodec_support.register_skip_expected(TestBig5HKSCSMap)
+if __name__ == "__main__":
+ test_main()
diff --git a/Lib/test/test_codecmaps_jp.py b/Lib/test/test_codecmaps_jp.py
index 9519da2..014a952 100644
--- a/Lib/test/test_codecmaps_jp.py
+++ b/Lib/test/test_codecmaps_jp.py
@@ -3,7 +3,7 @@
# test_codecmaps_jp.py
# Codec mapping tests for Japanese encodings
#
-# $CJKCodecs: test_codecmaps_jp.py,v 1.2 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_codecmaps_jp.py,v 1.3 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
@@ -48,15 +48,6 @@
('\x81_', u'\\'),
]
-
-class TestSJISSTRICTMap(test_multibytecodec_support.TestBase_Mapping,
- unittest.TestCase):
- encoding = 'shift_jis_strict'
- mapfilename = 'SHIFTJIS.TXT'
- mapfileurl = 'http://www.unicode.org/Public/MAPPINGS/OBSOLETE' \
- '/EASTASIA/JIS/SHIFTJIS.TXT'
-
-
class TestEUCJISX0213Map(test_multibytecodec_support.TestBase_Mapping,
unittest.TestCase):
encoding = 'euc_jisx0213'
@@ -76,8 +67,6 @@
suite.addTest(unittest.makeSuite(TestCP932Map))
suite.addTest(unittest.makeSuite(TestEUCJPCOMPATMap))
suite.addTest(unittest.makeSuite(TestSJISCOMPATMap))
- if test_multibytecodec_support.__cjkcodecs__:
- suite.addTest(unittest.makeSuite(TestSJISSTRICTMap))
suite.addTest(unittest.makeSuite(TestEUCJISX0213Map))
suite.addTest(unittest.makeSuite(TestSJISX0213Map))
test_support.run_suite(suite)
@@ -85,5 +74,6 @@
test_multibytecodec_support.register_skip_expected(TestCP932Map,
TestEUCJPCOMPATMap, TestSJISCOMPATMap, TestEUCJISX0213Map,
TestSJISX0213Map)
+
if __name__ == "__main__":
test_main()
diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py
index 9a71939..be19811 100644
--- a/Lib/test/test_codecmaps_kr.py
+++ b/Lib/test/test_codecmaps_kr.py
@@ -3,7 +3,7 @@
# test_codecmaps_kr.py
# Codec mapping tests for ROK encodings
#
-# $CJKCodecs: test_codecmaps_kr.py,v 1.2 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_codecmaps_kr.py,v 1.3 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py
index 2f41660..61d83cd 100644
--- a/Lib/test/test_codecmaps_tw.py
+++ b/Lib/test/test_codecmaps_tw.py
@@ -3,7 +3,7 @@
# test_codecmaps_tw.py
# Codec mapping tests for ROC encodings
#
-# $CJKCodecs: test_codecmaps_tw.py,v 1.2 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_codecmaps_tw.py,v 1.3 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py
index e96afba..acf333e 100644
--- a/Lib/test/test_multibytecodec.py
+++ b/Lib/test/test_multibytecodec.py
@@ -3,7 +3,7 @@
# test_multibytecodec.py
# Unit test for multibytecodec itself
#
-# $CJKCodecs: test_multibytecodec.py,v 1.5 2004/01/06 02:26:28 perky Exp $
+# $CJKCodecs: test_multibytecodec.py,v 1.8 2004/06/19 06:09:55 perky Exp $
from test import test_support
from test import test_multibytecodec_support
@@ -69,6 +69,12 @@
def test_str_decode(self):
self.assertEqual('abcd'.encode('gb18030'), 'abcd')
+ def test_streamwriter_strwrite(self):
+ s = StringIO.StringIO()
+ wr = codecs.getwriter('gb18030')(s)
+ wr.write('abcd')
+ self.assertEqual(s.getvalue(), 'abcd')
+
def test_main():
suite = unittest.TestSuite()
suite.addTest(unittest.makeSuite(Test_StreamWriter))
diff --git a/Lib/test/test_multibytecodec_support.py b/Lib/test/test_multibytecodec_support.py
index f471adf..22b52f7 100644
--- a/Lib/test/test_multibytecodec_support.py
+++ b/Lib/test/test_multibytecodec_support.py
@@ -3,7 +3,7 @@
# test_multibytecodec_support.py
# Common Unittest Routines for CJK codecs
#
-# $CJKCodecs: test_multibytecodec_support.py,v 1.5 2004/01/17 12:47:19 perky Exp $
+# $CJKCodecs: test_multibytecodec_support.py,v 1.6 2004/06/19 06:09:55 perky Exp $
import sys, codecs, os.path
import unittest
@@ -164,13 +164,8 @@
def __init__(self, *args, **kw):
unittest.TestCase.__init__(self, *args, **kw)
if not os.path.exists(self.mapfilename):
- parent = os.path.join(os.pardir, self.mapfilename)
- if not os.path.exists(parent):
- format = '%s not found, download from %s'
- raise test_support.TestSkipped(format %
- (self.mapfilename, self.mapfileurl))
- else:
- self.mapfilename = parent
+ raise test_support.TestSkipped('%s not found, download from %s' %
+ (self.mapfilename, self.mapfileurl))
def test_mapping_file(self):
unichrs = lambda s: u''.join(map(unichr, map(eval, s.split('+'))))