bpo-30003: Fix handling escape characters in HZ codec (#1720) (#1556)
diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py
index fdae538..a104937 100644
--- a/Lib/test/test_codecencodings_cn.py
+++ b/Lib/test/test_codecencodings_cn.py
@@ -82,6 +82,10 @@
(b'ab~cd', 'replace', u'ab\uFFFDd'),
(b'ab\xffcd', 'replace', u'ab\uFFFDcd'),
(b'ab~{\x81\x81\x41\x44~}cd', 'replace', u'ab\uFFFD\uFFFD\u804Acd'),
+ # issue 30003
+ (u'ab~cd', 'strict', b'ab~~cd'), # escape ~
+ (b'~{Dc~~:C~}', 'strict', None), # ~~ only in ASCII mode
+ (b'~{Dc~\n:C~}', 'strict', None), # ~\n only in ASCII mode
)
def test_main():
diff --git a/Misc/NEWS b/Misc/NEWS
index 254bb52..938a029 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -49,6 +49,9 @@
Library
-------
+- bpo-30003: Fix handling escape characters in HZ codec. Based on patch
+ by Ma Lin.
+
- bpo-30375: Warnings emitted when compile a regular expression now always
point to the line in the user code. Previously they could point into inners
of the re module if emitted from inside of groups or conditionals.
diff --git a/Modules/cjkcodecs/_codecs_cn.c b/Modules/cjkcodecs/_codecs_cn.c
index 3bc652f..92cf06d 100644
--- a/Modules/cjkcodecs/_codecs_cn.c
+++ b/Modules/cjkcodecs/_codecs_cn.c
@@ -335,15 +335,17 @@
DBCHAR code;
if (c < 0x80) {
- if (state->i == 0) {
- WRITE1((unsigned char)c)
- NEXT(1, 1)
- }
- else {
- WRITE3('~', '}', (unsigned char)c)
- NEXT(1, 3)
+ if (state->i) {
+ WRITE2('~', '}')
+ NEXT_OUT(2)
state->i = 0;
}
+ WRITE1((unsigned char)c)
+ NEXT(1, 1)
+ if (c == '~') {
+ WRITE1('~')
+ NEXT_OUT(1)
+ }
continue;
}
@@ -390,20 +392,19 @@
unsigned char c2 = IN2;
REQUIRE_INBUF(2)
- if (c2 == '~') {
+ if (c2 == '~' && state->i == 0) {
WRITE1('~')
- NEXT(2, 1)
- continue;
+ NEXT_OUT(1)
}
else if (c2 == '{' && state->i == 0)
state->i = 1; /* set GB */
+ else if (c2 == '\n' && state->i == 0)
+ ; /* line-continuation */
else if (c2 == '}' && state->i == 1)
state->i = 0; /* set ASCII */
- else if (c2 == '\n')
- ; /* line-continuation */
else
return 2;
- NEXT(2, 0);
+ NEXT_IN(2)
continue;
}