commit 2cded9c3f31d2fea4b033f44eaa828e508f03391
author Victor Stinner <victor.stinner@haypocalc.com> Fri Jul 08 01:45:13 2011 +0200
committer Victor Stinner <victor.stinner@haypocalc.com> Fri Jul 08 01:45:13 2011 +0200
tree 1554d9f0baa575b7ae791ff1267c4e493a1b36bf
parent 081fe46ff96bccb1a256c356443b625b467814c8

Issue #12016: Multibyte CJK decoders now resynchronize faster

They only ignore the first byte of an invalid byte sequence.

For example, b'\xff\n'.decode('gb2312', 'replace') gives '\ufffd\n' instead of
'\ufffd'.

Modules/cjkcodecs/_codecs_hk.c

diff --git a/Modules/cjkcodecs/_codecs_hk.c b/Modules/cjkcodecs/_codecs_hk.c
index 558a42f..d3ad04b 100644
--- a/Modules/cjkcodecs/_codecs_hk.c
+++ b/Modules/cjkcodecs/_codecs_hk.c
@@ -161,7 +161,7 @@
         case 0x8864: WRITE2(0x00ca, 0x030c); break;
         case 0x88a3: WRITE2(0x00ea, 0x0304); break;
         case 0x88a5: WRITE2(0x00ea, 0x030c); break;
-        default: return 2;
+        default: return 1;
         }
 
         NEXT(2, 2) /* all decoded codepoints are pairs, above. */