bpo-24214: Fixed the UTF-8 and UTF-16 incremental decoders. (GH-14304)
* The UTF-8 incremental decoders fails now fast if encounter
a sequence that can't be handled by the error handler.
* The UTF-16 incremental decoders with the surrogatepass error
handler decodes now a lone low surrogate with final=False.
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 625be4b..cb1456e 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4945,11 +4945,15 @@
endinpos = startinpos + 1;
break;
case 2:
- case 3:
- case 4:
- if (s == end || consumed) {
+ if (consumed && (unsigned char)s[0] == 0xED && end - s == 2
+ && (unsigned char)s[1] >= 0xA0 && (unsigned char)s[1] <= 0xBF)
+ {
+ /* Truncated surrogate code in range D800-DFFF */
goto End;
}
+ /* fall through */
+ case 3:
+ case 4:
errmsg = "invalid continuation byte";
startinpos = s - starts;
endinpos = startinpos + ch - 1;