commit d32594ad27f48a898d42a0ea30b9d007b1c57de9
author Miss Islington (bot) <31488909+miss-islington@users.noreply.github.com> Tue Jun 25 02:12:16 2019 -0700
committer GitHub <noreply@github.com> Tue Jun 25 02:12:16 2019 -0700
tree 5a21486f006076a0596955e272c90726fdcc1201
parent 26fe6c35374fa32577b230b856a92a3b094e08ed

bpo-24214: Fixed the UTF-8 and UTF-16 incremental decoders. (GH-14304)


* The UTF-8 incremental decoders fails now fast if encounter
  a sequence that can't be handled by the error handler.
* The UTF-16 incremental decoders with the surrogatepass error
  handler decodes now a lone low surrogate with final=False.
(cherry picked from commit 894263ba80af4b7733c2df95b527e96953922656)

Co-authored-by: Serhiy Storchaka <storchaka@gmail.com>

Objects/unicodeobject.c

diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 6ec4127..132f57a 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -4937,11 +4937,15 @@
             endinpos = startinpos + 1;
             break;
         case 2:
-        case 3:
-        case 4:
-            if (s == end || consumed) {
+            if (consumed && (unsigned char)s[0] == 0xED && end - s == 2
+                && (unsigned char)s[1] >= 0xA0 && (unsigned char)s[1] <= 0xBF)
+            {
+                /* Truncated surrogate code in range D800-DFFF */
                 goto End;
             }
+            /* fall through */
+        case 3:
+        case 4:
             errmsg = "invalid continuation byte";
             startinpos = s - starts;
             endinpos = startinpos + ch - 1;