ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 3d52bed..c4f447d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -715,6 +715,28 @@
return tok;
}
+struct tok_state *
+PyTokenizer_FromUTF8(const char *str)
+{
+ struct tok_state *tok = tok_new();
+ if (tok == NULL)
+ return NULL;
+ tok->decoding_state = STATE_RAW;
+ tok->read_coding_spec = 1;
+ tok->enc = NULL;
+ tok->str = str;
+ tok->encoding = (char *)PyMem_MALLOC(6);
+ if (!tok->encoding) {
+ PyTokenizer_Free(tok);
+ return NULL;
+ }
+ strcpy(tok->encoding, "utf-8");
+
+ /* XXX: constify members. */
+ tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
+ return tok;
+}
+
/* Set up tokenizer for file */