ignore the coding cookie in compile(), exec(), and eval() if the source is a string #4626
diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index d8ff6ee..4c3b506 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -49,7 +49,11 @@
initerr(err_ret, filename);
- if ((tok = PyTokenizer_FromString(s)) == NULL) {
+ if (*flags & PyPARSE_IGNORE_COOKIE)
+ tok = PyTokenizer_FromUTF8(s);
+ else
+ tok = PyTokenizer_FromString(s);
+ if (tok == NULL) {
err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
return NULL;
}
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 3d52bed..c4f447d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -715,6 +715,28 @@
return tok;
}
+struct tok_state *
+PyTokenizer_FromUTF8(const char *str)
+{
+ struct tok_state *tok = tok_new();
+ if (tok == NULL)
+ return NULL;
+ tok->decoding_state = STATE_RAW;
+ tok->read_coding_spec = 1;
+ tok->enc = NULL;
+ tok->str = str;
+ tok->encoding = (char *)PyMem_MALLOC(6);
+ if (!tok->encoding) {
+ PyTokenizer_Free(tok);
+ return NULL;
+ }
+ strcpy(tok->encoding, "utf-8");
+
+ /* XXX: constify members. */
+ tok->buf = tok->cur = tok->end = tok->inp = (char*)str;
+ return tok;
+}
+
/* Set up tokenizer for file */
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index df9cbc7..e3328f1 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -61,6 +61,7 @@
};
extern struct tok_state *PyTokenizer_FromString(const char *);
+extern struct tok_state *PyTokenizer_FromUTF8(const char *);
extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
char *, char *);
extern void PyTokenizer_Free(struct tok_state *);