Convert code from sys.stdin.encoding to UTF-8 in
interactive mode. Fixes #1100.
diff --git a/Include/parsetok.h b/Include/parsetok.h
index 2b4ce1e..71033dc 100644
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -34,7 +34,8 @@
PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *, grammar *,
+PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
+ const char*, grammar *,
int, char *, char *,
perrdetail *, int);
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index 08278cf..607826a 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -40,7 +40,8 @@
PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
int, PyCompilerFlags *flags,
PyArena *);
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *, int,
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *,
+ const char*, int,
char *, char *,
PyCompilerFlags *, int *,
PyArena *);
diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index 71bed29..b9664ea 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -59,19 +59,20 @@
PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
char *ps1, char *ps2, perrdetail *err_ret)
{
- return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
- err_ret, 0);
+ return PyParser_ParseFileFlags(fp, filename, NULL,
+ g, start, ps1, ps2, err_ret, 0);
}
node *
-PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
+PyParser_ParseFileFlags(FILE *fp, const char *filename, const char* enc,
+ grammar *g, int start,
char *ps1, char *ps2, perrdetail *err_ret, int flags)
{
struct tok_state *tok;
initerr(err_ret, filename);
- if ((tok = PyTokenizer_FromFile(fp, ps1, ps2)) == NULL) {
+ if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
err_ret->error = E_NOMEM;
return NULL;
}
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 776183d..7f51e14 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -677,7 +677,7 @@
/* Set up tokenizer for file */
struct tok_state *
-PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
+PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
@@ -691,6 +691,17 @@
tok->fp = fp;
tok->prompt = ps1;
tok->nextprompt = ps2;
+ if (enc != NULL) {
+ /* Must copy encoding declaration since it
+ gets copied into the parse tree. */
+ tok->encoding = PyMem_MALLOC(strlen(enc)+1);
+ if (!tok->encoding) {
+ PyTokenizer_Free(tok);
+ return NULL;
+ }
+ strcpy(tok->encoding, enc);
+ tok->decoding_state = -1;
+ }
return tok;
}
@@ -742,6 +753,29 @@
}
if (tok->prompt != NULL) {
char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
+#ifndef PGEN
+ if (tok->encoding && newtok && *newtok) {
+ /* Recode to UTF-8 */
+ Py_ssize_t buflen;
+ const char* buf;
+ PyObject *u = translate_into_utf8(newtok, tok->encoding);
+ PyMem_FREE(newtok);
+ if (!u) {
+ tok->done = E_DECODE;
+ return EOF;
+ }
+ buflen = PyBytes_Size(u);
+ buf = PyBytes_AsString(u);
+ if (!buf) {
+ Py_DECREF(u);
+ tok->done = E_DECODE;
+ return EOF;
+ }
+ newtok = PyMem_MALLOC(buflen+1);
+ strcpy(newtok, buf);
+ Py_DECREF(u);
+ }
+#endif
if (tok->nextprompt != NULL)
tok->prompt = tok->nextprompt;
if (newtok == NULL)
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 5e7ebf7..ba90a5f 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -55,7 +55,8 @@
};
extern struct tok_state *PyTokenizer_FromString(const char *);
-extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
+extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
+ char *, char *);
extern void PyTokenizer_Free(struct tok_state *);
extern int PyTokenizer_Get(struct tok_state *, char **, char **);
diff --git a/Python/import.c b/Python/import.c
index 7283783..c2f42e9 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -809,7 +809,8 @@
if (arena == NULL)
return NULL;
- mod = PyParser_ASTFromFile(fp, pathname, Py_file_input, 0, 0, 0,
+ mod = PyParser_ASTFromFile(fp, pathname, NULL,
+ Py_file_input, 0, 0, 0,
NULL, arena);
if (mod) {
co = PyAST_Compile(mod, pathname, NULL, arena);
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index eeed820..d89f5f9 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -744,12 +744,22 @@
int
PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags)
{
- PyObject *m, *d, *v, *w;
+ PyObject *m, *d, *v, *w, *oenc = NULL;
mod_ty mod;
PyArena *arena;
- char *ps1 = "", *ps2 = "";
+ char *ps1 = "", *ps2 = "", *enc = NULL;
int errcode = 0;
+ if (fp == stdin) {
+ /* Fetch encoding from sys.stdin */
+ v = PySys_GetObject("stdin");
+ if (!v)
+ return -1;
+ oenc = PyObject_GetAttrString(v, "encoding");
+ if (!oenc)
+ return -1;
+ enc = PyUnicode_AsString(oenc);
+ }
v = PySys_GetObject("ps1");
if (v != NULL) {
v = PyObject_Str(v);
@@ -770,13 +780,15 @@
if (arena == NULL) {
Py_XDECREF(v);
Py_XDECREF(w);
+ Py_XDECREF(oenc);
return -1;
}
- mod = PyParser_ASTFromFile(fp, filename,
+ mod = PyParser_ASTFromFile(fp, filename, enc,
Py_single_input, ps1, ps2,
flags, &errcode, arena);
Py_XDECREF(v);
Py_XDECREF(w);
+ Py_XDECREF(oenc);
if (mod == NULL) {
PyArena_Free(arena);
if (errcode == E_EOF) {
@@ -1254,7 +1266,7 @@
if (arena == NULL)
return NULL;
- mod = PyParser_ASTFromFile(fp, filename, start, 0, 0,
+ mod = PyParser_ASTFromFile(fp, filename, NULL, start, 0, 0,
flags, NULL, arena);
if (closeit)
fclose(fp);
@@ -1379,13 +1391,15 @@
}
mod_ty
-PyParser_ASTFromFile(FILE *fp, const char *filename, int start, char *ps1,
+PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc,
+ int start, char *ps1,
char *ps2, PyCompilerFlags *flags, int *errcode,
PyArena *arena)
{
mod_ty mod;
perrdetail err;
- node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+ node *n = PyParser_ParseFileFlags(fp, filename, enc,
+ &_PyParser_Grammar,
start, ps1, ps2, &err, PARSER_FLAGS(flags));
if (n) {
mod = PyAST_FromNode(n, flags, filename, arena);
@@ -1406,7 +1420,8 @@
PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags)
{
perrdetail err;
- node *n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+ node *n = PyParser_ParseFileFlags(fp, filename, NULL,
+ &_PyParser_Grammar,
start, NULL, NULL, &err, flags);
if (n == NULL)
err_input(&err);