Latin-1 source code was not being properly decoded when passed through
compile(). This was due to left-over special-casing before UTF-8 became the
default source encoding.
Closes issue #3574. Thanks to Victor Stinner for help with the patch.
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index c45dea1..df9cbc7 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -49,14 +49,14 @@
enum decoding_state decoding_state;
int decoding_erred; /* whether erred in decoding */
int read_coding_spec; /* whether 'coding:...' has been read */
- char *encoding;
+ char *encoding; /* Source encoding. */
int cont_line; /* whether we are in a continuation line. */
const char* line_start; /* pointer to start of current line */
#ifndef PGEN
PyObject *decoding_readline; /* codecs.open(...).readline */
PyObject *decoding_buffer;
#endif
- const char* enc;
+ const char* enc; /* Encoding for the current str. */
const char* str;
};