Latin-1 source code was not being properly decoded when passed through
compile(). This was due to left-over special-casing before UTF-8 became the
default source encoding.
Closes issue #3574. Thanks to Victor Stinner for help with the patch.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 4edf6d0..ce8129d 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -135,6 +135,7 @@
tok->decoding_state = STATE_INIT;
tok->decoding_erred = 0;
tok->read_coding_spec = 0;
+ tok->enc = NULL;
tok->encoding = NULL;
tok->cont_line = 0;
#ifndef PGEN
@@ -274,8 +275,7 @@
tok->read_coding_spec = 1;
if (tok->encoding == NULL) {
assert(tok->decoding_state == STATE_RAW);
- if (strcmp(cs, "utf-8") == 0 ||
- strcmp(cs, "iso-8859-1") == 0) {
+ if (strcmp(cs, "utf-8") == 0) {
tok->encoding = cs;
} else {
r = set_readline(tok, cs);