Merged revisions 78603 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r78603 | victor.stinner | 2010-03-03 00:20:02 +0100 (mer., 03 mars 2010) | 5 lines
Issue #7820: The parser tokenizer restores all bytes in the right if the BOM
check fails.
Fix an assertion in pydebug mode.
........
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 5a1c268..50b0ff1 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -318,46 +318,51 @@
int set_readline(struct tok_state *, const char *),
struct tok_state *tok)
{
- int ch = get_char(tok);
+ int ch1, ch2, ch3;
+ ch1 = get_char(tok);
tok->decoding_state = STATE_RAW;
- if (ch == EOF) {
+ if (ch1 == EOF) {
return 1;
- } else if (ch == 0xEF) {
- ch = get_char(tok);
- if (ch != 0xBB) {
- unget_char(ch, tok);
- unget_char(0xEF, tok);
- /* any token beginning with '\xEF' is a bad token */
+ } else if (ch1 == 0xEF) {
+ ch2 = get_char(tok);
+ if (ch2 != 0xBB) {
+ unget_char(ch2, tok);
+ unget_char(ch1, tok);
return 1;
}
- ch = get_char(tok);
- if (ch != 0xBF) {
- unget_char(ch, tok);
- unget_char(0xBB, tok);
- unget_char(0xEF, tok);
- /* any token beginning with '\xEF' is a bad token */
+ ch3 = get_char(tok);
+ if (ch3 != 0xBF) {
+ unget_char(ch3, tok);
+ unget_char(ch2, tok);
+ unget_char(ch1, tok);
return 1;
}
#if 0
/* Disable support for UTF-16 BOMs until a decision
is made whether this needs to be supported. */
- } else if (ch == 0xFE) {
- ch = get_char(tok);
- if (ch != 0xFF)
- goto NON_BOM;
+ } else if (ch1 == 0xFE) {
+ ch2 = get_char(tok);
+ if (ch2 != 0xFF) {
+ unget_char(ch2, tok);
+ unget_char(ch1, tok);
+ return 1;
+ }
if (!set_readline(tok, "utf-16-be"))
return 0;
tok->decoding_state = STATE_NORMAL;
- } else if (ch == 0xFF) {
- ch = get_char(tok);
- if (ch != 0xFE)
- goto NON_BOM;
+ } else if (ch1 == 0xFF) {
+ ch2 = get_char(tok);
+ if (ch2 != 0xFE) {
+ unget_char(ch2, tok);
+ unget_char(ch1, tok);
+ return 1;
+ }
if (!set_readline(tok, "utf-16-le"))
return 0;
tok->decoding_state = STATE_NORMAL;
#endif
} else {
- unget_char(ch, tok);
+ unget_char(ch1, tok);
return 1;
}
if (tok->encoding != NULL)