Merged revisions 76230 via svnmerge from
svn+ssh://pythondev@svn.python.org/python/trunk
........
r76230 | benjamin.peterson | 2009-11-12 17:39:44 -0600 (Thu, 12 Nov 2009) | 2 lines
fix several compile() issues by translating newlines in the tokenizer
........
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index e3328f1..1a81e33 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -58,10 +58,11 @@
#endif
const char* enc; /* Encoding for the current str. */
const char* str;
+ const char* input; /* Tokenizer's newline translated copy of the string. */
};
-extern struct tok_state *PyTokenizer_FromString(const char *);
-extern struct tok_state *PyTokenizer_FromUTF8(const char *);
+extern struct tok_state *PyTokenizer_FromString(const char *, int);
+extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
char *, char *);
extern void PyTokenizer_Free(struct tok_state *);