fix several compile() issues by translating newlines in the tokenizer
diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst
index ff7c4b6..ddece5a 100644
--- a/Doc/library/functions.rst
+++ b/Doc/library/functions.rst
@@ -173,11 +173,10 @@
.. note::
- When compiling a string with multi-line statements, line endings must be
- represented by a single newline character (``'\n'``), and the input must
- be terminated by at least one newline character. If line endings are
- represented by ``'\r\n'``, use :meth:`str.replace` to change them into
- ``'\n'``.
+ When compiling a string with multi-line statements in ``'single'`` or
+ ``'eval'`` mode, input must be terminated by at least one newline
+ character. This is to facilitate detection of incomplete and complete
+ statements in the :mod:`code` module.
.. versionchanged:: 2.3
The *flags* and *dont_inherit* arguments were added.
@@ -185,6 +184,10 @@
.. versionchanged:: 2.6
Support for compiling AST objects.
+ .. versionchanged:: 2.7
+ Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode
+ does not have to end in a newline anymore.
+
.. function:: complex([real[, imag]])
diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py
index c8fa990..da3b83f 100644
--- a/Lib/test/test_codeop.py
+++ b/Lib/test/test_codeop.py
@@ -295,10 +295,6 @@
self.assertNotEquals(compile_command("a = 1\n", "abc").co_filename,
compile("a = 1\n", "def", 'single').co_filename)
- def test_no_universal_newlines(self):
- code = compile_command("'\rfoo\r'", symbol='eval')
- self.assertEqual(eval(code), '\rfoo\r')
-
def test_main():
run_unittest(CodeopTests)
diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py
index 75c983a..28b7332 100644
--- a/Lib/test/test_compile.py
+++ b/Lib/test/test_compile.py
@@ -5,6 +5,19 @@
class TestSpecifics(unittest.TestCase):
+ def test_no_ending_newline(self):
+ compile("hi", "<test>", "exec")
+ compile("hi\r", "<test>", "exec")
+
+ def test_empty(self):
+ compile("", "<test>", "exec")
+
+ def test_other_newlines(self):
+ compile("\r\n", "<test>", "exec")
+ compile("\r", "<test>", "exec")
+ compile("hi\r\nstuff\r\ndef f():\n pass\r", "<test>", "exec")
+ compile("this_is\rreally_old_mac\rdef f():\n pass", "<test>", "exec")
+
def test_debug_assignment(self):
# catch assignments to __debug__
self.assertRaises(SyntaxError, compile, '__debug__ = 1', '?', 'single')
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py
index 7d059c2..ad5c5be 100644
--- a/Lib/test/test_parser.py
+++ b/Lib/test/test_parser.py
@@ -243,9 +243,9 @@
(14, '+', 2, 13),
(2, '1', 2, 15),
(4, '', 2, 16),
- (6, '', 2, -1),
- (4, '', 2, -1),
- (0, '', 2, -1)],
+ (6, '', 3, -1),
+ (4, '', 3, -1),
+ (0, '', 3, -1)],
terminals)
diff --git a/Misc/NEWS b/Misc/NEWS
index cf1f54a..2ae3736 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,9 @@
Core and Builtins
-----------------
+- Fix several issues with compile(). The input can now contain Windows and Mac
+ newlines and is no longer required to end in a newline.
+
- Remove length limitation when constructing a complex number from a
unicode string.
diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index 3994add..7f2fb36 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -51,7 +51,7 @@
initerr(err_ret, filename);
- if ((tok = PyTokenizer_FromString(s)) == NULL) {
+ if ((tok = PyTokenizer_FromString(s, start == file_input)) == NULL) {
err_ret->error = PyErr_Occurred() ? E_DECODE : E_NOMEM;
return NULL;
}
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index a49e9f0..1808c41 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -105,6 +105,7 @@
tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
tok->done = E_OK;
tok->fp = NULL;
+ tok->input = NULL;
tok->tabsize = TABSIZE;
tok->indent = 0;
tok->indstack[0] = 0;
@@ -130,6 +131,17 @@
return tok;
}
+static char *
+new_string(const char *s, Py_ssize_t len)
+{
+ char* result = (char *)PyMem_MALLOC(len + 1);
+ if (result != NULL) {
+ memcpy(result, s, len);
+ result[len] = '\0';
+ }
+ return result;
+}
+
#ifdef PGEN
static char *
@@ -144,10 +156,10 @@
return feof(tok->fp);
}
-static const char *
-decode_str(const char *str, struct tok_state *tok)
+static char *
+decode_str(const char *str, int exec_input, struct tok_state *tok)
{
- return str;
+ return new_string(str, strlen(str));
}
#else /* PGEN */
@@ -162,16 +174,6 @@
return NULL; /* as if it were EOF */
}
-static char *
-new_string(const char *s, Py_ssize_t len)
-{
- char* result = (char *)PyMem_MALLOC(len + 1);
- if (result != NULL) {
- memcpy(result, s, len);
- result[len] = '\0';
- }
- return result;
-}
static char *
get_normal_name(char *s) /* for utf-8 and latin-1 */
@@ -586,17 +588,63 @@
}
#endif
+
+static char *
+translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
+ int skip_next_lf = 0, length = strlen(s), final_length;
+ char *buf, *current;
+ char c;
+ buf = PyMem_MALLOC(length + 2);
+ if (buf == NULL) {
+ tok->done = E_NOMEM;
+ return NULL;
+ }
+ for (current = buf; (c = *s++);) {
+ if (skip_next_lf) {
+ skip_next_lf = 0;
+ if (c == '\n') {
+ c = *s;
+ s++;
+ if (!c)
+ break;
+ }
+ }
+ if (c == '\r') {
+ skip_next_lf = 1;
+ c = '\n';
+ }
+ *current = c;
+ current++;
+ }
+ /* If this is exec input, add a newline to the end of the file if
+ there isn't one already. */
+ if (exec_input && *current != '\n') {
+ *current = '\n';
+ current++;
+ }
+ *current = '\0';
+ final_length = current - buf;
+ if (final_length < length && final_length)
+ /* should never fail */
+ buf = PyMem_REALLOC(buf, final_length + 1);
+ return buf;
+}
+
/* Decode a byte string STR for use as the buffer of TOK.
Look for encoding declarations inside STR, and record them
inside TOK. */
static const char *
-decode_str(const char *str, struct tok_state *tok)
+decode_str(const char *input, int single, struct tok_state *tok)
{
PyObject* utf8 = NULL;
+ const char *str;
const char *s;
const char *newl[2] = {NULL, NULL};
int lineno = 0;
+ tok->input = str = translate_newlines(input, single, tok);
+ if (str == NULL)
+ return NULL;
tok->enc = NULL;
tok->str = str;
if (!check_bom(buf_getc, buf_ungetc, buf_setreadl, tok))
@@ -651,12 +699,12 @@
/* Set up tokenizer for string */
struct tok_state *
-PyTokenizer_FromString(const char *str)
+PyTokenizer_FromString(const char *str, int exec_input)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
- str = (char *)decode_str(str, tok);
+ str = (char *)decode_str(str, exec_input, tok);
if (str == NULL) {
PyTokenizer_Free(tok);
return NULL;
@@ -702,6 +750,8 @@
#endif
if (tok->fp != NULL && tok->buf != NULL)
PyMem_FREE(tok->buf);
+ if (tok->input)
+ PyMem_FREE((char *)tok->input);
PyMem_FREE(tok);
}
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index e10972c..79c9e28 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -52,9 +52,10 @@
#endif
const char* enc;
const char* str;
+ const char* input; /* Tokenizer's newline translated copy of the string. */
};
-extern struct tok_state *PyTokenizer_FromString(const char *);
+extern struct tok_state *PyTokenizer_FromString(const char *, int);
extern struct tok_state *PyTokenizer_FromFile(FILE *, char *, char *);
extern void PyTokenizer_Free(struct tok_state *);
extern int PyTokenizer_Get(struct tok_state *, char **, char **);