Ugly. A pile of new xxxFlags() functions, to communicate to the parser
that 'yield' is a keyword. This doesn't help test_generators at all! I
don't know why not. These things do work now (and didn't before this
patch):
1. "from __future__ import generators" now works in a native shell.
2. Similarly "python -i xxx.py" now has generators enabled in the
shell if xxx.py had them enabled.
3. This program (which was my doctest proxy) works fine:
from __future__ import generators
source = """\
def f():
yield 1
"""
exec compile(source, "", "single") in globals()
print type(f())
diff --git a/Include/parsetok.h b/Include/parsetok.h
index aebc83c..66359d42 100644
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -17,11 +17,19 @@
int expected;
} perrdetail;
+#define PyPARSE_YIELD_IS_KEYWORD 0x0001
+
extern DL_IMPORT(node *) PyParser_ParseString(char *, grammar *, int,
perrdetail *);
extern DL_IMPORT(node *) PyParser_ParseFile (FILE *, char *, grammar *, int,
char *, char *, perrdetail *);
+extern DL_IMPORT(node *) PyParser_ParseStringFlags(char *, grammar *, int,
+ perrdetail *, int);
+extern DL_IMPORT(node *) PyParser_ParseFileFlags(FILE *, char *, grammar *,
+ int, char *, char *,
+ perrdetail *, int);
+
#ifdef __cplusplus
}
#endif
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index 3877a82..6e9821b 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -45,6 +45,9 @@
DL_IMPORT(struct _node *) PyParser_SimpleParseString(char *, int);
DL_IMPORT(struct _node *) PyParser_SimpleParseFile(FILE *, char *, int);
+DL_IMPORT(struct _node *) PyParser_SimpleParseStringFlags(char *, int, int);
+DL_IMPORT(struct _node *) PyParser_SimpleParseFileFlags(FILE *, char *,
+ int, int);
DL_IMPORT(PyObject *) PyRun_String(char *, int, PyObject *, PyObject *);
DL_IMPORT(PyObject *) PyRun_File(FILE *, char *, int, PyObject *, PyObject *);
diff --git a/Lib/doctest.py b/Lib/doctest.py
index b59a68a..fc826df 100644
--- a/Lib/doctest.py
+++ b/Lib/doctest.py
@@ -348,6 +348,8 @@
# 0,9,7 9-Feb-2001
# string method conversion
+from __future__ import generators
+
__version__ = 0, 9, 7
import types
diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index b68fbd7..6017e5f 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -13,13 +13,19 @@
/* Forward */
-static node *parsetok(struct tok_state *, grammar *, int, perrdetail *);
+static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
/* Parse input coming from a string. Return error code, print some errors. */
-
node *
PyParser_ParseString(char *s, grammar *g, int start, perrdetail *err_ret)
{
+ return PyParser_ParseStringFlags(s, g, start, err_ret, 0);
+}
+
+node *
+PyParser_ParseStringFlags(char *s, grammar *g, int start,
+ perrdetail *err_ret, int flags)
+{
struct tok_state *tok;
err_ret->error = E_OK;
@@ -42,7 +48,7 @@
tok->alterror++;
}
- return parsetok(tok, g, start, err_ret);
+ return parsetok(tok, g, start, err_ret, flags);
}
@@ -52,6 +58,14 @@
PyParser_ParseFile(FILE *fp, char *filename, grammar *g, int start,
char *ps1, char *ps2, perrdetail *err_ret)
{
+ return PyParser_ParseFileFlags(fp, filename, g, start, ps1, ps2,
+ err_ret, 0);
+}
+
+node *
+PyParser_ParseFileFlags(FILE *fp, char *filename, grammar *g, int start,
+ char *ps1, char *ps2, perrdetail *err_ret, int flags)
+{
struct tok_state *tok;
err_ret->error = E_OK;
@@ -72,14 +86,15 @@
}
- return parsetok(tok, g, start, err_ret);
+ return parsetok(tok, g, start, err_ret, flags);
}
/* Parse input coming from the given tokenizer structure.
Return error code. */
static node *
-parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret)
+parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
+ int flags)
{
parser_state *ps;
node *n;
@@ -90,6 +105,8 @@
err_ret->error = E_NOMEM;
return NULL;
}
+ if (flags & PyPARSE_YIELD_IS_KEYWORD)
+ ps->p_generators = 1;
for (;;) {
char *a, *b;
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 0412b7e..1d20deb 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -533,6 +533,7 @@
node *n;
perrdetail err;
char *ps1 = "", *ps2 = "";
+
v = PySys_GetObject("ps1");
if (v != NULL) {
v = PyObject_Str(v);
@@ -549,8 +550,11 @@
else if (PyString_Check(w))
ps2 = PyString_AsString(w);
}
- n = PyParser_ParseFile(fp, filename, &_PyParser_Grammar,
- Py_single_input, ps1, ps2, &err);
+ n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar,
+ Py_single_input, ps1, ps2, &err,
+ (flags &&
+ flags->cf_flags & PyCF_GENERATORS) ?
+ PyPARSE_YIELD_IS_KEYWORD : 0);
Py_XDECREF(v);
Py_XDECREF(w);
if (n == NULL) {
@@ -1017,7 +1021,9 @@
PyRun_FileExFlags(FILE *fp, char *filename, int start, PyObject *globals,
PyObject *locals, int closeit, PyCompilerFlags *flags)
{
- node *n = PyParser_SimpleParseFile(fp, filename, start);
+ node *n = PyParser_SimpleParseFileFlags(fp, filename, start,
+ (flags && flags->cf_flags & PyCF_GENERATORS) ?
+ PyPARSE_YIELD_IS_KEYWORD : 0);
if (closeit)
fclose(fp);
return run_err_node(n, filename, globals, locals, flags);
@@ -1101,7 +1107,9 @@
{
node *n;
PyCodeObject *co;
- n = PyParser_SimpleParseString(str, start);
+ n = PyParser_SimpleParseStringFlags(str, start,
+ (flags && flags->cf_flags & PyCF_GENERATORS) ?
+ PyPARSE_YIELD_IS_KEYWORD : 0);
if (n == NULL)
return NULL;
co = PyNode_CompileFlags(n, filename, flags);
@@ -1125,28 +1133,41 @@
/* Simplified interface to parsefile -- return node or set exception */
node *
-PyParser_SimpleParseFile(FILE *fp, char *filename, int start)
+PyParser_SimpleParseFileFlags(FILE *fp, char *filename, int start, int flags)
{
node *n;
perrdetail err;
- n = PyParser_ParseFile(fp, filename, &_PyParser_Grammar, start,
- (char *)0, (char *)0, &err);
+ n = PyParser_ParseFileFlags(fp, filename, &_PyParser_Grammar, start,
+ (char *)0, (char *)0, &err, flags);
if (n == NULL)
err_input(&err);
return n;
}
+node *
+PyParser_SimpleParseFile(FILE *fp, char *filename, int start)
+{
+ return PyParser_SimpleParseFileFlags(fp, filename, start, 0);
+}
+
/* Simplified interface to parsestring -- return node or set exception */
node *
+PyParser_SimpleParseStringFlags(char *str, int start, int flags)
+{
+ node *n;
+ perrdetail err;
+ n = PyParser_ParseStringFlags(str, &_PyParser_Grammar, start, &err,
+ flags);
+ if (n == NULL)
+ err_input(&err);
+ return n;
+}
+
+node *
PyParser_SimpleParseString(char *str, int start)
{
- node *n;
- perrdetail err;
- n = PyParser_ParseString(str, &_PyParser_Grammar, start, &err);
- if (n == NULL)
- err_input(&err);
- return n;
+ return PyParser_SimpleParseStringFlags(str, start, 0);
}
/* Set the error appropriate to the given input error code (see errcode.h) */