Patch #2477: Added from __future__ import unicode_literals
The new PyParser_*Ex() functions are based on Neal's suggestion and initial patch. The new __future__ feature makes all '' and r'' unicode strings. b'' and br'' stay (byte) strings.
diff --git a/Parser/parser.c b/Parser/parser.c
index 61da37d..8d52153 100644
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -202,14 +202,18 @@
for (i = 0; i < NCH(ch); i += 2) {
cch = CHILD(ch, i);
- if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME &&
- strcmp(STR(CHILD(cch, 0)), "with_statement") == 0) {
- ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
- break;
- } else if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME &&
- strcmp(STR(CHILD(cch, 0)), "print_function") == 0) {
- ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
- break;
+ if (NCH(cch) >= 1 && TYPE(CHILD(cch, 0)) == NAME) {
+ char *str_ch = STR(CHILD(cch, 0));
+ if (strcmp(str_ch, FUTURE_WITH_STATEMENT) == 0) {
+ ps->p_flags |= CO_FUTURE_WITH_STATEMENT;
+ break;
+ } else if (strcmp(str_ch, FUTURE_PRINT_FUNCTION) == 0) {
+ ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
+ break;
+ } else if (strcmp(str_ch, FUTURE_UNICODE_LITERALS) == 0) {
+ ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
+ break;
+ }
}
}
}
diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index e4db574..d8c8f62 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -14,7 +14,7 @@
/* Forward */
-static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int);
+static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
static void initerr(perrdetail *err_ret, const char* filename);
/* Parse input coming from a string. Return error code, print some errors. */
@@ -37,6 +37,16 @@
grammar *g, int start,
perrdetail *err_ret, int flags)
{
+ int iflags = flags;
+ return PyParser_ParseStringFlagsFilenameEx(s, filename, g, start,
+ err_ret, &iflags);
+}
+
+node *
+PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
+ grammar *g, int start,
+ perrdetail *err_ret, int *flags)
+{
struct tok_state *tok;
initerr(err_ret, filename);
@@ -70,6 +80,14 @@
PyParser_ParseFileFlags(FILE *fp, const char *filename, grammar *g, int start,
char *ps1, char *ps2, perrdetail *err_ret, int flags)
{
+ int iflags = flags;
+ return PyParser_ParseFileFlagsEx(fp, filename, g, start, ps1, ps2, err_ret, &iflags);
+}
+
+node *
+PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, grammar *g, int start,
+ char *ps1, char *ps2, perrdetail *err_ret, int *flags)
+{
struct tok_state *tok;
initerr(err_ret, filename);
@@ -85,7 +103,6 @@
tok->alterror++;
}
-
return parsetok(tok, g, start, err_ret, flags);
}
@@ -110,7 +127,7 @@
static node *
parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
- int flags)
+ int *flags)
{
parser_state *ps;
node *n;
@@ -123,8 +140,13 @@
return NULL;
}
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
- if (flags & PyPARSE_PRINT_IS_FUNCTION)
+ if (*flags & PyPARSE_PRINT_IS_FUNCTION) {
ps->p_flags |= CO_FUTURE_PRINT_FUNCTION;
+ }
+ if (*flags & PyPARSE_UNICODE_LITERALS) {
+ ps->p_flags |= CO_FUTURE_UNICODE_LITERALS;
+ }
+
#endif
for (;;) {
@@ -147,7 +169,7 @@
except if a certain flag is given --
codeop.py uses this. */
if (tok->indent &&
- !(flags & PyPARSE_DONT_IMPLY_DEDENT))
+ !(*flags & PyPARSE_DONT_IMPLY_DEDENT))
{
tok->pendin = -tok->indent;
tok->indent = 0;
@@ -191,6 +213,7 @@
else
n = NULL;
+ *flags = ps->p_flags;
PyParser_Delete(ps);
if (n == NULL) {