Issue #9738: Document encodings of AST, compiler, parser and PyRun functions
diff --git a/Include/ast.h b/Include/ast.h
index cc14b7f..a015336 100644
--- a/Include/ast.h
+++ b/Include/ast.h
@@ -4,8 +4,11 @@
extern "C" {
#endif
-PyAPI_FUNC(mod_ty) PyAST_FromNode(const node *, PyCompilerFlags *flags,
- const char *, PyArena *);
+PyAPI_FUNC(mod_ty) PyAST_FromNode(
+ const node *n,
+ PyCompilerFlags *flags,
+ const char *filename, /* decoded from the filesystem encoding */
+ PyArena *arena);
#ifdef __cplusplus
}
diff --git a/Include/compile.h b/Include/compile.h
index 456a494..5ce5b77 100644
--- a/Include/compile.h
+++ b/Include/compile.h
@@ -30,8 +30,12 @@
struct _mod; /* Declare the existence of this type */
#define PyAST_Compile(mod, s, f, ar) PyAST_CompileEx(mod, s, f, -1, ar)
-PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(struct _mod *, const char *,
- PyCompilerFlags *, int, PyArena *);
+PyAPI_FUNC(PyCodeObject *) PyAST_CompileEx(
+ mod_ty mod,
+ const char *filename, /* decoded from the filesystem encoding */
+ PyCompilerFlags *flags,
+ int optimize,
+ PyArena *arena);
PyAPI_FUNC(PyFutureFeatures *) PyFuture_FromAST(struct _mod *, const char *);
diff --git a/Include/parsetok.h b/Include/parsetok.h
index d183784..4b7694f 100644
--- a/Include/parsetok.h
+++ b/Include/parsetok.h
@@ -9,10 +9,10 @@
typedef struct {
int error;
- const char *filename;
+ const char *filename; /* decoded from the filesystem encoding */
int lineno;
int offset;
- char *text;
+ char *text; /* UTF-8-encoded string */
int token;
int expected;
} perrdetail;
@@ -39,23 +39,32 @@
PyAPI_FUNC(node *) PyParser_ParseStringFlags(const char *, grammar *, int,
perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
+PyAPI_FUNC(node *) PyParser_ParseFileFlags(FILE *, const char *,
const char*, grammar *,
int, char *, char *,
perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(FILE *, const char *,
- const char*, grammar *,
- int, char *, char *,
- perrdetail *, int *);
+PyAPI_FUNC(node *) PyParser_ParseFileFlagsEx(
+ FILE *fp,
+ const char *filename, /* decoded from the filesystem encoding */
+ const char *enc,
+ grammar *g,
+ int start,
+ char *ps1,
+ char *ps2,
+ perrdetail *err_ret,
+ int *flags);
PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilename(const char *,
const char *,
grammar *, int,
perrdetail *, int);
-PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(const char *,
- const char *,
- grammar *, int,
- perrdetail *, int *);
+PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx(
+ const char *s,
+ const char *filename, /* decoded from the filesystem encoding */
+ grammar *g,
+ int start,
+ perrdetail *err_ret,
+ int *flags);
/* Note that he following function is defined in pythonrun.c not parsetok.c. */
PyAPI_FUNC(void) PyParser_SetError(perrdetail *);
diff --git a/Include/pythonrun.h b/Include/pythonrun.h
index f290687..86da6e0 100644
--- a/Include/pythonrun.h
+++ b/Include/pythonrun.h
@@ -38,19 +38,41 @@
#ifndef Py_LIMITED_API
PyAPI_FUNC(int) PyRun_SimpleStringFlags(const char *, PyCompilerFlags *);
PyAPI_FUNC(int) PyRun_AnyFileFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_AnyFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_SimpleFileExFlags(FILE *, const char *, int, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveOneFlags(FILE *, const char *, PyCompilerFlags *);
-PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(FILE *, const char *, PyCompilerFlags *);
+PyAPI_FUNC(int) PyRun_AnyFileExFlags(
+ FILE *fp,
+ const char *filename, /* decoded from the filesystem encoding */
+ int closeit,
+ PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_SimpleFileExFlags(
+ FILE *fp,
+ const char *filename, /* decoded from the filesystem encoding */
+ int closeit,
+ PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveOneFlags(
+ FILE *fp,
+ const char *filename, /* decoded from the filesystem encoding */
+ PyCompilerFlags *flags);
+PyAPI_FUNC(int) PyRun_InteractiveLoopFlags(
+ FILE *fp,
+ const char *filename, /* decoded from the filesystem encoding */
+ PyCompilerFlags *flags);
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(const char *, const char *,
- int, PyCompilerFlags *flags,
- PyArena *);
-PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(FILE *, const char *,
- const char*, int,
- char *, char *,
- PyCompilerFlags *, int *,
- PyArena *);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromString(
+ const char *s,
+ const char *filename, /* decoded from the filesystem encoding */
+ int start,
+ PyCompilerFlags *flags,
+ PyArena *arena);
+PyAPI_FUNC(struct _mod *) PyParser_ASTFromFile(
+ FILE *fp,
+ const char *filename, /* decoded from the filesystem encoding */
+ const char* enc,
+ int start,
+ char *ps1,
+ char *ps2,
+ PyCompilerFlags *flags,
+ int *errcode,
+ PyArena *arena);
#endif
#ifndef PyParser_SimpleParseString
@@ -68,9 +90,14 @@
PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *,
PyObject *, PyCompilerFlags *);
-PyAPI_FUNC(PyObject *) PyRun_FileExFlags(FILE *, const char *, int,
- PyObject *, PyObject *, int,
- PyCompilerFlags *);
+PyAPI_FUNC(PyObject *) PyRun_FileExFlags(
+ FILE *fp,
+ const char *filename, /* decoded from the filesystem encoding */
+ int start,
+ PyObject *globals,
+ PyObject *locals,
+ int closeit,
+ PyCompilerFlags *flags);
#endif
#ifdef Py_LIMITED_API
@@ -78,10 +105,17 @@
#else
#define Py_CompileString(str, p, s) Py_CompileStringExFlags(str, p, s, NULL, -1)
#define Py_CompileStringFlags(str, p, s, f) Py_CompileStringExFlags(str, p, s, f, -1)
-PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(const char *, const char *, int,
- PyCompilerFlags *, int);
+PyAPI_FUNC(PyObject *) Py_CompileStringExFlags(
+ const char *str,
+ const char *filename, /* decoded from the filesystem encoding */
+ int start,
+ PyCompilerFlags *flags,
+ int optimize);
#endif
-PyAPI_FUNC(struct symtable *) Py_SymtableString(const char *, const char *, int);
+PyAPI_FUNC(struct symtable *) Py_SymtableString(
+ const char *str,
+ const char *filename, /* decoded from the filesystem encoding */
+ int start);
PyAPI_FUNC(void) PyErr_Print(void);
PyAPI_FUNC(void) PyErr_PrintEx(int);
diff --git a/Include/symtable.h b/Include/symtable.h
index 2ad204d..fd7de04 100644
--- a/Include/symtable.h
+++ b/Include/symtable.h
@@ -16,7 +16,8 @@
struct _symtable_entry;
struct symtable {
- const char *st_filename; /* name of file being compiled */
+ const char *st_filename; /* name of file being compiled,
+ decoded from the filesystem encoding */
struct _symtable_entry *st_cur; /* current symbol table entry */
struct _symtable_entry *st_top; /* symbol table entry for module */
PyObject *st_blocks; /* dict: map AST node addresses
@@ -60,8 +61,10 @@
PyAPI_FUNC(int) PyST_GetScope(PySTEntryObject *, PyObject *);
-PyAPI_FUNC(struct symtable *) PySymtable_Build(mod_ty, const char *,
- PyFutureFeatures *);
+PyAPI_FUNC(struct symtable *) PySymtable_Build(
+ mod_ty mod,
+ const char *filename, /* decoded from the filesystem encoding */
+ PyFutureFeatures *future);
PyAPI_FUNC(PySTEntryObject *) PySymtable_Lookup(struct symtable *, void *);
PyAPI_FUNC(void) PySymtable_Free(struct symtable *);