Merge tag 'v3.10.0a2' into master
Python 3.10.0a2
diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h
index 6d84e37..cba3bbd 100644
--- a/Include/internal/pycore_pylifecycle.h
+++ b/Include/internal/pycore_pylifecycle.h
@@ -84,7 +84,7 @@ extern void _PyFaulthandler_Fini(void);
extern void _PyHash_Fini(void);
extern void _PyTraceMalloc_Fini(void);
extern void _PyWarnings_Fini(PyInterpreterState *interp);
-extern void _PyAST_Fini(PyThreadState *tstate);
+extern void _PyAST_Fini(PyInterpreterState *interp);
extern PyStatus _PyGILState_Init(PyThreadState *tstate);
extern void _PyGILState_Fini(PyThreadState *tstate);
diff --git a/Lib/plistlib.py b/Lib/plistlib.py
index 83b214e..2eeebe4 100644
--- a/Lib/plistlib.py
+++ b/Lib/plistlib.py
@@ -611,7 +611,7 @@ def _count_to_size(count):
elif count < 1 << 16:
return 2
- elif count << 1 << 32:
+ elif count < 1 << 32:
return 4
else:
diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-02-14-39-48.bpo-40077.grY9TG.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-02-14-39-48.bpo-40077.grY9TG.rst
new file mode 100644
index 0000000..48565a5
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-02-14-39-48.bpo-40077.grY9TG.rst
@@ -0,0 +1 @@
+Convert :mod:`mmap` to use heap types.
diff --git a/Misc/NEWS.d/next/Library/2020-11-03-09-22-56.bpo-42249.vfNO2u.rst b/Misc/NEWS.d/next/Library/2020-11-03-09-22-56.bpo-42249.vfNO2u.rst
new file mode 100644
index 0000000..071a0fd
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2020-11-03-09-22-56.bpo-42249.vfNO2u.rst
@@ -0,0 +1 @@
+Fixed writing binary Plist files larger than 4 GiB.
diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c
index 463bd40..1e96849 100644
--- a/Modules/mmapmodule.c
+++ b/Modules/mmapmodule.c
@@ -20,6 +20,7 @@
#define PY_SSIZE_T_CLEAN
#include <Python.h>
+#include "structmember.h" // PyMemberDef
#include <stddef.h> // offsetof()
#ifndef MS_WINDOWS
@@ -113,10 +114,23 @@ typedef struct {
access_mode access;
} mmap_object;
+typedef struct {
+ PyTypeObject *mmap_object_type;
+} mmap_state;
+
+static mmap_state *
+get_mmap_state(PyObject *module)
+{
+ mmap_state *state = PyModule_GetState(module);
+ assert(state);
+ return state;
+}
static void
mmap_object_dealloc(mmap_object *m_obj)
{
+ PyTypeObject *tp = Py_TYPE(m_obj);
+
#ifdef MS_WINDOWS
Py_BEGIN_ALLOW_THREADS
if (m_obj->data != NULL)
@@ -142,7 +156,9 @@ mmap_object_dealloc(mmap_object *m_obj)
if (m_obj->weakreflist != NULL)
PyObject_ClearWeakRefs((PyObject *) m_obj);
- Py_TYPE(m_obj)->tp_free((PyObject*)m_obj);
+
+ tp->tp_free(m_obj);
+ Py_DECREF(tp);
}
static PyObject *
@@ -793,6 +809,11 @@ mmap_madvise_method(mmap_object *self, PyObject *args)
}
#endif // HAVE_MADVISE
+static struct PyMemberDef mmap_object_members[] = {
+ {"__weaklistoffset__", T_PYSSIZET, offsetof(mmap_object, weakreflist), READONLY},
+ {NULL},
+};
+
static struct PyMethodDef mmap_object_methods[] = {
{"close", (PyCFunction) mmap_close_method, METH_NOARGS},
{"find", (PyCFunction) mmap_find_method, METH_VARARGS},
@@ -1035,27 +1056,6 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value)
}
}
-static PySequenceMethods mmap_as_sequence = {
- (lenfunc)mmap_length, /*sq_length*/
- 0, /*sq_concat*/
- 0, /*sq_repeat*/
- (ssizeargfunc)mmap_item, /*sq_item*/
- 0, /*sq_slice*/
- (ssizeobjargproc)mmap_ass_item, /*sq_ass_item*/
- 0, /*sq_ass_slice*/
-};
-
-static PyMappingMethods mmap_as_mapping = {
- (lenfunc)mmap_length,
- (binaryfunc)mmap_subscript,
- (objobjargproc)mmap_ass_subscript,
-};
-
-static PyBufferProcs mmap_as_buffer = {
- (getbufferproc)mmap_buffer_getbuf,
- (releasebufferproc)mmap_buffer_releasebuf,
-};
-
static PyObject *
new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict);
@@ -1083,47 +1083,39 @@ The default value is MAP_SHARED.\n\
To map anonymous memory, pass -1 as the fileno (both versions).");
-static PyTypeObject mmap_object_type = {
- PyVarObject_HEAD_INIT(NULL, 0)
- "mmap.mmap", /* tp_name */
- sizeof(mmap_object), /* tp_basicsize */
- 0, /* tp_itemsize */
- /* methods */
- (destructor)mmap_object_dealloc, /* tp_dealloc */
- 0, /* tp_vectorcall_offset */
- 0, /* tp_getattr */
- 0, /* tp_setattr */
- 0, /* tp_as_async */
- (reprfunc)mmap__repr__method, /* tp_repr */
- 0, /* tp_as_number */
- &mmap_as_sequence, /* tp_as_sequence */
- &mmap_as_mapping, /* tp_as_mapping */
- 0, /* tp_hash */
- 0, /* tp_call */
- 0, /* tp_str */
- PyObject_GenericGetAttr, /* tp_getattro */
- 0, /* tp_setattro */
- &mmap_as_buffer, /* tp_as_buffer */
- Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
- mmap_doc, /* tp_doc */
- 0, /* tp_traverse */
- 0, /* tp_clear */
- 0, /* tp_richcompare */
- offsetof(mmap_object, weakreflist), /* tp_weaklistoffset */
- 0, /* tp_iter */
- 0, /* tp_iternext */
- mmap_object_methods, /* tp_methods */
- 0, /* tp_members */
- mmap_object_getset, /* tp_getset */
- 0, /* tp_base */
- 0, /* tp_dict */
- 0, /* tp_descr_get */
- 0, /* tp_descr_set */
- 0, /* tp_dictoffset */
- 0, /* tp_init */
- PyType_GenericAlloc, /* tp_alloc */
- new_mmap_object, /* tp_new */
- PyObject_Del, /* tp_free */
+static PyType_Slot mmap_object_slots[] = {
+ {Py_tp_new, new_mmap_object},
+ {Py_tp_alloc, PyType_GenericAlloc},
+ {Py_tp_dealloc, mmap_object_dealloc},
+ {Py_tp_free, PyObject_Del},
+ {Py_tp_repr, mmap__repr__method},
+ {Py_tp_doc, (void *)mmap_doc},
+ {Py_tp_methods, mmap_object_methods},
+ {Py_tp_members, mmap_object_members},
+ {Py_tp_getset, mmap_object_getset},
+ {Py_tp_getattro, PyObject_GenericGetAttr},
+
+ /* as sequence */
+ {Py_sq_length, mmap_length},
+ {Py_sq_item, mmap_item},
+ {Py_sq_ass_item, mmap_ass_item},
+
+ /* as mapping */
+ {Py_mp_length, mmap_length},
+ {Py_mp_subscript, mmap_subscript},
+ {Py_mp_ass_subscript, mmap_ass_subscript},
+
+ /* as buffer */
+ {Py_bf_getbuffer, mmap_buffer_getbuf},
+ {Py_bf_releasebuffer, mmap_buffer_releasebuf},
+ {0, NULL},
+};
+
+static PyType_Spec mmap_object_spec = {
+ .name = "mmap.mmap",
+ .basicsize = sizeof(mmap_object),
+ .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+ .slots = mmap_object_slots,
};
@@ -1510,18 +1502,45 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
#endif /* MS_WINDOWS */
static int
+mmap_traverse(PyObject *module, visitproc visit, void *arg)
+{
+ mmap_state *state = get_mmap_state(module);
+ Py_VISIT(state->mmap_object_type);
+ return 0;
+}
+
+static int
+mmap_clear(PyObject *module)
+{
+ mmap_state *state = get_mmap_state(module);
+ Py_CLEAR(state->mmap_object_type);
+ return 0;
+}
+
+static void
+mmap_free(void *module)
+{
+ mmap_clear((PyObject *)module);
+}
+
+static int
mmap_exec(PyObject *module)
{
- if (PyType_Ready(&mmap_object_type) < 0) {
- return -1;
- }
+ mmap_state *state = get_mmap_state(module);
Py_INCREF(PyExc_OSError);
if (PyModule_AddObject(module, "error", PyExc_OSError) < 0) {
Py_DECREF(PyExc_OSError);
return -1;
}
- if (PyModule_AddType(module, &mmap_object_type) < 0) {
+
+ state->mmap_object_type = (PyTypeObject *)PyType_FromModuleAndSpec(module,
+ &mmap_object_spec,
+ NULL);
+ if (state->mmap_object_type == NULL) {
+ return -1;
+ }
+ if (PyModule_AddType(module, state->mmap_object_type) < 0) {
return -1;
}
@@ -1660,8 +1679,11 @@ static PyModuleDef_Slot mmap_slots[] = {
static struct PyModuleDef mmapmodule = {
PyModuleDef_HEAD_INIT,
.m_name = "mmap",
- .m_size = 0,
+ .m_size = sizeof(mmap_state),
.m_slots = mmap_slots,
+ .m_traverse = mmap_traverse,
+ .m_clear = mmap_clear,
+ .m_free = mmap_free,
};
PyMODINIT_FUNC
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 9a833e8..9fec7ae 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -1015,18 +1015,35 @@ def visitModule(self, mod):
""", 0, reflow=False)
- self.emit("static int init_types(struct ast_state *state)",0)
- self.emit("{", 0)
- self.emit("if (state->initialized) return 1;", 1)
- self.emit("if (init_identifiers(state) < 0) return 0;", 1)
- self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1)
- self.emit("if (!state->AST_type) return 0;", 1)
- self.emit("if (add_ast_fields(state) < 0) return 0;", 1)
+ self.file.write(textwrap.dedent('''
+ static int
+ init_types(struct ast_state *state)
+ {
+ // init_types() must not be called after _PyAST_Fini()
+ // has been called
+ assert(state->initialized >= 0);
+
+ if (state->initialized) {
+ return 1;
+ }
+ if (init_identifiers(state) < 0) {
+ return 0;
+ }
+ state->AST_type = PyType_FromSpec(&AST_type_spec);
+ if (!state->AST_type) {
+ return 0;
+ }
+ if (add_ast_fields(state) < 0) {
+ return 0;
+ }
+ '''))
for dfn in mod.dfns:
self.visit(dfn)
- self.emit("state->initialized = 1;", 1)
- self.emit("return 1;", 1);
- self.emit("}", 0)
+ self.file.write(textwrap.dedent('''
+ state->initialized = 1;
+ return 1;
+ }
+ '''))
def visitProduct(self, prod, name):
if prod.fields:
@@ -1353,23 +1370,27 @@ def generate_ast_state(module_state, f):
def generate_ast_fini(module_state, f):
- f.write("""
-void _PyAST_Fini(PyThreadState *tstate)
-{
-#ifdef Py_BUILD_CORE
- struct ast_state *state = &tstate->interp->ast;
-#else
- struct ast_state *state = &global_ast_state;
-#endif
+ f.write(textwrap.dedent("""
+ void _PyAST_Fini(PyInterpreterState *interp)
+ {
+ #ifdef Py_BUILD_CORE
+ struct ast_state *state = &interp->ast;
+ #else
+ struct ast_state *state = &global_ast_state;
+ #endif
-""")
+ """))
for s in module_state:
f.write(" Py_CLEAR(state->" + s + ');\n')
- f.write("""
- state->initialized = 0;
-}
+ f.write(textwrap.dedent("""
+ #if defined(Py_BUILD_CORE) && !defined(NDEBUG)
+ state->initialized = -1;
+ #else
+ state->initialized = 0;
+ #endif
+ }
-""")
+ """))
def generate_module_def(mod, f, internal_h):
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
index f04addb..a456b51 100644
--- a/Python/Python-ast.c
+++ b/Python/Python-ast.c
@@ -261,10 +261,10 @@ get_ast_state(void)
#include "Python-ast.h"
#include "structmember.h"
-void _PyAST_Fini(PyThreadState *tstate)
+void _PyAST_Fini(PyInterpreterState *interp)
{
#ifdef Py_BUILD_CORE
- struct ast_state *state = &tstate->interp->ast;
+ struct ast_state *state = &interp->ast;
#else
struct ast_state *state = &global_ast_state;
#endif
@@ -483,7 +483,11 @@ void _PyAST_Fini(PyThreadState *tstate)
Py_CLEAR(state->vararg);
Py_CLEAR(state->withitem_type);
+#if defined(Py_BUILD_CORE) && !defined(NDEBUG)
+ state->initialized = -1;
+#else
state->initialized = 0;
+#endif
}
static int init_identifiers(struct ast_state *state)
@@ -1227,13 +1231,27 @@ static int add_ast_fields(struct ast_state *state)
}
-static int init_types(struct ast_state *state)
+
+static int
+init_types(struct ast_state *state)
{
- if (state->initialized) return 1;
- if (init_identifiers(state) < 0) return 0;
+ // init_types() must not be called after _PyAST_Fini()
+ // has been called
+ assert(state->initialized >= 0);
+
+ if (state->initialized) {
+ return 1;
+ }
+ if (init_identifiers(state) < 0) {
+ return 0;
+ }
state->AST_type = PyType_FromSpec(&AST_type_spec);
- if (!state->AST_type) return 0;
- if (add_ast_fields(state) < 0) return 0;
+ if (!state->AST_type) {
+ return 0;
+ }
+ if (add_ast_fields(state) < 0) {
+ return 0;
+ }
state->mod_type = make_type(state, "mod", state->AST_type, NULL, 0,
"mod = Module(stmt* body, type_ignore* type_ignores)\n"
" | Interactive(stmt* body)\n"
@@ -1902,6 +1920,7 @@ static int init_types(struct ast_state *state)
TypeIgnore_fields, 2,
"TypeIgnore(int lineno, string tag)");
if (!state->TypeIgnore_type) return 0;
+
state->initialized = 1;
return 1;
}
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
index ff58c1b..cad0fa7 100644
--- a/Python/pylifecycle.c
+++ b/Python/pylifecycle.c
@@ -1545,12 +1545,6 @@ flush_std_files(void)
static void
finalize_interp_types(PyThreadState *tstate)
{
- // The _ast module state is shared by all interpreters.
- // The state must only be cleared by the main interpreter.
- if (_Py_IsMainInterpreter(tstate)) {
- _PyAST_Fini(tstate);
- }
-
_PyExc_Fini(tstate);
_PyFrame_Fini(tstate);
_PyAsyncGen_Fini(tstate);
@@ -1591,8 +1585,6 @@ finalize_interp_clear(PyThreadState *tstate)
_Py_ClearFileSystemEncoding();
}
- _PyWarnings_Fini(tstate->interp);
-
finalize_interp_types(tstate);
}
diff --git a/Python/pystate.c b/Python/pystate.c
index e37cbd5..c9882a7 100644
--- a/Python/pystate.c
+++ b/Python/pystate.c
@@ -300,13 +300,16 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate)
Py_CLEAR(interp->after_forkers_parent);
Py_CLEAR(interp->after_forkers_child);
#endif
- if (_PyRuntimeState_GetFinalizing(runtime) == NULL) {
- _PyWarnings_Fini(interp);
- }
+
+ _PyAST_Fini(interp);
+ _PyWarnings_Fini(interp);
+
+ // All Python types must be destroyed before the last GC collection. Python
+ // types create a reference cycle to themselves in their in their
+ // PyTypeObject.tp_mro member (the tuple contains the type).
/* Last garbage collection on this interpreter */
_PyGC_CollectNoFail(tstate);
-
_PyGC_Fini(tstate);
/* We don't clear sysdict and builtins until the end of this function.