Branch merge
diff --git a/Doc/library/io.rst b/Doc/library/io.rst
index a76b757..76c856e 100644
--- a/Doc/library/io.rst
+++ b/Doc/library/io.rst
@@ -607,25 +607,6 @@
if the buffer needs to be written out but the raw stream blocks.
-.. class:: BufferedRWPair(reader, writer, buffer_size=DEFAULT_BUFFER_SIZE)
-
- A buffered I/O object giving a combined, higher-level access to two
- sequential :class:`RawIOBase` objects: one readable, the other writeable.
- It is useful for pairs of unidirectional communication channels
- (pipes, for instance). It inherits :class:`BufferedIOBase`.
-
- *reader* and *writer* are :class:`RawIOBase` objects that are readable and
- writeable respectively. If the *buffer_size* is omitted it defaults to
- :data:`DEFAULT_BUFFER_SIZE`.
-
- A fourth argument, *max_buffer_size*, is supported, but unused and
- deprecated.
-
- :class:`BufferedRWPair` implements all of :class:`BufferedIOBase`\'s methods
- except for :meth:`~BufferedIOBase.detach`, which raises
- :exc:`UnsupportedOperation`.
-
-
.. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE)
A buffered interface to random access streams. It inherits
@@ -642,6 +623,29 @@
:class:`BufferedWriter` can do.
+.. class:: BufferedRWPair(reader, writer, buffer_size=DEFAULT_BUFFER_SIZE)
+
+ A buffered I/O object combining two unidirectional :class:`RawIOBase`
+ objects -- one readable, the other writeable -- into a single bidirectional
+ endpoint. It inherits :class:`BufferedIOBase`.
+
+ *reader* and *writer* are :class:`RawIOBase` objects that are readable and
+ writeable respectively. If the *buffer_size* is omitted it defaults to
+ :data:`DEFAULT_BUFFER_SIZE`.
+
+ A fourth argument, *max_buffer_size*, is supported, but unused and
+ deprecated.
+
+ :class:`BufferedRWPair` implements all of :class:`BufferedIOBase`\'s methods
+ except for :meth:`~BufferedIOBase.detach`, which raises
+ :exc:`UnsupportedOperation`.
+
+ .. warning::
+ :class:`BufferedRWPair` does not attempt to synchronize accesses to
+ its underlying raw streams. You should not pass it the same object
+ as reader and writer; use :class:`BufferedRandom` instead.
+
+
Text I/O
^^^^^^^^
diff --git a/Doc/library/re.rst b/Doc/library/re.rst
index 606825c..412e6ee 100644
--- a/Doc/library/re.rst
+++ b/Doc/library/re.rst
@@ -635,7 +635,7 @@
of *pattern* in *string* by the replacement *repl*. If the pattern isn't found,
*string* is returned unchanged. *repl* can be a string or a function; if it is
a string, any backslash escapes in it are processed. That is, ``\n`` is
- converted to a single newline character, ``\r`` is converted to a linefeed, and
+ converted to a single newline character, ``\r`` is converted to a carriage return, and
so forth. Unknown escapes such as ``\j`` are left alone. Backreferences, such
as ``\6``, are replaced with the substring matched by group 6 in the pattern.
For example:
diff --git a/Doc/library/time.rst b/Doc/library/time.rst
index 24461b3..d95d83c 100644
--- a/Doc/library/time.rst
+++ b/Doc/library/time.rst
@@ -509,6 +509,6 @@
preferred hour/minute offset is not supported by all ANSI C libraries. Also, a
strict reading of the original 1982 :rfc:`822` standard calls for a two-digit
year (%y rather than %Y), but practice moved to 4-digit years long before the
- year 2000. The 4-digit year has been mandated by :rfc:`2822`, which obsoletes
- :rfc:`822`.
+ year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has
+ been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`.
diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst
index bb058df..e72770a 100644
--- a/Doc/library/xmlrpc.client.rst
+++ b/Doc/library/xmlrpc.client.rst
@@ -402,8 +402,8 @@
MultiCall Objects
-----------------
-In http://www.xmlrpc.com/discuss/msgReader%241208, an approach is presented to
-encapsulate multiple calls to a remote server into a single request.
+The :class:`MultiCall` object provides a way to encapsulate multiple calls to a
+remote server into a single request [#]_.
.. class:: MultiCall(server)
@@ -534,3 +534,10 @@
See :ref:`simplexmlrpcserver-example`.
+.. rubric:: Footnotes
+
+.. [#] This approach has been first presented in `a discussion on xmlrpc.com
+ <http://web.archive.org/web/20060624230303/http://www.xmlrpc.com/discuss/msgReader$1208?mode=topic>`_.
+.. the link now points to webarchive since the one at
+.. http://www.xmlrpc.com/discuss/msgReader%241208 is broken (and webadmin
+.. doesn't reply)
diff --git a/Lib/test/json_tests/test_dump.py b/Lib/test/json_tests/test_dump.py
index 083c11f..f1c1b04 100644
--- a/Lib/test/json_tests/test_dump.py
+++ b/Lib/test/json_tests/test_dump.py
@@ -1,6 +1,7 @@
from io import StringIO
from test.json_tests import PyTest, CTest
+from test.support import precisionbigmemtest, _1G
class TestDump:
def test_dump(self):
@@ -21,4 +22,20 @@
class TestPyDump(TestDump, PyTest): pass
-class TestCDump(TestDump, CTest): pass
+
+class TestCDump(TestDump, CTest):
+
+ # The size requirement here is hopefully over-estimated (actual
+ # memory consumption depending on implementation details, and also
+ # system memory management, since this may allocate a lot of
+ # small objects).
+
+ @precisionbigmemtest(size=_1G, memuse=1)
+ def test_large_list(self, size):
+ N = int(30 * 1024 * 1024 * (size / _1G))
+ l = [1] * N
+ encoded = self.dumps(l)
+ self.assertEqual(len(encoded), N * 3)
+ self.assertEqual(encoded[:1], "[")
+ self.assertEqual(encoded[-2:], "1]")
+ self.assertEqual(encoded[1:-2], "1, " * (N - 1))
diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py
index f05d3c0..718d05c 100644
--- a/Lib/test/test_exceptions.py
+++ b/Lib/test/test_exceptions.py
@@ -608,6 +608,68 @@
gc_collect()
self.assertEqual(sys.exc_info(), (None, None, None))
+ def _check_generator_cleanup_exc_state(self, testfunc):
+ # Issue #12791: exception state is cleaned up as soon as a generator
+ # is closed (reference cycles are broken).
+ class MyException(Exception):
+ def __init__(self, obj):
+ self.obj = obj
+ class MyObj:
+ pass
+
+ def raising_gen():
+ try:
+ raise MyException(obj)
+ except MyException:
+ yield
+
+ obj = MyObj()
+ wr = weakref.ref(obj)
+ g = raising_gen()
+ next(g)
+ testfunc(g)
+ g = obj = None
+ obj = wr()
+ self.assertIs(obj, None)
+
+ def test_generator_throw_cleanup_exc_state(self):
+ def do_throw(g):
+ try:
+ g.throw(RuntimeError())
+ except RuntimeError:
+ pass
+ self._check_generator_cleanup_exc_state(do_throw)
+
+ def test_generator_close_cleanup_exc_state(self):
+ def do_close(g):
+ g.close()
+ self._check_generator_cleanup_exc_state(do_close)
+
+ def test_generator_del_cleanup_exc_state(self):
+ def do_del(g):
+ g = None
+ self._check_generator_cleanup_exc_state(do_del)
+
+ def test_generator_next_cleanup_exc_state(self):
+ def do_next(g):
+ try:
+ next(g)
+ except StopIteration:
+ pass
+ else:
+ self.fail("should have raised StopIteration")
+ self._check_generator_cleanup_exc_state(do_next)
+
+ def test_generator_send_cleanup_exc_state(self):
+ def do_send(g):
+ try:
+ g.send(None)
+ except StopIteration:
+ pass
+ else:
+ self.fail("should have raised StopIteration")
+ self._check_generator_cleanup_exc_state(do_send)
+
def test_3114(self):
# Bug #3114: in its destructor, MyObject retrieves a pointer to
# obsolete and/or deallocated objects.
diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py
index 0d68ca1..1129484 100644
--- a/Lib/test/test_io.py
+++ b/Lib/test/test_io.py
@@ -1401,15 +1401,18 @@
rw.seek(0, 0)
self.assertEqual(b"asdf", rw.read(4))
- rw.write(b"asdf")
+ rw.write(b"123f")
rw.seek(0, 0)
- self.assertEqual(b"asdfasdfl", rw.read())
+ self.assertEqual(b"asdf123fl", rw.read())
self.assertEqual(9, rw.tell())
rw.seek(-4, 2)
self.assertEqual(5, rw.tell())
rw.seek(2, 1)
self.assertEqual(7, rw.tell())
self.assertEqual(b"fl", rw.read(11))
+ rw.flush()
+ self.assertEqual(b"asdf123fl", raw.getvalue())
+
self.assertRaises(TypeError, rw.seek, 0.0)
def check_flush_and_read(self, read_func):
@@ -1554,6 +1557,43 @@
BufferedReaderTest.test_misbehaved_io(self)
BufferedWriterTest.test_misbehaved_io(self)
+ def test_interleaved_read_write(self):
+ # Test for issue #12213
+ with self.BytesIO(b'abcdefgh') as raw:
+ with self.tp(raw, 100) as f:
+ f.write(b"1")
+ self.assertEqual(f.read(1), b'b')
+ f.write(b'2')
+ self.assertEqual(f.read1(1), b'd')
+ f.write(b'3')
+ buf = bytearray(1)
+ f.readinto(buf)
+ self.assertEqual(buf, b'f')
+ f.write(b'4')
+ self.assertEqual(f.peek(1), b'h')
+ f.flush()
+ self.assertEqual(raw.getvalue(), b'1b2d3f4h')
+
+ with self.BytesIO(b'abc') as raw:
+ with self.tp(raw, 100) as f:
+ self.assertEqual(f.read(1), b'a')
+ f.write(b"2")
+ self.assertEqual(f.read(1), b'c')
+ f.flush()
+ self.assertEqual(raw.getvalue(), b'a2c')
+
+ def test_interleaved_readline_write(self):
+ with self.BytesIO(b'ab\ncdef\ng\n') as raw:
+ with self.tp(raw) as f:
+ f.write(b'1')
+ self.assertEqual(f.readline(), b'b\n')
+ f.write(b'2')
+ self.assertEqual(f.readline(), b'def\n')
+ f.write(b'3')
+ self.assertEqual(f.readline(), b'\n')
+ f.flush()
+ self.assertEqual(raw.getvalue(), b'1b\n2def\n3\n')
+
# You can't construct a BufferedRandom over a non-seekable stream.
test_unseekable = None
@@ -2804,11 +2844,11 @@
os.close(w)
os.close(r)
- def test_interrupterd_read_retry_buffered(self):
+ def test_interrupted_read_retry_buffered(self):
self.check_interrupted_read_retry(lambda x: x.decode('latin1'),
mode="rb")
- def test_interrupterd_read_retry_text(self):
+ def test_interrupted_read_retry_text(self):
self.check_interrupted_read_retry(lambda x: x,
mode="r")
@@ -2867,10 +2907,10 @@
if e.errno != errno.EBADF:
raise
- def test_interrupterd_write_retry_buffered(self):
+ def test_interrupted_write_retry_buffered(self):
self.check_interrupted_write_retry(b"x", mode="wb")
- def test_interrupterd_write_retry_text(self):
+ def test_interrupted_write_retry_text(self):
self.check_interrupted_write_retry("x", mode="w", encoding="latin1")
diff --git a/Misc/NEWS b/Misc/NEWS
index b96bb3d..29a2e6f 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -10,6 +10,9 @@
Core and Builtins
-----------------
+- Issue #12791: Break reference cycles early when a generator exits with
+ an exception.
+
- Issue #12773: Make __doc__ mutable on user-defined classes.
- Issue #12766: Raise an ValueError when creating a class with a class variable
@@ -262,6 +265,12 @@
Library
-------
+- Issue #12213: Fix a buffering bug with interleaved reads and writes that
+ could appear on BufferedRandom streams.
+
+- Issue #12778: Reduce memory consumption when JSON-encoding a large
+ container of many small objects.
+
- Issue #12650: Fix a race condition where a subprocess.Popen could leak
resources (FD/zombie) when killed at the wrong time.
diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c
index cdaa36e..a8631e0 100644
--- a/Modules/_io/bufferedio.c
+++ b/Modules/_io/bufferedio.c
@@ -753,6 +753,28 @@
*/
static PyObject *
+buffered_flush_and_rewind_unlocked(buffered *self)
+{
+ PyObject *res;
+
+ res = _bufferedwriter_flush_unlocked(self, 0);
+ if (res == NULL)
+ return NULL;
+ Py_DECREF(res);
+
+ if (self->readable) {
+ /* Rewind the raw stream so that its position corresponds to
+ the current logical position. */
+ Py_off_t n;
+ n = _buffered_raw_seek(self, -RAW_OFFSET(self), 1);
+ _bufferedreader_reset_buf(self);
+ if (n == -1)
+ return NULL;
+ }
+ Py_RETURN_NONE;
+}
+
+static PyObject *
buffered_flush(buffered *self, PyObject *args)
{
PyObject *res;
@@ -762,16 +784,7 @@
if (!ENTER_BUFFERED(self))
return NULL;
- res = _bufferedwriter_flush_unlocked(self, 0);
- if (res != NULL && self->readable) {
- /* Rewind the raw stream so that its position corresponds to
- the current logical position. */
- Py_off_t n;
- n = _buffered_raw_seek(self, -RAW_OFFSET(self), 1);
- if (n == -1)
- Py_CLEAR(res);
- _bufferedreader_reset_buf(self);
- }
+ res = buffered_flush_and_rewind_unlocked(self);
LEAVE_BUFFERED(self)
return res;
@@ -792,7 +805,7 @@
return NULL;
if (self->writable) {
- res = _bufferedwriter_flush_unlocked(self, 1);
+ res = buffered_flush_and_rewind_unlocked(self);
if (res == NULL)
goto end;
Py_CLEAR(res);
@@ -827,19 +840,18 @@
if (!ENTER_BUFFERED(self))
return NULL;
res = _bufferedreader_read_all(self);
- LEAVE_BUFFERED(self)
}
else {
res = _bufferedreader_read_fast(self, n);
- if (res == Py_None) {
- Py_DECREF(res);
- if (!ENTER_BUFFERED(self))
- return NULL;
- res = _bufferedreader_read_generic(self, n);
- LEAVE_BUFFERED(self)
- }
+ if (res != Py_None)
+ return res;
+ Py_DECREF(res);
+ if (!ENTER_BUFFERED(self))
+ return NULL;
+ res = _bufferedreader_read_generic(self, n);
}
+ LEAVE_BUFFERED(self)
return res;
}
@@ -865,13 +877,6 @@
if (!ENTER_BUFFERED(self))
return NULL;
- if (self->writable) {
- res = _bufferedwriter_flush_unlocked(self, 1);
- if (res == NULL)
- goto end;
- Py_CLEAR(res);
- }
-
/* Return up to n bytes. If at least one byte is buffered, we
only return buffered bytes. Otherwise, we do one raw read. */
@@ -891,6 +896,13 @@
goto end;
}
+ if (self->writable) {
+ res = buffered_flush_and_rewind_unlocked(self);
+ if (res == NULL)
+ goto end;
+ Py_DECREF(res);
+ }
+
/* Fill the buffer from the raw stream, and copy it to the result. */
_bufferedreader_reset_buf(self);
r = _bufferedreader_fill_buffer(self);
@@ -939,7 +951,7 @@
goto end_unlocked;
if (self->writable) {
- res = _bufferedwriter_flush_unlocked(self, 0);
+ res = buffered_flush_and_rewind_unlocked(self);
if (res == NULL)
goto end;
Py_CLEAR(res);
@@ -1022,12 +1034,6 @@
goto end_unlocked;
/* Now we try to get some more from the raw stream */
- if (self->writable) {
- res = _bufferedwriter_flush_unlocked(self, 1);
- if (res == NULL)
- goto end;
- Py_CLEAR(res);
- }
chunks = PyList_New(0);
if (chunks == NULL)
goto end;
@@ -1041,9 +1047,16 @@
}
Py_CLEAR(res);
written += n;
+ self->pos += n;
if (limit >= 0)
limit -= n;
}
+ if (self->writable) {
+ PyObject *r = buffered_flush_and_rewind_unlocked(self);
+ if (r == NULL)
+ goto end;
+ Py_DECREF(r);
+ }
for (;;) {
_bufferedreader_reset_buf(self);
@@ -1212,20 +1225,11 @@
return NULL;
if (self->writable) {
- res = _bufferedwriter_flush_unlocked(self, 0);
+ res = buffered_flush_and_rewind_unlocked(self);
if (res == NULL)
goto end;
Py_CLEAR(res);
}
- if (self->readable) {
- if (pos == Py_None) {
- /* Rewind the raw stream so that its position corresponds to
- the current logical position. */
- if (_buffered_raw_seek(self, -RAW_OFFSET(self), 1) == -1)
- goto end;
- }
- _bufferedreader_reset_buf(self);
- }
res = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_truncate, pos, NULL);
if (res == NULL)
goto end;
@@ -1416,15 +1420,16 @@
self->buffer + self->pos, current_size);
if (data == NULL)
return NULL;
+ self->pos += current_size;
}
- _bufferedreader_reset_buf(self);
/* We're going past the buffer's bounds, flush it */
if (self->writable) {
- res = _bufferedwriter_flush_unlocked(self, 1);
+ res = buffered_flush_and_rewind_unlocked(self);
if (res == NULL)
return NULL;
Py_CLEAR(res);
}
+ _bufferedreader_reset_buf(self);
if (PyObject_HasAttr(self->raw, _PyIO_str_readall)) {
chunk = PyObject_CallMethodObjArgs(self->raw, _PyIO_str_readall, NULL);
@@ -1540,6 +1545,14 @@
memcpy(out, self->buffer + self->pos, current_size);
remaining -= current_size;
written += current_size;
+ self->pos += current_size;
+ }
+ /* Flush the write buffer if necessary */
+ if (self->writable) {
+ PyObject *r = buffered_flush_and_rewind_unlocked(self);
+ if (r == NULL)
+ goto error;
+ Py_DECREF(r);
}
_bufferedreader_reset_buf(self);
while (remaining > 0) {
diff --git a/Modules/_json.c b/Modules/_json.c
index a392c97..d5120fa 100644
--- a/Modules/_json.c
+++ b/Modules/_json.c
@@ -75,6 +75,122 @@
{NULL}
};
+/*
+ * A two-level accumulator of unicode objects that avoids both the overhead
+ * of keeping a huge number of small separate objects, and the quadratic
+ * behaviour of using a naive repeated concatenation scheme.
+ */
+
+typedef struct {
+ PyObject *large; /* A list of previously accumulated large strings */
+ PyObject *small; /* Pending small strings */
+} accumulator;
+
+static PyObject *
+join_list_unicode(PyObject *lst)
+{
+ /* return u''.join(lst) */
+ static PyObject *sep = NULL;
+ if (sep == NULL) {
+ sep = PyUnicode_FromStringAndSize("", 0);
+ if (sep == NULL)
+ return NULL;
+ }
+ return PyUnicode_Join(sep, lst);
+}
+
+static int
+init_accumulator(accumulator *acc)
+{
+ acc->large = PyList_New(0);
+ if (acc->large == NULL)
+ return -1;
+ acc->small = PyList_New(0);
+ if (acc->small == NULL) {
+ Py_CLEAR(acc->large);
+ return -1;
+ }
+ return 0;
+}
+
+static int
+flush_accumulator(accumulator *acc)
+{
+ Py_ssize_t nsmall = PyList_GET_SIZE(acc->small);
+ if (nsmall) {
+ int ret;
+ PyObject *joined = join_list_unicode(acc->small);
+ if (joined == NULL)
+ return -1;
+ if (PyList_SetSlice(acc->small, 0, nsmall, NULL)) {
+ Py_DECREF(joined);
+ return -1;
+ }
+ ret = PyList_Append(acc->large, joined);
+ Py_DECREF(joined);
+ return ret;
+ }
+ return 0;
+}
+
+static int
+accumulate_unicode(accumulator *acc, PyObject *obj)
+{
+ int ret;
+ Py_ssize_t nsmall;
+ PyObject *joined;
+ assert(PyUnicode_Check(obj));
+
+ if (PyList_Append(acc->small, obj))
+ return -1;
+ nsmall = PyList_GET_SIZE(acc->small);
+ /* Each item in a list of unicode objects has an overhead (in 64-bit
+ * builds) of:
+ * - 8 bytes for the list slot
+ * - 56 bytes for the header of the unicode object
+ * that is, 64 bytes. 100000 such objects waste more than 6MB
+ * compared to a single concatenated string.
+ */
+ if (nsmall < 100000)
+ return 0;
+ joined = join_list_unicode(acc->small);
+ if (joined == NULL)
+ return -1;
+ if (PyList_SetSlice(acc->small, 0, nsmall, NULL)) {
+ Py_DECREF(joined);
+ return -1;
+ }
+ ret = PyList_Append(acc->large, joined);
+ Py_DECREF(joined);
+ return ret;
+}
+
+static PyObject *
+finish_accumulator(accumulator *acc)
+{
+ int ret;
+ PyObject *res;
+
+ ret = flush_accumulator(acc);
+ Py_CLEAR(acc->small);
+ if (ret) {
+ Py_CLEAR(acc->large);
+ return NULL;
+ }
+ res = acc->large;
+ acc->large = NULL;
+ return res;
+}
+
+static void
+destroy_accumulator(accumulator *acc)
+{
+ Py_CLEAR(acc->small);
+ Py_CLEAR(acc->large);
+}
+
+/* Forward decls */
+
static PyObject *
ascii_escape_unicode(PyObject *pystr);
static PyObject *
@@ -101,11 +217,11 @@
static int
encoder_clear(PyObject *self);
static int
-encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level);
+encoder_listencode_list(PyEncoderObject *s, accumulator *acc, PyObject *seq, Py_ssize_t indent_level);
static int
-encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level);
+encoder_listencode_obj(PyEncoderObject *s, accumulator *acc, PyObject *obj, Py_ssize_t indent_level);
static int
-encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level);
+encoder_listencode_dict(PyEncoderObject *s, accumulator *acc, PyObject *dct, Py_ssize_t indent_level);
static PyObject *
_encoded_const(PyObject *obj);
static void
@@ -267,19 +383,6 @@
}
static PyObject *
-join_list_unicode(PyObject *lst)
-{
- /* return u''.join(lst) */
- static PyObject *sep = NULL;
- if (sep == NULL) {
- sep = PyUnicode_FromStringAndSize("", 0);
- if (sep == NULL)
- return NULL;
- }
- return PyUnicode_Join(sep, lst);
-}
-
-static PyObject *
_build_rval_index_tuple(PyObject *rval, Py_ssize_t idx) {
/* return (rval, idx) tuple, stealing reference to rval */
PyObject *tpl;
@@ -1226,22 +1329,22 @@
/* Python callable interface to encode_listencode_obj */
static char *kwlist[] = {"obj", "_current_indent_level", NULL};
PyObject *obj;
- PyObject *rval;
Py_ssize_t indent_level;
PyEncoderObject *s;
+ accumulator acc;
+
assert(PyEncoder_Check(self));
s = (PyEncoderObject *)self;
if (!PyArg_ParseTupleAndKeywords(args, kwds, "OO&:_iterencode", kwlist,
&obj, _convertPyInt_AsSsize_t, &indent_level))
return NULL;
- rval = PyList_New(0);
- if (rval == NULL)
+ if (init_accumulator(&acc))
return NULL;
- if (encoder_listencode_obj(s, rval, obj, indent_level)) {
- Py_DECREF(rval);
+ if (encoder_listencode_obj(s, &acc, obj, indent_level)) {
+ destroy_accumulator(&acc);
return NULL;
}
- return rval;
+ return finish_accumulator(&acc);
}
static PyObject *
@@ -1313,18 +1416,19 @@
}
static int
-_steal_list_append(PyObject *lst, PyObject *stolen)
+_steal_accumulate(accumulator *acc, PyObject *stolen)
{
/* Append stolen and then decrement its reference count */
- int rval = PyList_Append(lst, stolen);
+ int rval = accumulate_unicode(acc, stolen);
Py_DECREF(stolen);
return rval;
}
static int
-encoder_listencode_obj(PyEncoderObject *s, PyObject *rval, PyObject *obj, Py_ssize_t indent_level)
+encoder_listencode_obj(PyEncoderObject *s, accumulator *acc,
+ PyObject *obj, Py_ssize_t indent_level)
{
- /* Encode Python object obj to a JSON term, rval is a PyList */
+ /* Encode Python object obj to a JSON term */
PyObject *newobj;
int rv;
@@ -1332,38 +1436,38 @@
PyObject *cstr = _encoded_const(obj);
if (cstr == NULL)
return -1;
- return _steal_list_append(rval, cstr);
+ return _steal_accumulate(acc, cstr);
}
else if (PyUnicode_Check(obj))
{
PyObject *encoded = encoder_encode_string(s, obj);
if (encoded == NULL)
return -1;
- return _steal_list_append(rval, encoded);
+ return _steal_accumulate(acc, encoded);
}
else if (PyLong_Check(obj)) {
PyObject *encoded = PyObject_Str(obj);
if (encoded == NULL)
return -1;
- return _steal_list_append(rval, encoded);
+ return _steal_accumulate(acc, encoded);
}
else if (PyFloat_Check(obj)) {
PyObject *encoded = encoder_encode_float(s, obj);
if (encoded == NULL)
return -1;
- return _steal_list_append(rval, encoded);
+ return _steal_accumulate(acc, encoded);
}
else if (PyList_Check(obj) || PyTuple_Check(obj)) {
if (Py_EnterRecursiveCall(" while encoding a JSON object"))
return -1;
- rv = encoder_listencode_list(s, rval, obj, indent_level);
+ rv = encoder_listencode_list(s, acc, obj, indent_level);
Py_LeaveRecursiveCall();
return rv;
}
else if (PyDict_Check(obj)) {
if (Py_EnterRecursiveCall(" while encoding a JSON object"))
return -1;
- rv = encoder_listencode_dict(s, rval, obj, indent_level);
+ rv = encoder_listencode_dict(s, acc, obj, indent_level);
Py_LeaveRecursiveCall();
return rv;
}
@@ -1394,7 +1498,7 @@
if (Py_EnterRecursiveCall(" while encoding a JSON object"))
return -1;
- rv = encoder_listencode_obj(s, rval, newobj, indent_level);
+ rv = encoder_listencode_obj(s, acc, newobj, indent_level);
Py_LeaveRecursiveCall();
Py_DECREF(newobj);
@@ -1414,9 +1518,10 @@
}
static int
-encoder_listencode_dict(PyEncoderObject *s, PyObject *rval, PyObject *dct, Py_ssize_t indent_level)
+encoder_listencode_dict(PyEncoderObject *s, accumulator *acc,
+ PyObject *dct, Py_ssize_t indent_level)
{
- /* Encode Python dict dct a JSON term, rval is a PyList */
+ /* Encode Python dict dct a JSON term */
static PyObject *open_dict = NULL;
static PyObject *close_dict = NULL;
static PyObject *empty_dict = NULL;
@@ -1436,7 +1541,7 @@
return -1;
}
if (Py_SIZE(dct) == 0)
- return PyList_Append(rval, empty_dict);
+ return accumulate_unicode(acc, empty_dict);
if (s->markers != Py_None) {
int has_key;
@@ -1454,7 +1559,7 @@
}
}
- if (PyList_Append(rval, open_dict))
+ if (accumulate_unicode(acc, open_dict))
goto bail;
if (s->indent != Py_None) {
@@ -1541,7 +1646,7 @@
}
if (idx) {
- if (PyList_Append(rval, s->item_separator))
+ if (accumulate_unicode(acc, s->item_separator))
goto bail;
}
@@ -1549,16 +1654,16 @@
Py_CLEAR(kstr);
if (encoded == NULL)
goto bail;
- if (PyList_Append(rval, encoded)) {
+ if (accumulate_unicode(acc, encoded)) {
Py_DECREF(encoded);
goto bail;
}
Py_DECREF(encoded);
- if (PyList_Append(rval, s->key_separator))
+ if (accumulate_unicode(acc, s->key_separator))
goto bail;
value = PyTuple_GET_ITEM(item, 1);
- if (encoder_listencode_obj(s, rval, value, indent_level))
+ if (encoder_listencode_obj(s, acc, value, indent_level))
goto bail;
idx += 1;
Py_DECREF(item);
@@ -1578,7 +1683,7 @@
yield '\n' + (' ' * (_indent * _current_indent_level))
}*/
- if (PyList_Append(rval, close_dict))
+ if (accumulate_unicode(acc, close_dict))
goto bail;
return 0;
@@ -1592,9 +1697,10 @@
static int
-encoder_listencode_list(PyEncoderObject *s, PyObject *rval, PyObject *seq, Py_ssize_t indent_level)
+encoder_listencode_list(PyEncoderObject *s, accumulator *acc,
+ PyObject *seq, Py_ssize_t indent_level)
{
- /* Encode Python list seq to a JSON term, rval is a PyList */
+ /* Encode Python list seq to a JSON term */
static PyObject *open_array = NULL;
static PyObject *close_array = NULL;
static PyObject *empty_array = NULL;
@@ -1618,7 +1724,7 @@
num_items = PySequence_Fast_GET_SIZE(s_fast);
if (num_items == 0) {
Py_DECREF(s_fast);
- return PyList_Append(rval, empty_array);
+ return accumulate_unicode(acc, empty_array);
}
if (s->markers != Py_None) {
@@ -1638,7 +1744,7 @@
}
seq_items = PySequence_Fast_ITEMS(s_fast);
- if (PyList_Append(rval, open_array))
+ if (accumulate_unicode(acc, open_array))
goto bail;
if (s->indent != Py_None) {
/* TODO: DOES NOT RUN */
@@ -1652,10 +1758,10 @@
for (i = 0; i < num_items; i++) {
PyObject *obj = seq_items[i];
if (i) {
- if (PyList_Append(rval, s->item_separator))
+ if (accumulate_unicode(acc, s->item_separator))
goto bail;
}
- if (encoder_listencode_obj(s, rval, obj, indent_level))
+ if (encoder_listencode_obj(s, acc, obj, indent_level))
goto bail;
}
if (ident != NULL) {
@@ -1670,7 +1776,7 @@
yield '\n' + (' ' * (_indent * _current_indent_level))
}*/
- if (PyList_Append(rval, close_array))
+ if (accumulate_unicode(acc, close_array))
goto bail;
Py_DECREF(s_fast);
return 0;
diff --git a/Objects/genobject.c b/Objects/genobject.c
index cb2980c..43d0335 100644
--- a/Objects/genobject.c
+++ b/Objects/genobject.c
@@ -100,6 +100,17 @@
if (!result || f->f_stacktop == NULL) {
/* generator can't be rerun, so release the frame */
+ /* first clean reference cycle through stored exception traceback */
+ PyObject *t, *v, *tb;
+ t = f->f_exc_type;
+ v = f->f_exc_value;
+ tb = f->f_exc_traceback;
+ f->f_exc_type = NULL;
+ f->f_exc_value = NULL;
+ f->f_exc_traceback = NULL;
+ Py_XDECREF(t);
+ Py_XDECREF(v);
+ Py_XDECREF(tb);
Py_DECREF(f);
gen->gi_frame = NULL;
}