[3.8] bpo-36389: Backport debug enhancements from master (GH-16796)
* bpo-36389: _PyObject_CheckConsistency() available in release mode (GH-16612)
bpo-36389, bpo-38376: The _PyObject_CheckConsistency() function is
now also available in release mode. For example, it can be used to
debug a crash in the visit_decref() function of the GC.
Modify the following functions to also work in release mode:
* _PyDict_CheckConsistency()
* _PyObject_CheckConsistency()
* _PyType_CheckConsistency()
* _PyUnicode_CheckConsistency()
Other changes:
* _PyMem_IsPtrFreed(ptr) now also returns 1 if ptr is NULL
(equals to 0).
* _PyBytesWriter_CheckConsistency() now returns 1 and is only used
with assert().
* Reorder _PyObject_Dump() to write safe fields first, and only
attempt to render repr() at the end.
(cherry picked from commit 6876257eaabdb30f27ebcbd7d2557278ce2e5705)
* bpo-36389: Fix _PyBytesWriter in release mode (GH-16624)
Fix _PyBytesWriter API when Python is built in release mode with
assertions.
(cherry picked from commit 60ec6efd96d95476fe5e38c491491add04f026e5)
* bpo-38070: Enhance visit_decref() debug trace (GH-16631)
subtract_refs() now pass the parent object to visit_decref() which
pass it to _PyObject_ASSERT(). So if the "is freed" assertion fails,
the parent is used in debug trace, rather than the freed object. The
parent object is more likely to contain useful information. Freed
objects cannot be inspected are are displayed as "<object at xxx is
freed>" with no other detail.
(cherry picked from commit 4d5f94b8cd20f804c7868c5395a15aa6032f874c)
* Fix also a typo in PYMEM_DEADBYTE macro comment
* bpo-36389: Add newline to _PyObject_AssertFailed() (GH-16629)
Add a newline between the verbose object dump and the Py_FatalError()
logs for readability.
(cherry picked from commit 7775349895088a7ae68cecf0c74cf817f15e2c74)
diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c
index f420e19..bd6480a 100644
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@@ -2006,20 +2006,6 @@
* it wraps a real allocator, adding extra debugging info to the memory blocks.
*/
-/* Special bytes broadcast into debug memory blocks at appropriate times.
- * Strings of these are unlikely to be valid addresses, floats, ints or
- * 7-bit ASCII. If modified, _PyMem_IsPtrFreed() should be updated as well.
- *
- * Byte patterns 0xCB, 0xBB and 0xFB have been replaced with 0xCD, 0xDD and
- * 0xFD to use the same values than Windows CRT debug malloc() and free().
- */
-#undef CLEANBYTE
-#undef DEADBYTE
-#undef FORBIDDENBYTE
-#define CLEANBYTE 0xCD /* clean (newly allocated) memory */
-#define DEADBYTE 0xDD /* dead (newly freed) memory */
-#define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */
-
/* Uncomment this define to add the "serialno" field */
/* #define PYMEM_DEBUG_SERIALNO */
@@ -2081,14 +2067,14 @@
p[S]
API ID. See PEP 445. This is a character, but seems undocumented.
p[S+1: 2*S]
- Copies of FORBIDDENBYTE. Used to catch under- writes and reads.
+ Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads.
p[2*S: 2*S+n]
- The requested memory, filled with copies of CLEANBYTE.
+ The requested memory, filled with copies of PYMEM_CLEANBYTE.
Used to catch reference to uninitialized memory.
&p[2*S] is returned. Note that this is 8-byte aligned if pymalloc
handled the request itself.
p[2*S+n: 2*S+n+S]
- Copies of FORBIDDENBYTE. Used to catch over- writes and reads.
+ Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads.
p[2*S+n+S: 2*S+n+2*S]
A serial number, incremented by 1 on each call to _PyMem_DebugMalloc
and _PyMem_DebugRealloc.
@@ -2145,15 +2131,15 @@
/* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
write_size_t(p, nbytes);
p[SST] = (uint8_t)api->api_id;
- memset(p + SST + 1, FORBIDDENBYTE, SST-1);
+ memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);
if (nbytes > 0 && !use_calloc) {
- memset(data, CLEANBYTE, nbytes);
+ memset(data, PYMEM_CLEANBYTE, nbytes);
}
/* at tail, write pad (SST bytes) and serialno (SST bytes) */
tail = data + nbytes;
- memset(tail, FORBIDDENBYTE, SST);
+ memset(tail, PYMEM_FORBIDDENBYTE, SST);
#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, serialno);
#endif
@@ -2179,7 +2165,7 @@
/* The debug free first checks the 2*SST bytes on each end for sanity (in
particular, that the FORBIDDENBYTEs with the api ID are still intact).
- Then fills the original bytes with DEADBYTE.
+ Then fills the original bytes with PYMEM_DEADBYTE.
Then calls the underlying free.
*/
static void
@@ -2197,7 +2183,7 @@
_PyMem_DebugCheckAddress(api->api_id, p);
nbytes = read_size_t(q);
nbytes += PYMEM_DEBUG_EXTRA_BYTES;
- memset(q, DEADBYTE, nbytes);
+ memset(q, PYMEM_DEADBYTE, nbytes);
api->alloc.free(api->alloc.ctx, q);
}
@@ -2239,14 +2225,14 @@
*/
if (original_nbytes <= sizeof(save)) {
memcpy(save, data, original_nbytes);
- memset(data - 2 * SST, DEADBYTE,
+ memset(data - 2 * SST, PYMEM_DEADBYTE,
original_nbytes + PYMEM_DEBUG_EXTRA_BYTES);
}
else {
memcpy(save, data, ERASED_SIZE);
- memset(head, DEADBYTE, ERASED_SIZE + 2 * SST);
+ memset(head, PYMEM_DEADBYTE, ERASED_SIZE + 2 * SST);
memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE);
- memset(tail - ERASED_SIZE, DEADBYTE,
+ memset(tail - ERASED_SIZE, PYMEM_DEADBYTE,
ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST);
}
@@ -2268,10 +2254,10 @@
write_size_t(head, nbytes);
head[SST] = (uint8_t)api->api_id;
- memset(head + SST + 1, FORBIDDENBYTE, SST-1);
+ memset(head + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);
tail = data + nbytes;
- memset(tail, FORBIDDENBYTE, SST);
+ memset(tail, PYMEM_FORBIDDENBYTE, SST);
#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, block_serialno);
#endif
@@ -2295,7 +2281,8 @@
if (nbytes > original_nbytes) {
/* growing: mark new extra memory clean */
- memset(data + original_nbytes, CLEANBYTE, nbytes - original_nbytes);
+ memset(data + original_nbytes, PYMEM_CLEANBYTE,
+ nbytes - original_nbytes);
}
return data;
@@ -2374,7 +2361,7 @@
* the tail could lead to a segfault then.
*/
for (i = SST-1; i >= 1; --i) {
- if (*(q-i) != FORBIDDENBYTE) {
+ if (*(q-i) != PYMEM_FORBIDDENBYTE) {
msg = "bad leading pad byte";
goto error;
}
@@ -2383,7 +2370,7 @@
nbytes = read_size_t(q - 2*SST);
tail = q + nbytes;
for (i = 0; i < SST; ++i) {
- if (tail[i] != FORBIDDENBYTE) {
+ if (tail[i] != PYMEM_FORBIDDENBYTE) {
msg = "bad trailing pad byte";
goto error;
}
@@ -2423,7 +2410,7 @@
fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1);
ok = 1;
for (i = 1; i <= SST-1; ++i) {
- if (*(q-i) != FORBIDDENBYTE) {
+ if (*(q-i) != PYMEM_FORBIDDENBYTE) {
ok = 0;
break;
}
@@ -2432,11 +2419,11 @@
fputs("FORBIDDENBYTE, as expected.\n", stderr);
else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
- FORBIDDENBYTE);
+ PYMEM_FORBIDDENBYTE);
for (i = SST-1; i >= 1; --i) {
const uint8_t byte = *(q-i);
fprintf(stderr, " at p-%d: 0x%02x", i, byte);
- if (byte != FORBIDDENBYTE)
+ if (byte != PYMEM_FORBIDDENBYTE)
fputs(" *** OUCH", stderr);
fputc('\n', stderr);
}
@@ -2451,7 +2438,7 @@
fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail);
ok = 1;
for (i = 0; i < SST; ++i) {
- if (tail[i] != FORBIDDENBYTE) {
+ if (tail[i] != PYMEM_FORBIDDENBYTE) {
ok = 0;
break;
}
@@ -2460,12 +2447,12 @@
fputs("FORBIDDENBYTE, as expected.\n", stderr);
else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
- FORBIDDENBYTE);
+ PYMEM_FORBIDDENBYTE);
for (i = 0; i < SST; ++i) {
const uint8_t byte = tail[i];
fprintf(stderr, " at tail+%d: 0x%02x",
i, byte);
- if (byte != FORBIDDENBYTE)
+ if (byte != PYMEM_FORBIDDENBYTE)
fputs(" *** OUCH", stderr);
fputc('\n', stderr);
}