Years in the making.
objimpl.h, pymem.h: Stop mapping PyMem_{Del, DEL} and PyMem_{Free, FREE}
to PyObject_{Free, FREE} in a release build. They're aliases for the
system free() now.
_subprocess.c/sp_handle_dealloc(): Since the memory was originally
obtained via PyObject_NEW, it must be released via PyObject_FREE (or
_DEL).
pythonrun.c, tokenizer.c, parsermodule.c: I lost count of the number of
PyObject vs PyMem mismatches in these -- it's like the specific
function called at each site was picked at random, sometimes even with
memory obtained via PyMem getting released via PyObject. Changed most
to use PyObject uniformly, since the blobs allocated are predictably
small in most cases, and obmalloc is generally faster than system
mallocs then.
If extension modules in real life prove as sloppy as Python's front
end, we'll have to revert the objimpl.h + pymem.h part of this patch.
Note that no problems will show up in a debug build (all calls still go
thru obmalloc then). Problems will show up only in a release build, most
likely segfaults.
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index b0d9b80..001d31a 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -163,7 +163,7 @@
{
tok->decoding_erred = 1;
if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
- PyMem_DEL(tok->buf);
+ PyObject_FREE(tok->buf);
tok->buf = NULL;
return NULL; /* as if it were EOF */
}
@@ -171,7 +171,7 @@
static char *
new_string(const char *s, Py_ssize_t len)
{
- char* result = PyMem_NEW(char, len + 1);
+ char* result = (char *)PyObject_MALLOC(len + 1);
if (result != NULL) {
memcpy(result, s, len);
result[len] = '\0';
@@ -236,7 +236,7 @@
char* r = new_string(begin, t - begin);
char* q = get_normal_name(r);
if (r != q) {
- PyMem_DEL(r);
+ PyObject_FREE(r);
r = new_string(q, strlen(q));
}
return r;
@@ -277,18 +277,18 @@
tok->decoding_state = -1;
}
else
- PyMem_DEL(cs);
+ PyObject_FREE(cs);
#else
/* Without Unicode support, we cannot
process the coding spec. Since there
won't be any Unicode literals, that
won't matter. */
- PyMem_DEL(cs);
+ PyObject_FREE(cs);
#endif
}
} else { /* then, compare cs with BOM */
r = (strcmp(tok->encoding, cs) == 0);
- PyMem_DEL(cs);
+ PyObject_FREE(cs);
}
}
if (!r) {
@@ -334,7 +334,7 @@
return 1;
}
if (tok->encoding != NULL)
- PyMem_DEL(tok->encoding);
+ PyObject_FREE(tok->encoding);
tok->encoding = new_string("utf-8", 5); /* resulting is in utf-8 */
return 1;
NON_BOM:
@@ -345,7 +345,7 @@
/* Read a line of text from TOK into S, using the stream in TOK.
Return NULL on failure, else S.
-
+
On entry, tok->decoding_buffer will be one of:
1) NULL: need to call tok->decoding_readline to get a new line
2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
@@ -354,7 +354,7 @@
(in the s buffer) to copy entire contents of the line read
by tok->decoding_readline. tok->decoding_buffer has the overflow.
In this case, fp_readl is called in a loop (with an expanded buffer)
- until the buffer ends with a '\n' (or until the end of the file is
+ until the buffer ends with a '\n' (or until the end of the file is
reached): see tok_nextc and its calls to decoding_fgets.
*/
@@ -470,7 +470,7 @@
break;
} else if (tok->decoding_state > 0) {
/* We want a 'raw' read. */
- line = Py_UniversalNewlineFgets(s, size,
+ line = Py_UniversalNewlineFgets(s, size,
tok->fp, NULL);
break;
} else {
@@ -502,11 +502,11 @@
char buf[500];
/* Need to add 1 to the line number, since this line
has not been counted, yet. */
- sprintf(buf,
+ sprintf(buf,
"Non-ASCII character '\\x%.2x' "
"in file %.200s on line %i, "
"but no encoding declared; "
- "see http://www.python.org/peps/pep-0263.html for details",
+ "see http://www.python.org/peps/pep-0263.html for details",
badchar, tok->filename, tok->lineno + 1);
PyErr_SetString(PyExc_SyntaxError, buf);
return error_ret(tok);
@@ -537,13 +537,15 @@
/* Fetch a byte from TOK, using the string buffer. */
-static int buf_getc(struct tok_state *tok) {
+static int
+buf_getc(struct tok_state *tok) {
return Py_CHARMASK(*tok->str++);
}
/* Unfetch a byte from TOK, using the string buffer. */
-static void buf_ungetc(int c, struct tok_state *tok) {
+static void
+buf_ungetc(int c, struct tok_state *tok) {
tok->str--;
assert(Py_CHARMASK(*tok->str) == c); /* tok->cur may point to read-only segment */
}
@@ -551,7 +553,8 @@
/* Set the readline function for TOK to ENC. For the string-based
tokenizer, this means to just record the encoding. */
-static int buf_setreadl(struct tok_state *tok, const char* enc) {
+static int
+buf_setreadl(struct tok_state *tok, const char* enc) {
tok->enc = enc;
return 1;
}
@@ -653,7 +656,7 @@
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
- if ((tok->buf = PyMem_NEW(char, BUFSIZ)) == NULL) {
+ if ((tok->buf = (char *)PyObject_MALLOC(BUFSIZ)) == NULL) {
PyTokenizer_Free(tok);
return NULL;
}
@@ -672,14 +675,14 @@
PyTokenizer_Free(struct tok_state *tok)
{
if (tok->encoding != NULL)
- PyMem_DEL(tok->encoding);
+ PyObject_FREE(tok->encoding);
#ifndef PGEN
Py_XDECREF(tok->decoding_readline);
Py_XDECREF(tok->decoding_buffer);
#endif
if (tok->fp != NULL && tok->buf != NULL)
- PyMem_DEL(tok->buf);
- PyMem_DEL(tok);
+ PyObject_FREE(tok->buf);
+ PyMem_FREE(tok);
}
#if !defined(PGEN) && defined(Py_USING_UNICODE)
@@ -721,7 +724,7 @@
PyMem_FREE(*inp);
*inp = converted;
if (tok->encoding != NULL)
- PyMem_DEL(tok->encoding);
+ PyObject_FREE(tok->encoding);
tok->encoding = new_string(encoding, strlen(encoding));
if (tok->encoding == NULL)
goto error_nomem;
@@ -790,10 +793,10 @@
size_t oldlen = tok->cur - tok->buf;
size_t newlen = oldlen + strlen(new);
char *buf = tok->buf;
- PyMem_RESIZE(buf, char, newlen+1);
+ buf = (char *)PyObject_REALLOC(buf, newlen+1);
tok->lineno++;
if (buf == NULL) {
- PyMem_DEL(tok->buf);
+ PyObject_FREE(tok->buf);
tok->buf = NULL;
PyMem_FREE(new);
tok->done = E_NOMEM;
@@ -811,7 +814,7 @@
else {
tok->lineno++;
if (tok->buf != NULL)
- PyMem_DEL(tok->buf);
+ PyObject_FREE(tok->buf);
tok->buf = new;
tok->line_start = tok->buf;
tok->cur = tok->buf;
@@ -826,7 +829,8 @@
char *pt;
if (tok->start == NULL) {
if (tok->buf == NULL) {
- tok->buf = PyMem_NEW(char, BUFSIZ);
+ tok->buf = (char *)
+ PyObject_MALLOC(BUFSIZ);
if (tok->buf == NULL) {
tok->done = E_NOMEM;
return EOF;
@@ -861,7 +865,8 @@
Py_ssize_t curvalid = tok->inp - tok->buf;
Py_ssize_t newsize = curvalid + BUFSIZ;
char *newbuf = tok->buf;
- PyMem_RESIZE(newbuf, char, newsize);
+ newbuf = (char *)PyObject_REALLOC(newbuf,
+ newsize);
if (newbuf == NULL) {
tok->done = E_NOMEM;
tok->cur = tok->inp;
@@ -1184,9 +1189,9 @@
}
}
}
-
+
tok->start = tok->cur;
-
+
/* Return pending indents/dedents */
if (tok->pendin != 0) {
if (tok->pendin < 0) {
@@ -1198,17 +1203,17 @@
return INDENT;
}
}
-
+
again:
tok->start = NULL;
/* Skip spaces */
do {
c = tok_nextc(tok);
} while (c == ' ' || c == '\t' || c == '\014');
-
+
/* Set start of current token */
tok->start = tok->cur - 1;
-
+
/* Skip comment, while looking for tab-setting magic */
if (c == '#') {
static char *tabforms[] = {
@@ -1226,7 +1231,7 @@
} while (c != EOF && c != '\n' &&
tp - cbuf + 1 < sizeof(cbuf));
*tp = '\0';
- for (cp = tabforms;
+ for (cp = tabforms;
cp < tabforms + sizeof(tabforms)/sizeof(tabforms[0]);
cp++) {
if ((tp = strstr(cbuf, *cp))) {
@@ -1244,12 +1249,12 @@
while (c != EOF && c != '\n')
c = tok_nextc(tok);
}
-
+
/* Check for EOF and errors now */
if (c == EOF) {
return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
}
-
+
/* Identifier (most frequent token!) */
if (isalpha(c) || c == '_') {
/* Process r"", u"" and ur"" */
@@ -1277,7 +1282,7 @@
*p_end = tok->cur;
return NAME;
}
-
+
/* Newline */
if (c == '\n') {
tok->atbol = 1;
@@ -1288,7 +1293,7 @@
tok->cont_line = 0;
return NEWLINE;
}
-
+
/* Period or number starting with period? */
if (c == '.') {
c = tok_nextc(tok);
@@ -1451,7 +1456,7 @@
*p_end = tok->cur;
return STRING;
}
-
+
/* Line continuation */
if (c == '\\') {
c = tok_nextc(tok);
@@ -1463,7 +1468,7 @@
tok->cont_line = 1;
goto again; /* Read next line */
}
-
+
/* Check for two-character token */
{
int c2 = tok_nextc(tok);
@@ -1482,7 +1487,7 @@
}
tok_backup(tok, c2);
}
-
+
/* Keep track of parentheses nesting level */
switch (c) {
case '(':
@@ -1496,7 +1501,7 @@
tok->level--;
break;
}
-
+
/* Punctuation character */
*p_start = tok->start;
*p_end = tok->cur;