commit c04ddff290fc203d05b75c8569b748525fb76b5b
author Mark Dickinson <mdickinson@enthought.com> Sat Oct 06 18:04:49 2012 +0100
committer Mark Dickinson <mdickinson@enthought.com> Sat Oct 06 18:04:49 2012 +0100
tree 184849e76ebe965016c2737939f9eaa0dfb900ee
parent a2028733ef072740921017e544d29d191fdb2c9c

Issue #16096: Fix several occurrences of potential signed integer overflow.  Thanks Serhiy Storchaka.

Modules/_codecsmodule.c

diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c
index 7818f9a..40037b1 100644
--- a/Modules/_codecsmodule.c
+++ b/Modules/_codecsmodule.c
@@ -177,12 +177,12 @@
         return NULL;
 
     size = PyBytes_GET_SIZE(str);
-    newsize = 4*size;
-    if (newsize > PY_SSIZE_T_MAX || newsize / 4 != size) {
+    if (size > PY_SSIZE_T_MAX / 4) {
         PyErr_SetString(PyExc_OverflowError,
             "string is too large to encode");
             return NULL;
     }
+    newsize = 4*size;
     v = PyBytes_FromStringAndSize(NULL, newsize);
 
     if (v == NULL) {

Modules/_datetimemodule.c

diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c
index 01c85d1..873d46f 100644
--- a/Modules/_datetimemodule.c
+++ b/Modules/_datetimemodule.c
@@ -1265,14 +1265,13 @@
         assert(ptoappend != NULL);
         assert(ntoappend > 0);
         while (usednew + ntoappend > totalnew) {
-            size_t bigger = totalnew << 1;
-            if ((bigger >> 1) != totalnew) { /* overflow */
+            if (totalnew > (PY_SSIZE_T_MAX >> 1)) { /* overflow */
                 PyErr_NoMemory();
                 goto Done;
             }
-            if (_PyBytes_Resize(&newfmt, bigger) < 0)
+            totalnew <<= 1;
+            if (_PyBytes_Resize(&newfmt, totalnew) < 0)
                 goto Done;
-            totalnew = bigger;
             pnew = PyBytes_AsString(newfmt) + usednew;
         }
         memcpy(pnew, ptoappend, ntoappend);

Modules/_randommodule.c

diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c
index 421a0d8..6540ab9 100644
--- a/Modules/_randommodule.c
+++ b/Modules/_randommodule.c
@@ -284,7 +284,8 @@
         n = newn;
         if (keyused >= keymax) {
             unsigned long bigger = keymax << 1;
-            if ((bigger >> 1) != keymax) {
+            if ((bigger >> 1) != keymax ||
+                bigger > PY_SSIZE_T_MAX / sizeof(*key)) {
                 PyErr_NoMemory();
                 goto Done;
             }

Modules/arraymodule.c

diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c
index 04eb67c..3f5aa8b 100644
--- a/Modules/arraymodule.c
+++ b/Modules/arraymodule.c
@@ -483,11 +483,11 @@
         return NULL;
     }
 
-    nbytes = size * descr->itemsize;
     /* Check for overflow */
-    if (nbytes / descr->itemsize != (size_t)size) {
+    if (size > PY_SSIZE_T_MAX / descr->itemsize) {
         return PyErr_NoMemory();
     }
+    nbytes = size * descr->itemsize;
     op = (arrayobject *) type->tp_alloc(type, 0);
     if (op == NULL) {
         return NULL;
@@ -1251,11 +1251,15 @@
     if (!PyArg_ParseTuple(args, "On:fromfile", &f, &n))
         return NULL;
 
-    nbytes = n * itemsize;
-    if (nbytes < 0 || nbytes/itemsize != n) {
+    if (n < 0) {
+        PyErr_SetString(PyExc_ValueError, "negative count");
+        return NULL;
+    }
+    if (n > PY_SSIZE_T_MAX / itemsize) {
         PyErr_NoMemory();
         return NULL;
     }
+    nbytes = n * itemsize;
 
     b = _PyObject_CallMethodId(f, &PyId_read, "n", nbytes);
     if (b == NULL)

Modules/audioop.c

diff --git a/Modules/audioop.c b/Modules/audioop.c
index 0375e4e..2bca391 100644
--- a/Modules/audioop.c
+++ b/Modules/audioop.c
@@ -1108,8 +1108,7 @@
         PyErr_SetString(AudioopError, "# of channels should be >= 1");
         return NULL;
     }
-    bytes_per_frame = size * nchannels;
-    if (bytes_per_frame / nchannels != size) {
+    if (size > INT_MAX / nchannels) {
         /* This overflow test is rigorously correct because
            both multiplicands are >= 1.  Use the argument names
            from the docs for the error msg. */
@@ -1117,6 +1116,7 @@
                         "width * nchannels too big for a C int");
         return NULL;
     }
+    bytes_per_frame = size * nchannels;
     if (weightA < 1 || weightB < 0) {
         PyErr_SetString(AudioopError,
             "weightA should be >= 1, weightB should be >= 0");