commit a36ead4a7d5641181ab994a13b7d5027c1b5bd07
author Geoff Lang <geofflang@google.com> Fri Aug 02 11:54:08 2013 -0400
committer Geoff Lang <geofflang@google.com> Fri Aug 02 17:04:26 2013 -0400
tree 88f0c0db16833b793ea85a201f873c32f90c7d90
parent af88d7290afacb0a8abd13d331f7ed7290d4918b

Updated VertexBuffer's getSpaceRequired and storeVertexAttributes methods to return bools and fixed some validation of parameters to prevent under and overflows.

TRAC #23643

Signed-off-by: Nicolas Capens
Signed-off-by: Shannon Woods
Author: Geoff Lang

src/libGLESv2/renderer/IndexDataManager.cpp

diff --git a/src/libGLESv2/renderer/IndexDataManager.cpp b/src/libGLESv2/renderer/IndexDataManager.cpp
index 5056bcb..666f493 100644
--- a/src/libGLESv2/renderer/IndexDataManager.cpp
+++ b/src/libGLESv2/renderer/IndexDataManager.cpp
@@ -132,7 +132,16 @@
           default: UNREACHABLE(); alignedOffset = false;
         }
 
-        if (gl::GetTypeBytes(type) * count + offset > storage->getSize())
+        unsigned int typeSize = gl::GetTypeBytes(type);
+
+        // check for interger overflows and underflows
+        if (static_cast<unsigned int>(offset) > (std::numeric_limits<unsigned int>::max() / typeSize) ||
+            static_cast<unsigned int>(count) > ((std::numeric_limits<unsigned int>::max() / typeSize) - offset))
+        {
+            return GL_OUT_OF_MEMORY;
+        }
+
+        if (typeSize * static_cast<unsigned int>(count) + offset > storage->getSize())
         {
             return GL_INVALID_OPERATION;
         }
@@ -146,7 +155,7 @@
     IndexBufferInterface *indexBuffer = streamingBuffer;
     bool directStorage = alignedOffset && storage && storage->supportsDirectBinding() &&
                          destinationIndexType == type;
-    UINT streamOffset = 0;
+    unsigned int streamOffset = 0;
 
     if (directStorage)
     {
@@ -176,7 +185,7 @@
     }
     else
     {
-        int convertCount = count;
+        unsigned int convertCount = count;
 
         if (staticBuffer)
         {
@@ -198,12 +207,22 @@
             return GL_INVALID_OPERATION;
         }
 
-        unsigned int bufferSizeRequired = convertCount * gl::GetTypeBytes(destinationIndexType);
-        indexBuffer->reserveBufferSpace(bufferSizeRequired, type);
+        unsigned int indexTypeSize = gl::GetTypeBytes(destinationIndexType);
+        if (convertCount > std::numeric_limits<unsigned int>::max() / indexTypeSize)
+        {
+            ERR("Reserving %u indicies of %u bytes each exceeds the maximum buffer size.", convertCount, indexTypeSize);
+            return GL_OUT_OF_MEMORY;
+        }
+
+        unsigned int bufferSizeRequired = convertCount * indexTypeSize;
+        if (!indexBuffer->reserveBufferSpace(bufferSizeRequired, type))
+        {
+            ERR("Failed to reserve %u bytes in an index buffer.", bufferSizeRequired);
+            return GL_OUT_OF_MEMORY;
+        }
 
         void* output = NULL;
-        streamOffset = indexBuffer->mapBuffer(bufferSizeRequired, &output);
-        if (streamOffset == -1 || output == NULL)
+        if (!indexBuffer->mapBuffer(bufferSizeRequired, &output, &streamOffset))
         {
             ERR("Failed to map index buffer.");
             return GL_OUT_OF_MEMORY;
@@ -254,7 +273,7 @@
             mCountingBuffer->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_SHORT);
 
             void* mappedMemory = NULL;
-            if (mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory) == -1 || mappedMemory == NULL)
+            if (!mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory, NULL))
             {
                 ERR("Failed to map counting buffer.");
                 return NULL;
@@ -284,7 +303,7 @@
             mCountingBuffer->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_INT);
 
             void* mappedMemory = NULL;
-            if (mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory) == -1 || mappedMemory == NULL)
+            if (!mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory, NULL))
             {
                 ERR("Failed to map counting buffer.");
                 return NULL;