commit a36ead4a7d5641181ab994a13b7d5027c1b5bd07
author Geoff Lang <geofflang@google.com> Fri Aug 02 11:54:08 2013 -0400
committer Geoff Lang <geofflang@google.com> Fri Aug 02 17:04:26 2013 -0400
tree 88f0c0db16833b793ea85a201f873c32f90c7d90
parent af88d7290afacb0a8abd13d331f7ed7290d4918b

Updated VertexBuffer's getSpaceRequired and storeVertexAttributes methods to return bools and fixed some validation of parameters to prevent under and overflows.

TRAC #23643

Signed-off-by: Nicolas Capens
Signed-off-by: Shannon Woods
Author: Geoff Lang

src/libGLESv2/renderer/VertexDataManager.cpp

diff --git a/src/libGLESv2/renderer/VertexDataManager.cpp b/src/libGLESv2/renderer/VertexDataManager.cpp
index 4a57d91..9e663e2 100644
--- a/src/libGLESv2/renderer/VertexDataManager.cpp
+++ b/src/libGLESv2/renderer/VertexDataManager.cpp
@@ -26,9 +26,15 @@
 namespace rx
 {
 
-static int ElementsInBuffer(const gl::VertexAttribute &attribute, int size)
+static int ElementsInBuffer(const gl::VertexAttribute &attribute, unsigned int size)
 {
-    int stride = attribute.stride();
+    // Size cannot be larger than a GLsizei
+    if (size > static_cast<unsigned int>(std::numeric_limits<int>::max()))
+    {
+        size = static_cast<unsigned int>(std::numeric_limits<int>::max());
+    }
+
+    GLsizei stride = attribute.stride();
     return (size - attribute.mOffset % stride + (stride - attribute.typeSize())) / stride;
 }
 
@@ -97,7 +103,7 @@
             gl::Buffer *buffer = attribs[i].mBoundBuffer.get();
             StaticVertexBufferInterface *staticBuffer = buffer ? buffer->getStaticVertexBuffer() : NULL;
 
-            if (staticBuffer && staticBuffer->getBufferSize() > 0 && staticBuffer->lookupAttribute(attribs[i]) == -1 &&
+            if (staticBuffer && staticBuffer->getBufferSize() > 0 && !staticBuffer->lookupAttribute(attribs[i], NULL) &&
                 !DirectStoragePossible(staticBuffer, attribs[i], currentValues[i]))
             {
                 buffer->invalidateStaticData();
@@ -160,7 +166,7 @@
                 BufferStorage *storage = buffer ? buffer->getStorage() : NULL;
                 bool directStorage = DirectStoragePossible(vertexBuffer, attribs[i], currentValues[i]);
 
-                std::size_t streamOffset = -1;
+                unsigned int streamOffset = 0;
                 unsigned int outputElementSize = 0;
 
                 if (directStorage)
@@ -170,37 +176,41 @@
                 }
                 else if (staticBuffer)
                 {
-                    streamOffset = staticBuffer->lookupAttribute(attribs[i]);
-                    outputElementSize = staticBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0);
+                    if (!staticBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0, &outputElementSize))
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
 
-                    if (streamOffset == -1)
+                    if (!staticBuffer->lookupAttribute(attribs[i], &streamOffset))
                     {
                         // Convert the entire buffer
                         int totalCount = ElementsInBuffer(attribs[i], storage->getSize());
                         int startIndex = attribs[i].mOffset / attribs[i].stride();
 
-                        streamOffset = staticBuffer->storeVertexAttributes(attribs[i], currentValues[i], -startIndex, totalCount, 0);
-                    }
-
-                    if (streamOffset != -1)
-                    {
-                        streamOffset += (attribs[i].mOffset / attribs[i].stride()) * outputElementSize;
-
-                        if (instances == 0 || attribs[i].mDivisor == 0)
+                        if (!staticBuffer->storeVertexAttributes(attribs[i], currentValues[i], -startIndex, totalCount,
+                                                                 0, &streamOffset))
                         {
-                            streamOffset += start * outputElementSize;
+                            return GL_OUT_OF_MEMORY;
                         }
                     }
+
+                    unsigned int firstElementOffset = (attribs[i].mOffset / attribs[i].stride()) * outputElementSize;
+                    unsigned int startOffset = (instances == 0 || attribs[i].mDivisor == 0) ? start * outputElementSize : 0;
+                    if (streamOffset + firstElementOffset + startOffset < streamOffset)
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
+
+                    streamOffset += firstElementOffset + startOffset;
                 }
                 else
                 {
-                    outputElementSize = mStreamingBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0);
-                    streamOffset = mStreamingBuffer->storeVertexAttributes(attribs[i], currentValues[i], start, count, instances);
-                }
-
-                if (streamOffset == -1)
-                {
-                    return GL_OUT_OF_MEMORY;
+                    if (!mStreamingBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0, &outputElementSize) ||
+                        !mStreamingBuffer->storeVertexAttributes(attribs[i], currentValues[i], start, count, instances,
+                                                                 &streamOffset))
+                    {
+                        return GL_OUT_OF_MEMORY;
+                    }
                 }
 
                 translated[i].storage = directStorage ? storage : NULL;
@@ -229,8 +239,8 @@
                         return GL_OUT_OF_MEMORY;
                     }
 
-                    int streamOffset = buffer->storeVertexAttributes(attribs[i], currentValues[i], 0, 1, 0);
-                    if (streamOffset == -1)
+                    unsigned int streamOffset;
+                    if (!buffer->storeVertexAttributes(attribs[i], currentValues[i], 0, 1, 0, &streamOffset))
                     {
                         return GL_OUT_OF_MEMORY;
                     }