Updated VertexBuffer's getSpaceRequired and storeVertexAttributes methods to return bools and fixed some validation of parameters to prevent under and overflows.
TRAC #23643
Signed-off-by: Nicolas Capens
Signed-off-by: Shannon Woods
Author: Geoff Lang
diff --git a/src/libGLESv2/renderer/IndexBuffer.cpp b/src/libGLESv2/renderer/IndexBuffer.cpp
index e24583a..7fe2a3a 100644
--- a/src/libGLESv2/renderer/IndexBuffer.cpp
+++ b/src/libGLESv2/renderer/IndexBuffer.cpp
@@ -67,18 +67,30 @@
return mIndexBuffer->getSerial();
}
-int IndexBufferInterface::mapBuffer(unsigned int size, void** outMappedMemory)
+bool IndexBufferInterface::mapBuffer(unsigned int size, void** outMappedMemory, unsigned int *streamOffset)
{
- if (!mIndexBuffer->mapBuffer(mWritePosition, size, outMappedMemory))
+ // Protect against integer overflow
+ if (mWritePosition + size < mWritePosition)
{
- *outMappedMemory = NULL;
- return -1;
+ return false;
}
- int oldWritePos = static_cast<int>(mWritePosition);
- mWritePosition += size;
+ if (!mIndexBuffer->mapBuffer(mWritePosition, size, outMappedMemory))
+ {
+ if (outMappedMemory)
+ {
+ *outMappedMemory = NULL;
+ }
+ return false;
+ }
- return oldWritePos;
+ if (streamOffset)
+ {
+ *streamOffset = mWritePosition;
+ }
+
+ mWritePosition += size;
+ return true;
}
bool IndexBufferInterface::unmapBuffer()
diff --git a/src/libGLESv2/renderer/IndexBuffer.h b/src/libGLESv2/renderer/IndexBuffer.h
index 98fa5fe..6fb885a 100644
--- a/src/libGLESv2/renderer/IndexBuffer.h
+++ b/src/libGLESv2/renderer/IndexBuffer.h
@@ -59,7 +59,7 @@
unsigned int getSerial() const;
- int mapBuffer(unsigned int size, void** outMappedMemory);
+ bool mapBuffer(unsigned int size, void** outMappedMemory, unsigned int *streamOffset);
bool unmapBuffer();
IndexBuffer *getIndexBuffer() const;
diff --git a/src/libGLESv2/renderer/IndexBuffer11.cpp b/src/libGLESv2/renderer/IndexBuffer11.cpp
index d66509c..f364adc 100644
--- a/src/libGLESv2/renderer/IndexBuffer11.cpp
+++ b/src/libGLESv2/renderer/IndexBuffer11.cpp
@@ -67,7 +67,8 @@
{
if (mBuffer)
{
- if (offset + size > mBufferSize)
+ // Check for integer overflows and out-out-bounds map requests
+ if (offset + size < offset || offset + size > mBufferSize)
{
ERR("Index buffer map range is not inside the buffer.");
return false;
diff --git a/src/libGLESv2/renderer/IndexDataManager.cpp b/src/libGLESv2/renderer/IndexDataManager.cpp
index 5056bcb..666f493 100644
--- a/src/libGLESv2/renderer/IndexDataManager.cpp
+++ b/src/libGLESv2/renderer/IndexDataManager.cpp
@@ -132,7 +132,16 @@
default: UNREACHABLE(); alignedOffset = false;
}
- if (gl::GetTypeBytes(type) * count + offset > storage->getSize())
+ unsigned int typeSize = gl::GetTypeBytes(type);
+
+ // check for interger overflows and underflows
+ if (static_cast<unsigned int>(offset) > (std::numeric_limits<unsigned int>::max() / typeSize) ||
+ static_cast<unsigned int>(count) > ((std::numeric_limits<unsigned int>::max() / typeSize) - offset))
+ {
+ return GL_OUT_OF_MEMORY;
+ }
+
+ if (typeSize * static_cast<unsigned int>(count) + offset > storage->getSize())
{
return GL_INVALID_OPERATION;
}
@@ -146,7 +155,7 @@
IndexBufferInterface *indexBuffer = streamingBuffer;
bool directStorage = alignedOffset && storage && storage->supportsDirectBinding() &&
destinationIndexType == type;
- UINT streamOffset = 0;
+ unsigned int streamOffset = 0;
if (directStorage)
{
@@ -176,7 +185,7 @@
}
else
{
- int convertCount = count;
+ unsigned int convertCount = count;
if (staticBuffer)
{
@@ -198,12 +207,22 @@
return GL_INVALID_OPERATION;
}
- unsigned int bufferSizeRequired = convertCount * gl::GetTypeBytes(destinationIndexType);
- indexBuffer->reserveBufferSpace(bufferSizeRequired, type);
+ unsigned int indexTypeSize = gl::GetTypeBytes(destinationIndexType);
+ if (convertCount > std::numeric_limits<unsigned int>::max() / indexTypeSize)
+ {
+ ERR("Reserving %u indicies of %u bytes each exceeds the maximum buffer size.", convertCount, indexTypeSize);
+ return GL_OUT_OF_MEMORY;
+ }
+
+ unsigned int bufferSizeRequired = convertCount * indexTypeSize;
+ if (!indexBuffer->reserveBufferSpace(bufferSizeRequired, type))
+ {
+ ERR("Failed to reserve %u bytes in an index buffer.", bufferSizeRequired);
+ return GL_OUT_OF_MEMORY;
+ }
void* output = NULL;
- streamOffset = indexBuffer->mapBuffer(bufferSizeRequired, &output);
- if (streamOffset == -1 || output == NULL)
+ if (!indexBuffer->mapBuffer(bufferSizeRequired, &output, &streamOffset))
{
ERR("Failed to map index buffer.");
return GL_OUT_OF_MEMORY;
@@ -254,7 +273,7 @@
mCountingBuffer->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_SHORT);
void* mappedMemory = NULL;
- if (mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory) == -1 || mappedMemory == NULL)
+ if (!mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory, NULL))
{
ERR("Failed to map counting buffer.");
return NULL;
@@ -284,7 +303,7 @@
mCountingBuffer->reserveBufferSpace(spaceNeeded, GL_UNSIGNED_INT);
void* mappedMemory = NULL;
- if (mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory) == -1 || mappedMemory == NULL)
+ if (!mCountingBuffer->mapBuffer(spaceNeeded, &mappedMemory, NULL))
{
ERR("Failed to map counting buffer.");
return NULL;
diff --git a/src/libGLESv2/renderer/Renderer11.cpp b/src/libGLESv2/renderer/Renderer11.cpp
index fe62cab..dcf12b7 100644
--- a/src/libGLESv2/renderer/Renderer11.cpp
+++ b/src/libGLESv2/renderer/Renderer11.cpp
@@ -1178,15 +1178,15 @@
}
void* mappedMemory = NULL;
- int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
- if (offset == -1 || mappedMemory == NULL)
+ unsigned int offset;
+ if (!mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
{
ERR("Could not map index buffer for GL_LINE_LOOP.");
return gl::error(GL_OUT_OF_MEMORY);
}
unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
- unsigned int indexBufferOffset = static_cast<unsigned int>(offset);
+ unsigned int indexBufferOffset = offset;
switch (type)
{
@@ -1283,15 +1283,15 @@
}
void* mappedMemory = NULL;
- int offset = mTriangleFanIB->mapBuffer(spaceNeeded, &mappedMemory);
- if (offset == -1 || mappedMemory == NULL)
+ unsigned int offset;
+ if (!mTriangleFanIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
{
ERR("Could not map scratch index buffer for GL_TRIANGLE_FAN.");
return gl::error(GL_OUT_OF_MEMORY);
}
unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
- unsigned int indexBufferOffset = static_cast<unsigned int>(offset);
+ unsigned int indexBufferOffset = offset;
switch (type)
{
diff --git a/src/libGLESv2/renderer/Renderer9.cpp b/src/libGLESv2/renderer/Renderer9.cpp
index c7386c6..237986f 100644
--- a/src/libGLESv2/renderer/Renderer9.cpp
+++ b/src/libGLESv2/renderer/Renderer9.cpp
@@ -1427,7 +1427,7 @@
indices = static_cast<const GLubyte*>(storage->getData()) + offset;
}
- UINT startIndex = 0;
+ unsigned int startIndex = 0;
if (get32BitIndexSupport())
{
@@ -1461,14 +1461,14 @@
}
void* mappedMemory = NULL;
- int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
- if (offset == -1 || mappedMemory == NULL)
+ unsigned int offset = 0;
+ if (!mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
{
ERR("Could not map index buffer for GL_LINE_LOOP.");
return gl::error(GL_OUT_OF_MEMORY);
}
- startIndex = static_cast<UINT>(offset) / 4;
+ startIndex = static_cast<unsigned int>(offset) / 4;
unsigned int *data = reinterpret_cast<unsigned int*>(mappedMemory);
switch (type)
@@ -1542,14 +1542,14 @@
}
void* mappedMemory = NULL;
- int offset = mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory);
- if (offset == -1 || mappedMemory == NULL)
+ unsigned int offset;
+ if (mLineLoopIB->mapBuffer(spaceNeeded, &mappedMemory, &offset))
{
ERR("Could not map index buffer for GL_LINE_LOOP.");
return gl::error(GL_OUT_OF_MEMORY);
}
- startIndex = static_cast<UINT>(offset) / 2;
+ startIndex = static_cast<unsigned int>(offset) / 2;
unsigned short *data = reinterpret_cast<unsigned short*>(mappedMemory);
switch (type)
diff --git a/src/libGLESv2/renderer/VertexBuffer.cpp b/src/libGLESv2/renderer/VertexBuffer.cpp
index c8ea612..e0f135e 100644
--- a/src/libGLESv2/renderer/VertexBuffer.cpp
+++ b/src/libGLESv2/renderer/VertexBuffer.cpp
@@ -87,29 +87,48 @@
return mVertexBuffer->discard();
}
-int VertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
- GLint start, GLsizei count, GLsizei instances)
+bool VertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
+ GLint start, GLsizei count, GLsizei instances, unsigned int *outStreamOffset)
{
+ unsigned int spaceRequired;
+ if (!mVertexBuffer->getSpaceRequired(attrib, count, instances, &spaceRequired))
+ {
+ return false;
+ }
+
+ if (mWritePosition + spaceRequired < mWritePosition)
+ {
+ return false;
+ }
+
if (!reserveSpace(mReservedSpace))
{
- return -1;
+ return false;
}
mReservedSpace = 0;
if (!mVertexBuffer->storeVertexAttributes(attrib, currentValue, start, count, instances, mWritePosition))
{
- return -1;
+ return false;
}
- int oldWritePos = static_cast<int>(mWritePosition);
- mWritePosition += mVertexBuffer->getSpaceRequired(attrib, count, instances);
+ if (outStreamOffset)
+ {
+ *outStreamOffset = mWritePosition;
+ }
- return oldWritePos;
+ mWritePosition += spaceRequired;
+
+ return true;
}
bool VertexBufferInterface::reserveVertexSpace(const gl::VertexAttribute &attribute, GLsizei count, GLsizei instances)
{
- unsigned int requiredSpace = mVertexBuffer->getSpaceRequired(attribute, count, instances);
+ unsigned int requiredSpace;
+ if (!mVertexBuffer->getSpaceRequired(attribute, count, instances, &requiredSpace))
+ {
+ return false;
+ }
// Protect against integer overflow
if (mReservedSpace + requiredSpace < mReservedSpace)
@@ -165,7 +184,7 @@
{
}
-int StaticVertexBufferInterface::lookupAttribute(const gl::VertexAttribute &attribute)
+bool StaticVertexBufferInterface::lookupAttribute(const gl::VertexAttribute &attribute, unsigned int *outStreamOffset)
{
for (unsigned int element = 0; element < mCache.size(); element++)
{
@@ -177,12 +196,16 @@
{
if (mCache[element].attributeOffset == attribute.mOffset % attribute.stride())
{
- return mCache[element].streamOffset;
+ if (outStreamOffset)
+ {
+ *outStreamOffset = mCache[element].streamOffset;
+ }
+ return true;
}
}
}
- return -1;
+ return false;
}
bool StaticVertexBufferInterface::reserveSpace(unsigned int size)
@@ -204,14 +227,27 @@
}
}
-int StaticVertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
- GLint start, GLsizei count, GLsizei instances)
+bool StaticVertexBufferInterface::storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
+ GLint start, GLsizei count, GLsizei instances, unsigned int *outStreamOffset)
{
- int attributeOffset = attrib.mOffset % attrib.stride();
- VertexElement element = { attrib.mType, attrib.mSize, attrib.stride(), attrib.mNormalized, attrib.mPureInteger, attributeOffset, getWritePosition() };
- mCache.push_back(element);
+ unsigned int streamOffset;
+ if (VertexBufferInterface::storeVertexAttributes(attrib, currentValue, start, count, instances, &streamOffset))
+ {
+ int attributeOffset = attrib.mOffset % attrib.stride();
+ VertexElement element = { attrib.mType, attrib.mSize, attrib.stride(), attrib.mNormalized, attrib.mPureInteger, attributeOffset, streamOffset };
+ mCache.push_back(element);
- return VertexBufferInterface::storeVertexAttributes(attrib, currentValue, start, count, instances);
+ if (outStreamOffset)
+ {
+ *outStreamOffset = streamOffset;
+ }
+
+ return true;
+ }
+ else
+ {
+ return false;
+ }
}
}
diff --git a/src/libGLESv2/renderer/VertexBuffer.h b/src/libGLESv2/renderer/VertexBuffer.h
index aecad58..ffeb8ab 100644
--- a/src/libGLESv2/renderer/VertexBuffer.h
+++ b/src/libGLESv2/renderer/VertexBuffer.h
@@ -32,8 +32,8 @@
virtual bool storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
GLint start, GLsizei count, GLsizei instances, unsigned int offset) = 0;
- virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
- GLsizei instances) const = 0;
+ virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+ unsigned int *outSpaceRequired) const = 0;
virtual bool requiresConversion(const gl::VertexAttribute &attrib) const = 0;
virtual bool requiresConversion(const gl::VertexAttribCurrentValueData ¤tValue) const = 0;
@@ -66,8 +66,8 @@
unsigned int getSerial() const;
- virtual int storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
- GLint start, GLsizei count, GLsizei instances);
+ virtual bool storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
+ GLint start, GLsizei count, GLsizei instances, unsigned int *outStreamOffset);
VertexBuffer* getVertexBuffer() const;
@@ -109,11 +109,10 @@
explicit StaticVertexBufferInterface(rx::Renderer *renderer);
~StaticVertexBufferInterface();
- int storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
- GLint start, GLsizei count, GLsizei instances);
+ bool storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
+ GLint start, GLsizei count, GLsizei instances, unsigned int *outStreamOffset);
- // Returns the offset into the vertex buffer, or -1 if not found
- int lookupAttribute(const gl::VertexAttribute &attribute);
+ bool lookupAttribute(const gl::VertexAttribute &attribute, unsigned int* outStreamFffset);
protected:
bool reserveSpace(unsigned int size);
diff --git a/src/libGLESv2/renderer/VertexBuffer11.cpp b/src/libGLESv2/renderer/VertexBuffer11.cpp
index 268cebb..c605921 100644
--- a/src/libGLESv2/renderer/VertexBuffer11.cpp
+++ b/src/libGLESv2/renderer/VertexBuffer11.cpp
@@ -166,26 +166,52 @@
}
}
-unsigned int VertexBuffer11::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
- GLsizei instances) const
+bool VertexBuffer11::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count,
+ GLsizei instances, unsigned int *outSpaceRequired) const
{
+ unsigned int elementCount = 0;
if (attrib.mArrayEnabled)
{
unsigned int elementSize = getVertexConversion(attrib).outputElementSize;
if (instances == 0 || attrib.mDivisor == 0)
{
- return elementSize * count;
+ elementCount = count;
}
else
{
- return elementSize * ((instances + attrib.mDivisor - 1) / attrib.mDivisor);
+ if (static_cast<unsigned int>(instances) < std::numeric_limits<unsigned int>::max() - (attrib.mDivisor - 1))
+ {
+ // Round up
+ elementCount = (static_cast<unsigned int>(instances) + (attrib.mDivisor - 1)) / attrib.mDivisor;
+ }
+ else
+ {
+ elementCount = instances / attrib.mDivisor;
+ }
+ }
+
+ if (elementSize <= std::numeric_limits<unsigned int>::max() / elementCount)
+ {
+ if (outSpaceRequired)
+ {
+ *outSpaceRequired = elementSize * elementCount;
+ }
+ return true;
+ }
+ else
+ {
+ return false;
}
}
else
{
- unsigned int elementSize = 4;
- return elementSize * 4;
+ const unsigned int elementSize = 4;
+ if (outSpaceRequired)
+ {
+ *outSpaceRequired = elementSize * 4;
+ }
+ return true;
}
}
diff --git a/src/libGLESv2/renderer/VertexBuffer11.h b/src/libGLESv2/renderer/VertexBuffer11.h
index 66f7f61..b68013a 100644
--- a/src/libGLESv2/renderer/VertexBuffer11.h
+++ b/src/libGLESv2/renderer/VertexBuffer11.h
@@ -28,7 +28,8 @@
virtual bool storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
GLint start, GLsizei count, GLsizei instances, unsigned int offset);
- virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const;
+ virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+ unsigned int *outSpaceRequired) const;
virtual bool requiresConversion(const gl::VertexAttribute &attrib) const;
virtual bool requiresConversion(const gl::VertexAttribCurrentValueData ¤tValue) const;
diff --git a/src/libGLESv2/renderer/VertexBuffer9.cpp b/src/libGLESv2/renderer/VertexBuffer9.cpp
index 4dc6ac6..4173af9 100644
--- a/src/libGLESv2/renderer/VertexBuffer9.cpp
+++ b/src/libGLESv2/renderer/VertexBuffer9.cpp
@@ -87,7 +87,14 @@
DWORD lockFlags = mDynamicUsage ? D3DLOCK_NOOVERWRITE : 0;
void *mapPtr = NULL;
- HRESULT result = mVertexBuffer->Lock(offset, spaceRequired(attrib, count, instances), &mapPtr, lockFlags);
+
+ unsigned int mapSize;
+ if (!spaceRequired(attrib, count, instances, &mapSize))
+ {
+ return false;
+ }
+
+ HRESULT result = mVertexBuffer->Lock(offset, mapSize, &mapPtr, lockFlags);
if (FAILED(result))
{
@@ -138,9 +145,10 @@
}
}
-unsigned int VertexBuffer9::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const
+bool VertexBuffer9::getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances,
+ unsigned int *outSpaceRequired) const
{
- return spaceRequired(attrib, count, instances);
+ return spaceRequired(attrib, count, instances, outSpaceRequired);
}
bool VertexBuffer9::requiresConversion(const gl::VertexAttribute &attrib) const
@@ -453,24 +461,52 @@
return mFormatConverters[typeIndex(GL_FLOAT)][0][3];
}
-unsigned int VertexBuffer9::spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances)
+bool VertexBuffer9::spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances,
+ unsigned int *outSpaceRequired)
{
unsigned int elementSize = formatConverter(attrib).outputElementSize;
if (attrib.mArrayEnabled)
{
+ unsigned int elementCount = 0;
if (instances == 0 || attrib.mDivisor == 0)
{
- return elementSize * count;
+ elementCount = count;
}
else
{
- return elementSize * ((instances + attrib.mDivisor - 1) / attrib.mDivisor);
+ if (static_cast<unsigned int>(instances) < std::numeric_limits<unsigned int>::max() - (attrib.mDivisor - 1))
+ {
+ // Round up
+ elementCount = (static_cast<unsigned int>(instances) + (attrib.mDivisor - 1)) / attrib.mDivisor;
+ }
+ else
+ {
+ elementCount = static_cast<unsigned int>(instances) / attrib.mDivisor;
+ }
+ }
+
+ if (elementSize <= std::numeric_limits<unsigned int>::max() / elementCount)
+ {
+ if (outSpaceRequired)
+ {
+ *outSpaceRequired = elementSize * elementCount;
+ }
+ return true;
+ }
+ else
+ {
+ return false;
}
}
else
{
- return elementSize * 4;
+ const unsigned int elementSize = 4;
+ if (outSpaceRequired)
+ {
+ *outSpaceRequired = elementSize * 4;
+ }
+ return true;
}
}
diff --git a/src/libGLESv2/renderer/VertexBuffer9.h b/src/libGLESv2/renderer/VertexBuffer9.h
index 73708b4..2bdd906 100644
--- a/src/libGLESv2/renderer/VertexBuffer9.h
+++ b/src/libGLESv2/renderer/VertexBuffer9.h
@@ -28,7 +28,7 @@
virtual bool storeVertexAttributes(const gl::VertexAttribute &attrib, const gl::VertexAttribCurrentValueData ¤tValue,
GLint start, GLsizei count, GLsizei instances, unsigned int offset);
- virtual unsigned int getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances) const;
+ virtual bool getSpaceRequired(const gl::VertexAttribute &attrib, GLsizei count, GLsizei instances, unsigned int *outSpaceRequired) const;
virtual bool requiresConversion(const gl::VertexAttribute &attrib) const;
virtual bool requiresConversion(const gl::VertexAttribCurrentValueData ¤tValue) const;
@@ -82,7 +82,8 @@
static const FormatConverter &formatConverter(const gl::VertexAttribute &attribute);
static const FormatConverter &getCurrentValueFormatConverter();
- static unsigned int spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances);
+ static bool spaceRequired(const gl::VertexAttribute &attrib, std::size_t count, GLsizei instances,
+ unsigned int *outSpaceRequired);
};
}
diff --git a/src/libGLESv2/renderer/VertexDataManager.cpp b/src/libGLESv2/renderer/VertexDataManager.cpp
index 4a57d91..9e663e2 100644
--- a/src/libGLESv2/renderer/VertexDataManager.cpp
+++ b/src/libGLESv2/renderer/VertexDataManager.cpp
@@ -26,9 +26,15 @@
namespace rx
{
-static int ElementsInBuffer(const gl::VertexAttribute &attribute, int size)
+static int ElementsInBuffer(const gl::VertexAttribute &attribute, unsigned int size)
{
- int stride = attribute.stride();
+ // Size cannot be larger than a GLsizei
+ if (size > static_cast<unsigned int>(std::numeric_limits<int>::max()))
+ {
+ size = static_cast<unsigned int>(std::numeric_limits<int>::max());
+ }
+
+ GLsizei stride = attribute.stride();
return (size - attribute.mOffset % stride + (stride - attribute.typeSize())) / stride;
}
@@ -97,7 +103,7 @@
gl::Buffer *buffer = attribs[i].mBoundBuffer.get();
StaticVertexBufferInterface *staticBuffer = buffer ? buffer->getStaticVertexBuffer() : NULL;
- if (staticBuffer && staticBuffer->getBufferSize() > 0 && staticBuffer->lookupAttribute(attribs[i]) == -1 &&
+ if (staticBuffer && staticBuffer->getBufferSize() > 0 && !staticBuffer->lookupAttribute(attribs[i], NULL) &&
!DirectStoragePossible(staticBuffer, attribs[i], currentValues[i]))
{
buffer->invalidateStaticData();
@@ -160,7 +166,7 @@
BufferStorage *storage = buffer ? buffer->getStorage() : NULL;
bool directStorage = DirectStoragePossible(vertexBuffer, attribs[i], currentValues[i]);
- std::size_t streamOffset = -1;
+ unsigned int streamOffset = 0;
unsigned int outputElementSize = 0;
if (directStorage)
@@ -170,37 +176,41 @@
}
else if (staticBuffer)
{
- streamOffset = staticBuffer->lookupAttribute(attribs[i]);
- outputElementSize = staticBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0);
+ if (!staticBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0, &outputElementSize))
+ {
+ return GL_OUT_OF_MEMORY;
+ }
- if (streamOffset == -1)
+ if (!staticBuffer->lookupAttribute(attribs[i], &streamOffset))
{
// Convert the entire buffer
int totalCount = ElementsInBuffer(attribs[i], storage->getSize());
int startIndex = attribs[i].mOffset / attribs[i].stride();
- streamOffset = staticBuffer->storeVertexAttributes(attribs[i], currentValues[i], -startIndex, totalCount, 0);
- }
-
- if (streamOffset != -1)
- {
- streamOffset += (attribs[i].mOffset / attribs[i].stride()) * outputElementSize;
-
- if (instances == 0 || attribs[i].mDivisor == 0)
+ if (!staticBuffer->storeVertexAttributes(attribs[i], currentValues[i], -startIndex, totalCount,
+ 0, &streamOffset))
{
- streamOffset += start * outputElementSize;
+ return GL_OUT_OF_MEMORY;
}
}
+
+ unsigned int firstElementOffset = (attribs[i].mOffset / attribs[i].stride()) * outputElementSize;
+ unsigned int startOffset = (instances == 0 || attribs[i].mDivisor == 0) ? start * outputElementSize : 0;
+ if (streamOffset + firstElementOffset + startOffset < streamOffset)
+ {
+ return GL_OUT_OF_MEMORY;
+ }
+
+ streamOffset += firstElementOffset + startOffset;
}
else
{
- outputElementSize = mStreamingBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0);
- streamOffset = mStreamingBuffer->storeVertexAttributes(attribs[i], currentValues[i], start, count, instances);
- }
-
- if (streamOffset == -1)
- {
- return GL_OUT_OF_MEMORY;
+ if (!mStreamingBuffer->getVertexBuffer()->getSpaceRequired(attribs[i], 1, 0, &outputElementSize) ||
+ !mStreamingBuffer->storeVertexAttributes(attribs[i], currentValues[i], start, count, instances,
+ &streamOffset))
+ {
+ return GL_OUT_OF_MEMORY;
+ }
}
translated[i].storage = directStorage ? storage : NULL;
@@ -229,8 +239,8 @@
return GL_OUT_OF_MEMORY;
}
- int streamOffset = buffer->storeVertexAttributes(attribs[i], currentValues[i], 0, 1, 0);
- if (streamOffset == -1)
+ unsigned int streamOffset;
+ if (!buffer->storeVertexAttributes(attribs[i], currentValues[i], 0, 1, 0, &streamOffset))
{
return GL_OUT_OF_MEMORY;
}
diff --git a/src/libGLESv2/renderer/VertexDataManager.h b/src/libGLESv2/renderer/VertexDataManager.h
index eadf584..1a69245 100644
--- a/src/libGLESv2/renderer/VertexDataManager.h
+++ b/src/libGLESv2/renderer/VertexDataManager.h
@@ -34,8 +34,8 @@
const gl::VertexAttribute *attribute;
GLenum currentValueType;
- UINT offset;
- UINT stride; // 0 means not to advance the read pointer at all
+ unsigned int offset;
+ unsigned int stride; // 0 means not to advance the read pointer at all
VertexBuffer *vertexBuffer;
BufferStorage *storage;