D3D11: Fix out-of-range access with robust access.
When using a vertex buffer with DYNAMIC usage, with robust buffer
access enabled, we would sometimes read out-of-bounds when using very
large values for the index range. An unchecked signed addition would
overflow and lead to reading a negative offset.
Fix this problem by keeping the value size_t whenever possible. Also do
clamped casts when converting to a smaller values.
Also adds a regression test.
Bug: chromium:842028
Change-Id: Ie630ac857c6acfc0bace849a03eebfbaa2fbe89a
Reviewed-on: https://chromium-review.googlesource.com/1055928
Commit-Queue: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Geoff Lang <geofflang@chromium.org>
diff --git a/src/libANGLE/renderer/d3d/VertexDataManager.cpp b/src/libANGLE/renderer/d3d/VertexDataManager.cpp
index f20386b..183c895 100644
--- a/src/libANGLE/renderer/d3d/VertexDataManager.cpp
+++ b/src/libANGLE/renderer/d3d/VertexDataManager.cpp
@@ -392,7 +392,7 @@
std::vector<TranslatedAttribute> *translatedAttribs,
const gl::AttributesMask &dynamicAttribsMask,
GLint start,
- GLsizei count,
+ size_t count,
GLsizei instances)
{
// Instantiating this class will ensure the streaming buffer is never left mapped.
@@ -434,7 +434,7 @@
const gl::Context *context,
const std::vector<TranslatedAttribute> &translatedAttribs,
const gl::AttributesMask &dynamicAttribsMask,
- GLsizei count)
+ size_t count)
{
for (auto attribIndex : dynamicAttribsMask)
{
@@ -445,16 +445,17 @@
gl::Buffer *buffer = binding.getBuffer().get();
if (buffer)
{
+ // Note: this multiplication can overflow. It should not be a security problem.
BufferD3D *bufferD3D = GetImplAs<BufferD3D>(buffer);
size_t typeSize = ComputeVertexAttributeTypeSize(*dynamicAttrib.attribute);
- bufferD3D->promoteStaticUsage(context, count * static_cast<int>(typeSize));
+ bufferD3D->promoteStaticUsage(context, count * typeSize);
}
}
}
gl::Error VertexDataManager::reserveSpaceForAttrib(const TranslatedAttribute &translatedAttrib,
GLint start,
- GLsizei count,
+ size_t count,
GLsizei instances) const
{
ASSERT(translatedAttrib.attribute && translatedAttrib.binding);
@@ -467,8 +468,8 @@
BufferD3D *bufferD3D = buffer ? GetImplAs<BufferD3D>(buffer) : nullptr;
ASSERT(!bufferD3D || bufferD3D->getStaticVertexBuffer(attrib, binding) == nullptr);
- size_t totalCount = gl::ComputeVertexBindingElementCount(
- binding.getDivisor(), static_cast<size_t>(count), static_cast<size_t>(instances));
+ size_t totalCount = gl::ComputeVertexBindingElementCount(binding.getDivisor(), count,
+ static_cast<size_t>(instances));
// TODO(jiajia.qin@intel.com): force the index buffer to clamp any out of range indices instead
// of invalid operation here.
if (bufferD3D)
@@ -486,15 +487,14 @@
return gl::InvalidOperation() << "Vertex buffer is not big enough for the draw call.";
}
}
- return mStreamingBuffer->reserveVertexSpace(attrib, binding, static_cast<GLsizei>(totalCount),
- instances);
+ return mStreamingBuffer->reserveVertexSpace(attrib, binding, totalCount, instances);
}
gl::Error VertexDataManager::storeDynamicAttrib(const gl::Context *context,
TranslatedAttribute *translated,
GLint start,
- GLsizei count,
- GLsizei instances)
+ size_t count,
+ GLsizei instances) const
{
ASSERT(translated->attribute && translated->binding);
const auto &attrib = *translated->attribute;
@@ -529,8 +529,8 @@
translated->storage = nullptr;
ANGLE_TRY_RESULT(mFactory->getVertexSpaceRequired(attrib, binding, 1, 0), translated->stride);
- size_t totalCount = gl::ComputeVertexBindingElementCount(
- binding.getDivisor(), static_cast<size_t>(count), static_cast<size_t>(instances));
+ size_t totalCount = gl::ComputeVertexBindingElementCount(binding.getDivisor(), count,
+ static_cast<size_t>(instances));
ANGLE_TRY(mStreamingBuffer->storeDynamicAttribute(
attrib, binding, translated->currentValueType, firstVertexIndex,