commit ece7c5a848df6b64e361184bbf3a9739b209e40c
author Corentin Wallez <cwallez@chromium.org> Wed Sep 21 15:28:23 2016 -0400
committer Commit Bot <commit-bot@chromium.org> Tue Sep 27 15:58:00 2016 +0000
tree 7cbe5217698549d7d20517a0e29bdf4f7b59da7b
parent 2cacb7783116bdd94b2dd70315418a62e035bd1d

Add validation for the pack buffer in ReadPixels

BUG=angleproject:1512

Change-Id: Ia6bac628c35f04bc5d3adfde1569902475519698
Reviewed-on: https://chromium-review.googlesource.com/387668
Commit-Queue: Corentin Wallez <cwallez@chromium.org>
Reviewed-by: Geoff Lang <geofflang@chromium.org>
Reviewed-by: Jamie Madill <jmadill@chromium.org>

src/libANGLE/formatutils.cpp

diff --git a/src/libANGLE/formatutils.cpp b/src/libANGLE/formatutils.cpp
index 780f2f7..4222d72 100644
--- a/src/libANGLE/formatutils.cpp
+++ b/src/libANGLE/formatutils.cpp
@@ -939,6 +939,32 @@
     return skipBytes.ValueOrDie();
 }
 
+gl::ErrorOrResult<GLuint> InternalFormat::computePackSize(GLenum formatType,
+                                                          const gl::Extents &size,
+                                                          const gl::PixelPackState &pack) const
+{
+    ASSERT(!compressed);
+
+    if (size.height == 0)
+    {
+        return 0;
+    }
+
+    CheckedNumeric<GLuint> rowPitch;
+    ANGLE_TRY_RESULT(computeRowPitch(formatType, size.width, pack.alignment, pack.rowLength),
+                     rowPitch);
+
+    CheckedNumeric<GLuint> heightMinusOne = size.height - 1;
+    CheckedNumeric<GLuint> bytes          = computePixelBytes(formatType);
+
+    CheckedNumeric<GLuint> totalSize = heightMinusOne * rowPitch;
+    totalSize += size.width * bytes;
+
+    ANGLE_TRY_CHECKED_MATH(totalSize);
+
+    return totalSize.ValueOrDie();
+}
+
 gl::ErrorOrResult<GLuint> InternalFormat::computeUnpackSize(
     GLenum formatType,
     const gl::Extents &size,
@@ -950,6 +976,11 @@
         return computeCompressedImageSize(formatType, size);
     }
 
+    if (size.height == 0 || size.depth == 0)
+    {
+        return 0;
+    }
+
     CheckedNumeric<GLuint> rowPitch;
     CheckedNumeric<GLuint> depthPitch;
     ANGLE_TRY_RESULT(computeRowPitch(formatType, size.width, unpack.alignment, unpack.rowLength),
@@ -971,6 +1002,26 @@
     return totalSize.ValueOrDie();
 }
 
+gl::ErrorOrResult<GLuint> InternalFormat::computePackEndByte(GLenum formatType,
+                                                             const gl::Extents &size,
+                                                             const gl::PixelPackState &pack) const
+{
+    GLuint rowPitch;
+    CheckedNumeric<GLuint> checkedSkipBytes;
+    CheckedNumeric<GLuint> checkedCopyBytes;
+
+    ANGLE_TRY_RESULT(computeRowPitch(formatType, size.width, pack.alignment, pack.rowLength),
+                     rowPitch);
+    ANGLE_TRY_RESULT(computeSkipBytes(rowPitch, 0, 0, pack.skipRows, pack.skipPixels, false),
+                     checkedSkipBytes);
+    ANGLE_TRY_RESULT(computePackSize(formatType, size, pack), checkedCopyBytes);
+
+    CheckedNumeric<GLuint> endByte = checkedCopyBytes + checkedSkipBytes;
+
+    ANGLE_TRY_CHECKED_MATH(endByte);
+    return endByte.ValueOrDie();
+}
+
 gl::ErrorOrResult<GLuint> InternalFormat::computeUnpackEndByte(GLenum formatType,
                                                                const gl::Extents &size,
                                                                const gl::PixelUnpackState &unpack,