commit b155bbcbacceeee8bed049d26ed0ee62d9bfd208
author Jamie Madill <jmadill@chromium.org> Mon Jan 13 09:51:03 2014 -0500
committer Jamie Madill <jmadill@chromium.org> Tue Jan 21 20:54:10 2014 +0000
tree 4c0ddafff7602194407775fec34326f82de55d76
parent a50232fd52b34608bcd053072f636d12e0d8b7e5

Properly clamp stencil mask query values to max int.

We could overflow when casting from max unsigned int to integer, and
return a negative mask.

See dEQP functional.state_query.stencil_value_mask_getfloat

Change-Id: I59189a40b662ad9d8ded79cdfeded923b98f0b85
Reviewed-on: https://chromium-review.googlesource.com/181780
Reviewed-by: Geoff Lang <geofflang@chromium.org>
Reviewed-by: Nicolas Capens <nicolascapens@chromium.org>
Commit-Queue: Nicolas Capens <nicolascapens@chromium.org>
Tested-by: Jamie Madill <jmadill@chromium.org>

src/common/mathutil.h

diff --git a/src/common/mathutil.h b/src/common/mathutil.h
index b84d9b8..9b3ef8a 100644
--- a/src/common/mathutil.h
+++ b/src/common/mathutil.h
@@ -16,6 +16,9 @@
 #include <intrin.h>
 #endif
 
+#include <limits>
+#include <algorithm>
+
 namespace gl
 {
 
@@ -58,6 +61,11 @@
     return x;
 }
 
+inline int clampToInt(unsigned int x)
+{
+    return static_cast<int>(std::min(x, static_cast<unsigned int>(std::numeric_limits<int>::max())));
+}
+
 template<typename T, typename MIN, typename MAX>
 inline T clamp(T x, MIN min, MAX max)
 {

src/libGLESv2/Context.cpp

diff --git a/src/libGLESv2/Context.cpp b/src/libGLESv2/Context.cpp
index 53c26d5..e3188dd 100644
--- a/src/libGLESv2/Context.cpp
+++ b/src/libGLESv2/Context.cpp
@@ -1656,10 +1656,10 @@
       case GL_ACTIVE_TEXTURE:                           *params = (mState.activeSampler + GL_TEXTURE0);                 break;
       case GL_STENCIL_FUNC:                             *params = mState.depthStencil.stencilFunc;                      break;
       case GL_STENCIL_REF:                              *params = mState.stencilRef;                                    break;
-      case GL_STENCIL_VALUE_MASK:                       *params = mState.depthStencil.stencilMask;                      break;
+      case GL_STENCIL_VALUE_MASK:                       *params = clampToInt(mState.depthStencil.stencilMask);          break;
       case GL_STENCIL_BACK_FUNC:                        *params = mState.depthStencil.stencilBackFunc;                  break;
       case GL_STENCIL_BACK_REF:                         *params = mState.stencilBackRef;                                break;
-      case GL_STENCIL_BACK_VALUE_MASK:                  *params = mState.depthStencil.stencilBackMask;                  break;
+      case GL_STENCIL_BACK_VALUE_MASK:                  *params = clampToInt(mState.depthStencil.stencilBackMask);      break;
       case GL_STENCIL_FAIL:                             *params = mState.depthStencil.stencilFail;                      break;
       case GL_STENCIL_PASS_DEPTH_FAIL:                  *params = mState.depthStencil.stencilPassDepthFail;             break;
       case GL_STENCIL_PASS_DEPTH_PASS:                  *params = mState.depthStencil.stencilPassDepthPass;             break;
@@ -1673,8 +1673,8 @@
       case GL_BLEND_DST_ALPHA:                          *params = mState.blend.destBlendAlpha;                          break;
       case GL_BLEND_EQUATION_RGB:                       *params = mState.blend.blendEquationRGB;                        break;
       case GL_BLEND_EQUATION_ALPHA:                     *params = mState.blend.blendEquationAlpha;                      break;
-      case GL_STENCIL_WRITEMASK:                        *params = mState.depthStencil.stencilWritemask;                 break;
-      case GL_STENCIL_BACK_WRITEMASK:                   *params = mState.depthStencil.stencilBackWritemask;             break;
+      case GL_STENCIL_WRITEMASK:                        *params = clampToInt(mState.depthStencil.stencilWritemask);     break;
+      case GL_STENCIL_BACK_WRITEMASK:                   *params = clampToInt(mState.depthStencil.stencilBackWritemask); break;
       case GL_STENCIL_CLEAR_VALUE:                      *params = mState.stencilClearValue;                             break;
       case GL_SUBPIXEL_BITS:                            *params = 4;                                                    break;
       case GL_MAX_TEXTURE_SIZE:                         *params = getMaximum2DTextureDimension();                       break;