commit 2ad1c4905e80502362a53844278936dd0bb25704
author Jamie Madill <jmadill@chromium.org> Wed Dec 07 14:46:18 2016 -0500
committer Commit Bot <commit-bot@chromium.org> Thu Dec 08 21:36:54 2016 +0000
tree dc226a69011707898245d0e6636dc0dc05b366dd
parent 120040e2392e70cf34610459463733ba214979a9

D3D: Fix overflow in varying packing.

Also add a more robust set of unit tests for this internal class.
Because the GL spec allows for succeess when packing any set of
varyings, it's impossible to write negative end to end tests. This CL
also rewrites our disabled varying packing GLSL tests as unit tests.

BUG=angleproject:1296
BUG=angleproject:1638

Change-Id: I78153742517d5c72ddb13ff59dc44ddc4af42fc2
Reviewed-on: https://chromium-review.googlesource.com/415555
Commit-Queue: Jamie Madill <jmadill@chromium.org>
Reviewed-by: Geoff Lang <geofflang@chromium.org>

src/libANGLE/renderer/d3d/ProgramD3D.cpp

diff --git a/src/libANGLE/renderer/d3d/ProgramD3D.cpp b/src/libANGLE/renderer/d3d/ProgramD3D.cpp
index 4728e79..a1a67db 100644
--- a/src/libANGLE/renderer/d3d/ProgramD3D.cpp
+++ b/src/libANGLE/renderer/d3d/ProgramD3D.cpp
@@ -1490,8 +1490,8 @@
 
     // Map the varyings to the register file
     VaryingPacking varyingPacking(data.getCaps().maxVaryingVectors);
-    if (!varyingPacking.packVaryings(infoLog, packedVaryings,
-                                     mState.getTransformFeedbackVaryingNames()))
+    if (!varyingPacking.packUserVaryings(infoLog, packedVaryings,
+                                         mState.getTransformFeedbackVaryingNames()))
     {
         return false;
     }
@@ -1501,7 +1501,7 @@
     metadata.updatePackingBuiltins(SHADER_VERTEX, &varyingPacking);
     metadata.updatePackingBuiltins(SHADER_PIXEL, &varyingPacking);
 
-    if (static_cast<GLuint>(varyingPacking.getRegisterCount()) > data.getCaps().maxVaryingVectors)
+    if (!varyingPacking.validateBuiltins())
     {
         infoLog << "No varying registers left to support gl_FragCoord/gl_PointCoord";
         return false;