Renderer classes now set their blend states to mask out channels that do not exist in the render target.

Change-Id: Ia49bf8de07bbdfa31275ec9835de28adc3717485
Reviewed-on: https://chromium-review.googlesource.com/176855
Reviewed-by: Shannon Woods <shannonwoods@chromium.org>
Commit-Queue: Shannon Woods <shannonwoods@chromium.org>
Tested-by: Shannon Woods <shannonwoods@chromium.org>
diff --git a/src/libGLESv2/Context.cpp b/src/libGLESv2/Context.cpp
index 633805d..db2ddb3 100644
--- a/src/libGLESv2/Context.cpp
+++ b/src/libGLESv2/Context.cpp
@@ -2340,7 +2340,7 @@
     {
         mask = 0xFFFFFFFF;
     }
-    mRenderer->setBlendState(mState.blend, mState.blendColor, mask);
+    mRenderer->setBlendState(framebufferObject, mState.blend, mState.blendColor, mask);
 
     mRenderer->setDepthStencilState(mState.depthStencil, mState.stencilRef, mState.stencilBackRef,
                                     mState.rasterizer.frontFace == GL_CCW);
diff --git a/src/libGLESv2/renderer/Renderer.h b/src/libGLESv2/renderer/Renderer.h
index da2b9f2..912fe90 100644
--- a/src/libGLESv2/renderer/Renderer.h
+++ b/src/libGLESv2/renderer/Renderer.h
@@ -125,7 +125,7 @@
     virtual bool setUniformBuffers(const gl::Buffer *vertexUniformBuffers[], const gl::Buffer *fragmentUniformBuffers[]) = 0;
 
     virtual void setRasterizerState(const gl::RasterizerState &rasterState) = 0;
-    virtual void setBlendState(const gl::BlendState &blendState, const gl::ColorF &blendColor,
+    virtual void setBlendState(gl::Framebuffer *framebuffer, const gl::BlendState &blendState, const gl::ColorF &blendColor,
                                unsigned int sampleMask) = 0;
     virtual void setDepthStencilState(const gl::DepthStencilState &depthStencilState, int stencilRef,
                                       int stencilBackRef, bool frontFaceCCW) = 0;
diff --git a/src/libGLESv2/renderer/d3d11/RenderStateCache.cpp b/src/libGLESv2/renderer/d3d11/RenderStateCache.cpp
index 0a3df6b..45ce93f 100644
--- a/src/libGLESv2/renderer/d3d11/RenderStateCache.cpp
+++ b/src/libGLESv2/renderer/d3d11/RenderStateCache.cpp
@@ -10,6 +10,9 @@
 
 #include "libGLESv2/renderer/d3d11/RenderStateCache.h"
 #include "libGLESv2/renderer/d3d11/renderer11_utils.h"
+#include "libGLESv2/renderer/Renderer.h"
+#include "libGLESv2/Framebuffer.h"
+#include "libGLESv2/Renderbuffer.h"
 
 #include "common/debug.h"
 #include "third_party/murmurhash/MurmurHash3.h"
@@ -62,7 +65,7 @@
     ClearStateMap(mSamplerStateCache);
 }
 
-std::size_t RenderStateCache::hashBlendState(const gl::BlendState &blendState)
+std::size_t RenderStateCache::hashBlendState(const BlendStateKey &blendState)
 {
     static const unsigned int seed = 0xABCDEF98;
 
@@ -71,12 +74,12 @@
     return hash;
 }
 
-bool RenderStateCache::compareBlendStates(const gl::BlendState &a, const gl::BlendState &b)
+bool RenderStateCache::compareBlendStates(const BlendStateKey &a, const BlendStateKey &b)
 {
     return memcmp(&a, &b, sizeof(gl::BlendState)) == 0;
 }
 
-ID3D11BlendState *RenderStateCache::getBlendState(const gl::BlendState &blendState)
+ID3D11BlendState *RenderStateCache::getBlendState(const gl::Framebuffer *framebuffer, const gl::BlendState &blendState)
 {
     if (!mDevice)
     {
@@ -84,7 +87,35 @@
         return NULL;
     }
 
-    BlendStateMap::iterator i = mBlendStateCache.find(blendState);
+    bool mrt = false;
+
+    BlendStateKey key = { 0 };
+    key.blendState = blendState;
+    for (unsigned int i = 0; i < D3D11_SIMULTANEOUS_RENDER_TARGET_COUNT; i++)
+    {
+        gl::Renderbuffer *renderBuffer = framebuffer->getColorbuffer(i);
+        if (renderBuffer)
+        {
+            if (i > 0)
+            {
+                mrt = true;
+            }
+
+            key.rtChannels[i][0] = renderBuffer->getRedSize()   > 0;
+            key.rtChannels[i][1] = renderBuffer->getGreenSize() > 0;
+            key.rtChannels[i][2] = renderBuffer->getBlueSize()  > 0;
+            key.rtChannels[i][3] = renderBuffer->getAlphaSize() > 0;
+        }
+        else
+        {
+            key.rtChannels[i][0] = false;
+            key.rtChannels[i][1] = false;
+            key.rtChannels[i][2] = false;
+            key.rtChannels[i][3] = false;
+        }
+    }
+
+    BlendStateMap::iterator i = mBlendStateCache.find(key);
     if (i != mBlendStateCache.end())
     {
         BlendStateCounterPair &state = i->second;
@@ -113,7 +144,7 @@
         // Create a new blend state and insert it into the cache
         D3D11_BLEND_DESC blendDesc = { 0 };
         blendDesc.AlphaToCoverageEnable = blendState.sampleAlphaToCoverage;
-        blendDesc.IndependentBlendEnable = FALSE;
+        blendDesc.IndependentBlendEnable = mrt ? TRUE : FALSE;
 
         for (unsigned int i = 0; i < D3D11_SIMULTANEOUS_RENDER_TARGET_COUNT; i++)
         {
@@ -131,10 +162,10 @@
                 rtBlend.BlendOpAlpha = gl_d3d11::ConvertBlendOp(blendState.blendEquationAlpha);
             }
 
-            rtBlend.RenderTargetWriteMask = gl_d3d11::ConvertColorMask(blendState.colorMaskRed,
-                                                                       blendState.colorMaskGreen,
-                                                                       blendState.colorMaskBlue,
-                                                                       blendState.colorMaskAlpha);
+            rtBlend.RenderTargetWriteMask = gl_d3d11::ConvertColorMask(key.rtChannels[i][0] && blendState.colorMaskRed,
+                                                                       key.rtChannels[i][1] && blendState.colorMaskGreen,
+                                                                       key.rtChannels[i][2] && blendState.colorMaskBlue,
+                                                                       key.rtChannels[i][3] && blendState.colorMaskAlpha);
         }
 
         ID3D11BlendState *dx11BlendState = NULL;
@@ -145,7 +176,7 @@
             return NULL;
         }
 
-        mBlendStateCache.insert(std::make_pair(blendState, std::make_pair(dx11BlendState, mCounter++)));
+        mBlendStateCache.insert(std::make_pair(key, std::make_pair(dx11BlendState, mCounter++)));
 
         return dx11BlendState;
     }
@@ -174,7 +205,7 @@
         return NULL;
     }
 
-    RasterizerStateKey key;
+    RasterizerStateKey key = { 0 };
     key.rasterizerState = rasterState;
     key.scissorEnabled = scissorEnabled;
     key.depthSize = depthSize;
diff --git a/src/libGLESv2/renderer/d3d11/RenderStateCache.h b/src/libGLESv2/renderer/d3d11/RenderStateCache.h
index f8b5111..660fe7a 100644
--- a/src/libGLESv2/renderer/d3d11/RenderStateCache.h
+++ b/src/libGLESv2/renderer/d3d11/RenderStateCache.h
@@ -13,6 +13,11 @@
 #include "libGLESv2/angletypes.h"
 #include "common/angleutils.h"
 
+namespace gl
+{
+class Framebuffer;
+}
+
 namespace rx
 {
 
@@ -25,8 +30,7 @@
     void initialize(ID3D11Device *device);
     void clear();
 
-    // Increments refcount on the returned blend state, Release() must be called.
-    ID3D11BlendState *getBlendState(const gl::BlendState &blendState);
+    ID3D11BlendState *getBlendState(const gl::Framebuffer *framebuffer, const gl::BlendState &blendState);
     ID3D11RasterizerState *getRasterizerState(const gl::RasterizerState &rasterState,
                                               bool scissorEnabled, unsigned int depthSize);
     ID3D11DepthStencilState *getDepthStencilState(const gl::DepthStencilState &dsState);
@@ -38,14 +42,19 @@
     unsigned long long mCounter;
 
     // Blend state cache
-    static std::size_t hashBlendState(const gl::BlendState &blendState);
-    static bool compareBlendStates(const gl::BlendState &a, const gl::BlendState &b);
+    struct BlendStateKey
+    {
+        gl::BlendState blendState;
+        bool rtChannels[D3D11_SIMULTANEOUS_RENDER_TARGET_COUNT][4];
+    };
+    static std::size_t hashBlendState(const BlendStateKey &blendState);
+    static bool compareBlendStates(const BlendStateKey &a, const BlendStateKey &b);
     static const unsigned int kMaxBlendStates;
 
-    typedef std::size_t (*BlendStateHashFunction)(const gl::BlendState &);
-    typedef bool (*BlendStateEqualityFunction)(const gl::BlendState &, const gl::BlendState &);
+    typedef std::size_t (*BlendStateHashFunction)(const BlendStateKey &);
+    typedef bool (*BlendStateEqualityFunction)(const BlendStateKey &, const BlendStateKey &);
     typedef std::pair<ID3D11BlendState*, unsigned long long> BlendStateCounterPair;
-    typedef std::unordered_map<gl::BlendState, BlendStateCounterPair, BlendStateHashFunction, BlendStateEqualityFunction> BlendStateMap;
+    typedef std::unordered_map<BlendStateKey, BlendStateCounterPair, BlendStateHashFunction, BlendStateEqualityFunction> BlendStateMap;
     BlendStateMap mBlendStateCache;
 
     // Rasterizer state cache
diff --git a/src/libGLESv2/renderer/d3d11/Renderer11.cpp b/src/libGLESv2/renderer/d3d11/Renderer11.cpp
index 40b6ee7..652d1bc 100644
--- a/src/libGLESv2/renderer/d3d11/Renderer11.cpp
+++ b/src/libGLESv2/renderer/d3d11/Renderer11.cpp
@@ -723,7 +723,7 @@
     mForceSetRasterState = false;
 }
 
-void Renderer11::setBlendState(const gl::BlendState &blendState, const gl::ColorF &blendColor,
+void Renderer11::setBlendState(gl::Framebuffer *framebuffer, const gl::BlendState &blendState, const gl::ColorF &blendColor,
                                unsigned int sampleMask)
 {
     if (mForceSetBlendState ||
@@ -731,7 +731,7 @@
         memcmp(&blendColor, &mCurBlendColor, sizeof(gl::ColorF)) != 0 ||
         sampleMask != mCurSampleMask)
     {
-        ID3D11BlendState *dxBlendState = mStateCache.getBlendState(blendState);
+        ID3D11BlendState *dxBlendState = mStateCache.getBlendState(framebuffer, blendState);
         if (!dxBlendState)
         {
             ERR("NULL blend state returned by RenderStateCache::getBlendState, setting the default "
@@ -1081,6 +1081,7 @@
         mRenderTargetDesc.format = renderTargetFormat;
         mForceSetViewport = true;
         mForceSetScissor = true;
+        mForceSetBlendState = true;
 
         if (!mDepthStencilInitialized || depthSize != mCurDepthSize)
         {
diff --git a/src/libGLESv2/renderer/d3d11/Renderer11.h b/src/libGLESv2/renderer/d3d11/Renderer11.h
index d33e5af..4169bbe 100644
--- a/src/libGLESv2/renderer/d3d11/Renderer11.h
+++ b/src/libGLESv2/renderer/d3d11/Renderer11.h
@@ -64,7 +64,7 @@
     virtual bool setUniformBuffers(const gl::Buffer *vertexUniformBuffers[], const gl::Buffer *fragmentUniformBuffers[]);
 
     virtual void setRasterizerState(const gl::RasterizerState &rasterState);
-    virtual void setBlendState(const gl::BlendState &blendState, const gl::ColorF &blendColor,
+    virtual void setBlendState(gl::Framebuffer *framebuffer, const gl::BlendState &blendState, const gl::ColorF &blendColor,
                                unsigned int sampleMask);
     virtual void setDepthStencilState(const gl::DepthStencilState &depthStencilState, int stencilRef,
                                       int stencilBackRef, bool frontFaceCCW);
diff --git a/src/libGLESv2/renderer/d3d9/Renderer9.cpp b/src/libGLESv2/renderer/d3d9/Renderer9.cpp
index 294e6fe..5633564 100644
--- a/src/libGLESv2/renderer/d3d9/Renderer9.cpp
+++ b/src/libGLESv2/renderer/d3d9/Renderer9.cpp
@@ -872,7 +872,8 @@
     mForceSetRasterState = false;
 }
 
-void Renderer9::setBlendState(const gl::BlendState &blendState, const gl::ColorF &blendColor, unsigned int sampleMask)
+void Renderer9::setBlendState(gl::Framebuffer *framebuffer, const gl::BlendState &blendState, const gl::ColorF &blendColor,
+                              unsigned int sampleMask)
 {
     bool blendStateChanged = mForceSetBlendState || memcmp(&blendState, &mCurBlendState, sizeof(gl::BlendState)) != 0;
     bool blendColorChanged = mForceSetBlendState || memcmp(&blendColor, &mCurBlendColor, sizeof(gl::ColorF)) != 0;
@@ -926,6 +927,10 @@
             FIXME("Sample alpha to coverage is unimplemented.");
         }
 
+        gl::Renderbuffer *renderBuffer = framebuffer->getFirstColorbuffer();
+        GLenum internalFormat = renderBuffer ? renderBuffer->getInternalFormat() : GL_NONE;
+        GLuint clientVersion = getCurrentClientVersion();
+
         // Set the color mask
         bool zeroColorMaskAllowed = getAdapterVendor() != VENDOR_ID_AMD;
         // Apparently some ATI cards have a bug where a draw with a zero color
@@ -934,8 +939,10 @@
         // drawing is done.
         // http://code.google.com/p/angleproject/issues/detail?id=169
 
-        DWORD colorMask = gl_d3d9::ConvertColorMask(blendState.colorMaskRed, blendState.colorMaskGreen,
-                                                    blendState.colorMaskBlue, blendState.colorMaskAlpha);
+        DWORD colorMask = gl_d3d9::ConvertColorMask(gl::GetRedBits(internalFormat, clientVersion) > 0 && blendState.colorMaskRed,
+                                                    gl::GetGreenBits(internalFormat, clientVersion) > 0 && blendState.colorMaskGreen,
+                                                    gl::GetBlueBits(internalFormat, clientVersion) > 0 && blendState.colorMaskBlue,
+                                                    gl::GetAlphaBits(internalFormat, clientVersion) > 0 && blendState.colorMaskAlpha);
         if (colorMask == 0 && !zeroColorMaskAllowed)
         {
             // Enable green channel, but set blending so nothing will be drawn.
@@ -1386,6 +1393,7 @@
     {
         mForceSetScissor = true;
         mForceSetViewport = true;
+        mForceSetBlendState = true;
 
         mRenderTargetDesc.width = renderbufferObject->getWidth();
         mRenderTargetDesc.height = renderbufferObject->getHeight();
diff --git a/src/libGLESv2/renderer/d3d9/Renderer9.h b/src/libGLESv2/renderer/d3d9/Renderer9.h
index 6ca25ab..d7a1b28 100644
--- a/src/libGLESv2/renderer/d3d9/Renderer9.h
+++ b/src/libGLESv2/renderer/d3d9/Renderer9.h
@@ -76,7 +76,7 @@
     virtual bool setUniformBuffers(const gl::Buffer *vertexUniformBuffers[], const gl::Buffer *fragmentUniformBuffers[]);
 
     virtual void setRasterizerState(const gl::RasterizerState &rasterState);
-    virtual void setBlendState(const gl::BlendState &blendState, const gl::ColorF &blendColor,
+    virtual void setBlendState(gl::Framebuffer *framebuffer, const gl::BlendState &blendState, const gl::ColorF &blendColor,
                                unsigned int sampleMask);
     virtual void setDepthStencilState(const gl::DepthStencilState &depthStencilState, int stencilRef,
                                       int stencilBackRef, bool frontFaceCCW);