According to spec, the stencil ref is clamped to the range [0,2^s-1]
The clamp to zero occurs on set since negative values will never be useful.
The clamp to 2^s - 1 occurs on use since it is dependant on the currently bound stencil buffer
Original-Author: Jim Hauxwell <james@dattrax.co.uk>
Signed-off-by: Daniel Koch
git-svn-id: https://angleproject.googlecode.com/svn/trunk@324 736b8ea6-26fd-11df-bfd4-992fa37f6226
diff --git a/src/libGLESv2/Context.cpp b/src/libGLESv2/Context.cpp
index 721b29a..1bad7b6 100644
--- a/src/libGLESv2/Context.cpp
+++ b/src/libGLESv2/Context.cpp
@@ -454,7 +454,7 @@
mState.stencilMask != stencilMask)
{
mState.stencilFunc = stencilFunc;
- mState.stencilRef = stencilRef;
+ mState.stencilRef = (stencilRef > 0) ? stencilRef : 0;
mState.stencilMask = stencilMask;
mStencilStateDirty = true;
}
@@ -467,7 +467,7 @@
mState.stencilBackMask != stencilBackMask)
{
mState.stencilBackFunc = stencilBackFunc;
- mState.stencilBackRef = stencilBackRef;
+ mState.stencilBackRef = (stencilBackRef > 0) ? stencilBackRef : 0;
mState.stencilBackMask = stencilBackMask;
mStencilStateDirty = true;
}
@@ -1840,11 +1840,16 @@
return error(GL_INVALID_OPERATION);
}
+ // get the maximum size of the stencil ref
+ gl::Framebuffer *framebuffer = getFramebuffer();
+ gl::Stencilbuffer *stencilbuffer = framebuffer->getStencilbuffer();
+ GLuint maxStencil = (1 << stencilbuffer->getStencilSize()) - 1;
+
device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILWRITEMASK : D3DRS_CCW_STENCILWRITEMASK, mState.stencilWritemask);
device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILFUNC : D3DRS_CCW_STENCILFUNC,
es2dx::ConvertComparison(mState.stencilFunc));
- device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILREF : D3DRS_CCW_STENCILREF, mState.stencilRef); // FIXME: Clamp to range
+ device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILREF : D3DRS_CCW_STENCILREF, (mState.stencilRef < (GLint)maxStencil) ? mState.stencilRef : maxStencil);
device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILMASK : D3DRS_CCW_STENCILMASK, mState.stencilMask);
device->SetRenderState(mState.frontFace == GL_CCW ? D3DRS_STENCILFAIL : D3DRS_CCW_STENCILFAIL,
@@ -1858,7 +1863,7 @@
device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILFUNC : D3DRS_CCW_STENCILFUNC,
es2dx::ConvertComparison(mState.stencilBackFunc));
- device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILREF : D3DRS_CCW_STENCILREF, mState.stencilBackRef); // FIXME: Clamp to range
+ device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILREF : D3DRS_CCW_STENCILREF, (mState.stencilBackRef < (GLint)maxStencil) ? mState.stencilBackRef : maxStencil);
device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILMASK : D3DRS_CCW_STENCILMASK, mState.stencilBackMask);
device->SetRenderState(mState.frontFace == GL_CW ? D3DRS_STENCILFAIL : D3DRS_CCW_STENCILFAIL,