Add love.graphics.setStencilMode. Remove old stencil API.

- Add setStencilMode(drawaction, comparemode, value = 0, readmask = 0xFF, writemask = 0xFF).
- Add getStencilMode.
- Remove love.graphics.stencil.
- Remove love.graphics.set/getStencilTest.

Note that the new setStencilMode API does not clear the stencil buffer, and does not turn off color writes (it can still be done manually), unlike love.graphics.stencil.

Fixes #1161.
Fixes #1251.
This commit is contained in:
Alex Szpakowski
2022-01-22 23:36:07 -04:00
parent 173fe3dab5
commit 6dbb7fa5de
10 changed files with 148 additions and 221 deletions
+15 -26
View File
@@ -515,7 +515,6 @@ bool Graphics::isActive() const
void Graphics::reset()
{
DisplayState s;
stopDrawToStencilBuffer();
restoreState(s);
origin();
}
@@ -542,14 +541,6 @@ void Graphics::restoreState(const DisplayState &s)
else
setScissor();
if (s.stencil.action != STENCIL_KEEP)
drawToStencilBuffer(s.stencil.action, s.stencil.value);
else
stopDrawToStencilBuffer();
setStencilTest(s.stencil.compare, s.stencil.value);
setDepthMode(s.depthTest, s.depthWrite);
setMeshCullMode(s.meshCullMode);
setFrontFaceWinding(s.winding);
@@ -557,6 +548,9 @@ void Graphics::restoreState(const DisplayState &s)
setShader(s.shader.get());
setRenderTargets(s.renderTargets);
setStencilMode(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
setDepthMode(s.depthTest, s.depthWrite);
setColorMask(s.colorMask);
setWireframe(s.wireframe);
@@ -596,20 +590,6 @@ void Graphics::restoreStateChecked(const DisplayState &s)
setScissor();
}
if (s.stencil.action != cur.stencil.action)
{
if (s.stencil.action != STENCIL_KEEP)
drawToStencilBuffer(s.stencil.action, s.stencil.value);
else
stopDrawToStencilBuffer();
}
if (s.stencil.compare != cur.stencil.compare || s.stencil.value != cur.stencil.value)
setStencilTest(s.stencil.compare, s.stencil.value);
if (s.depthTest != cur.depthTest || s.depthWrite != cur.depthWrite)
setDepthMode(s.depthTest, s.depthWrite);
setMeshCullMode(s.meshCullMode);
if (s.winding != cur.winding)
@@ -643,6 +623,12 @@ void Graphics::restoreStateChecked(const DisplayState &s)
if (rtschanged)
setRenderTargets(s.renderTargets);
if (!(s.stencil == cur.stencil))
setStencilMode(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
if (s.depthTest != cur.depthTest || s.depthWrite != cur.depthWrite)
setDepthMode(s.depthTest, s.depthWrite);
if (s.colorMask != cur.colorMask)
setColorMask(s.colorMask);
@@ -1046,16 +1032,19 @@ bool Graphics::getScissor(Rect &rect) const
return state.scissor;
}
void Graphics::setStencilTest()
void Graphics::setStencilMode()
{
setStencilTest(COMPARE_ALWAYS, 0);
setStencilMode(STENCIL_KEEP, COMPARE_ALWAYS, 0, LOVE_UINT32_MAX, LOVE_UINT32_MAX);
}
void Graphics::getStencilTest(CompareMode &compare, int &value) const
void Graphics::getStencilMode(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const
{
const DisplayState &state = states.back();
action = state.stencil.action;
compare = state.stencil.compare;
value = state.stencil.value;
readmask = state.stencil.readMask;
writemask = state.stencil.writeMask;
}
void Graphics::setDepthMode()
+3 -13
View File
@@ -585,19 +585,9 @@ public:
*/
bool getScissor(Rect &rect) const;
/**
* Enables or disables drawing to the stencil buffer. When enabled, the
* color buffer is disabled.
**/
virtual void drawToStencilBuffer(StencilAction action, int value) = 0;
virtual void stopDrawToStencilBuffer() = 0;
/**
* Sets whether stencil testing is enabled.
**/
virtual void setStencilTest(CompareMode compare, int value) = 0;
void setStencilTest();
void getStencilTest(CompareMode &compare, int &value) const;
virtual void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) = 0;
void setStencilMode();
void getStencilMode(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const;
virtual void setDepthMode(CompareMode compare, bool write) = 0;
void setDepthMode();
+1 -4
View File
@@ -95,10 +95,7 @@ public:
void setScissor(const Rect &rect) override;
void setScissor() override;
void drawToStencilBuffer(StencilAction action, int value) override;
void stopDrawToStencilBuffer() override;
void setStencilTest(CompareMode compare, int value) override;
void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
void setDepthMode(CompareMode compare, bool write) override;
+24 -50
View File
@@ -797,7 +797,15 @@ id<MTLDepthStencilState> Graphics::getCachedDepthStencilState(const DepthState &
MTLStencilDescriptor *stencildesc = [MTLStencilDescriptor new];
stencildesc.stencilCompareFunction = getMTLCompareFunction(stencil.compare);
/**
* GPUs do the comparison opposite to what makes sense for love's API. For
* example, if the compare function is GREATER then the stencil test will
* pass if the reference value is greater than the value in the stencil
* buffer. With our API it's more intuitive to assume that
* setStencilMode(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
* stencil buffer has a value greater than 4.
**/
stencildesc.stencilCompareFunction = getMTLCompareFunction(getReversedCompareMode(stencil.compare));
stencildesc.stencilFailureOperation = MTLStencilOperationKeep;
stencildesc.depthFailureOperation = MTLStencilOperationKeep;
stencildesc.depthStencilPassOperation = getMTLStencilOperation(stencil.action);
@@ -922,15 +930,7 @@ void Graphics::applyRenderState(id<MTLRenderCommandEncoder> encoder, const Verte
depth.compare = state.depthTest;
depth.write = state.depthWrite;
StencilState stencil = state.stencil;
if (stencil.action != STENCIL_KEEP)
{
// FIXME
stencil.compare = COMPARE_ALWAYS;
}
id<MTLDepthStencilState> mtlstate = getCachedDepthStencilState(depth, stencil);
id<MTLDepthStencilState> mtlstate = getCachedDepthStencilState(depth, state.stencil);
[encoder setDepthStencilState:mtlstate];
}
@@ -1671,56 +1671,30 @@ void Graphics::setScissor()
}
}
void Graphics::drawToStencilBuffer(StencilAction action, int value)
void Graphics::setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
{
DisplayState &state = states.back();
const auto &rts = state.renderTargets;
love::graphics::Texture *dstexture = rts.depthStencil.texture.get();
if (!isRenderTargetActive() && !windowHasStencil)
throw love::Exception("The window must have stenciling enabled to draw to the main screen's stencil buffer.");
else if (isRenderTargetActive() && (rts.temporaryRTFlags & TEMPORARY_RT_STENCIL) == 0 && (dstexture == nullptr || !isPixelFormatStencil(dstexture->getPixelFormat())))
throw love::Exception("Drawing to the stencil buffer with a Canvas active requires either stencil=true or a custom stencil-type Canvas to be used, in setCanvas.");
if (action != STENCIL_KEEP)
{
const auto &rts = state.renderTargets;
love::graphics::Texture *dstexture = rts.depthStencil.texture.get();
if (!isRenderTargetActive() && !windowHasStencil)
throw love::Exception("The window must have stenciling enabled to draw to the main screen's stencil buffer.");
else if (isRenderTargetActive() && (rts.temporaryRTFlags & TEMPORARY_RT_STENCIL) == 0 && (dstexture == nullptr || !isPixelFormatStencil(dstexture->getPixelFormat())))
throw love::Exception("Drawing to the stencil buffer with a Canvas active requires either stencil=true or a custom stencil-type Canvas to be used, in setCanvas.");
}
flushBatchedDraws();
state.stencil.action = action;
state.stencil.compare = compare;
state.stencil.value = value;
state.stencil.readMask = readmask;
state.stencil.writeMask = writemask;
dirtyRenderState |= STATEBIT_STENCIL;
// TODO
}
void Graphics::stopDrawToStencilBuffer()
{
DisplayState &state = states.back();
if (state.stencil.action == STENCIL_KEEP)
return;
flushBatchedDraws();
state.stencil.action = STENCIL_KEEP;
// Revert the color write mask.
setColorMask(state.colorMask);
// Use the user-set stencil test state when writes are disabled.
setStencilTest(state.stencil.compare, state.stencil.value);
dirtyRenderState |= STATEBIT_STENCIL;
}
void Graphics::setStencilTest(CompareMode compare, int value)
{
// TODO
DisplayState &state = states.back();
if (state.stencil.compare != compare || state.stencil.value != value)
{
state.stencil.compare = compare;
state.stencil.value = value;
dirtyRenderState |= STATEBIT_STENCIL;
}
}
void Graphics::setDepthMode(CompareMode compare, bool write)
+49 -68
View File
@@ -869,8 +869,13 @@ void Graphics::clear(OptionalColorD c, OptionalInt stencil, OptionalDouble depth
flags |= GL_COLOR_BUFFER_BIT;
}
uint32 stencilwrites = gl.getStencilWriteMask();
if (stencil.hasValue)
{
if (stencilwrites != LOVE_UINT32_MAX)
gl.setStencilWriteMask(LOVE_UINT32_MAX);
glClearStencil(stencil.value);
flags |= GL_STENCIL_BUFFER_BIT;
}
@@ -889,6 +894,9 @@ void Graphics::clear(OptionalColorD c, OptionalInt stencil, OptionalDouble depth
if (flags != 0)
glClear(flags);
if (stencil.hasValue && stencilwrites != LOVE_UINT32_MAX)
gl.setStencilWriteMask(stencilwrites);
if (depth.hasValue && !hadDepthWrites)
gl.setDepthWrites(hadDepthWrites);
@@ -980,8 +988,13 @@ void Graphics::clear(const std::vector<OptionalColorD> &colors, OptionalInt sten
GLbitfield flags = 0;
uint32 stencilwrites = gl.getStencilWriteMask();
if (stencil.hasValue)
{
if (stencilwrites != LOVE_UINT32_MAX)
gl.setStencilWriteMask(LOVE_UINT32_MAX);
glClearStencil(stencil.value);
flags |= GL_STENCIL_BUFFER_BIT;
}
@@ -1000,6 +1013,9 @@ void Graphics::clear(const std::vector<OptionalColorD> &colors, OptionalInt sten
if (flags != 0)
glClear(flags);
if (stencil.hasValue && stencilwrites != LOVE_UINT32_MAX)
gl.setStencilWriteMask(stencilwrites);
if (depth.hasValue && !hadDepthWrites)
gl.setDepthWrites(hadDepthWrites);
@@ -1377,24 +1393,26 @@ void Graphics::setScissor()
gl.setEnableState(OpenGL::ENABLE_SCISSOR_TEST, false);
}
void Graphics::drawToStencilBuffer(StencilAction action, int value)
void Graphics::setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
{
DisplayState &state = states.back();
const auto &rts = state.renderTargets;
love::graphics::Texture *dstexture = rts.depthStencil.texture.get();
if (!isRenderTargetActive() && !windowHasStencil)
throw love::Exception("The window must have stenciling enabled to draw to the main screen's stencil buffer.");
else if (isRenderTargetActive() && (rts.temporaryRTFlags & TEMPORARY_RT_STENCIL) == 0 && (dstexture == nullptr || !isPixelFormatStencil(dstexture->getPixelFormat())))
throw love::Exception("Drawing to the stencil buffer with a render target active requires either stencil=true or a custom stencil-type texture to be used, in setRenderTarget.");
if (action != STENCIL_KEEP)
{
const auto &rts = state.renderTargets;
love::graphics::Texture *dstexture = rts.depthStencil.texture.get();
if (!isRenderTargetActive() && !windowHasStencil)
throw love::Exception("The window must have stenciling enabled to draw to the main screen's stencil buffer.");
else if (isRenderTargetActive() && (rts.temporaryRTFlags & TEMPORARY_RT_STENCIL) == 0 && (dstexture == nullptr || !isPixelFormatStencil(dstexture->getPixelFormat())))
throw love::Exception("Drawing to the stencil buffer with a render target active requires either stencil=true or a custom stencil-type texture to be used, in setRenderTarget.");
}
flushBatchedDraws();
state.stencil.action = action;
state.stencil.value = value;
// Disable color writes but don't save the state for it.
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
bool enablestencil = action != STENCIL_KEEP || compare != COMPARE_ALWAYS;
if (enablestencil != gl.isStateEnabled(OpenGL::ENABLE_STENCIL_TEST))
gl.setEnableState(OpenGL::ENABLE_STENCIL_TEST, enablestencil);
GLenum glaction = GL_REPLACE;
@@ -1421,67 +1439,30 @@ void Graphics::drawToStencilBuffer(StencilAction action, int value)
break;
}
// The stencil test must be enabled in order to write to the stencil buffer.
if (!gl.isStateEnabled(OpenGL::ENABLE_STENCIL_TEST))
gl.setEnableState(OpenGL::ENABLE_STENCIL_TEST, true);
glStencilFunc(GL_ALWAYS, value, 0xFFFFFFFF);
glStencilOp(GL_KEEP, GL_KEEP, glaction);
}
void Graphics::stopDrawToStencilBuffer()
{
DisplayState &state = states.back();
if (state.stencil.action == STENCIL_KEEP)
return;
flushBatchedDraws();
state.stencil.action = STENCIL_KEEP;
// Revert the color write mask.
setColorMask(state.colorMask);
// Use the user-set stencil test state when writes are disabled.
setStencilTest(state.stencil.compare, state.stencil.value);
}
void Graphics::setStencilTest(CompareMode compare, int value)
{
DisplayState &state = states.back();
if (state.stencil.compare != compare || state.stencil.value != value)
flushBatchedDraws();
state.stencil.compare = compare;
state.stencil.value = value;
if (state.stencil.action != STENCIL_KEEP)
return;
if (compare == COMPARE_ALWAYS)
{
if (gl.isStateEnabled(OpenGL::ENABLE_STENCIL_TEST))
gl.setEnableState(OpenGL::ENABLE_STENCIL_TEST, false);
return;
}
/**
* OpenGL / GPUs do the comparison in the opposite way that makes sense
* for this API. For example, if the compare function is GL_GREATER then the
* stencil test will pass if the reference value is greater than the value
* in the stencil buffer. With our API it's more intuitive to assume that
* setStencilTest(COMPARE_GREATER, 4) will make it pass if the stencil
* buffer has a value greater than 4.
* GPUs do the comparison opposite to what makes sense for love's API. For
* example, if the compare function is GREATER then the stencil test will
* pass if the reference value is greater than the value in the stencil
* buffer. With our API it's more intuitive to assume that
* setStencilMode(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
* stencil buffer has a value greater than 4.
**/
GLenum glcompare = OpenGL::getGLCompareMode(getReversedCompareMode(compare));
if (!gl.isStateEnabled(OpenGL::ENABLE_STENCIL_TEST))
gl.setEnableState(OpenGL::ENABLE_STENCIL_TEST, true);
if (enablestencil)
{
glStencilFunc(glcompare, value, readmask);
glStencilOp(GL_KEEP, GL_KEEP, glaction);
}
glStencilFunc(glcompare, value, 0xFFFFFFFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
if (writemask != gl.getStencilWriteMask())
gl.setStencilWriteMask(writemask);
state.stencil.action = action;
state.stencil.compare = compare;
state.stencil.value = value;
state.stencil.readMask = readmask;
state.stencil.writeMask = writemask;
}
void Graphics::setDepthMode(CompareMode compare, bool write)
+1 -4
View File
@@ -91,10 +91,7 @@ public:
void setScissor(const Rect &rect) override;
void setScissor() override;
void drawToStencilBuffer(StencilAction action, int value) override;
void stopDrawToStencilBuffer() override;
void setStencilTest(CompareMode compare, int value) override;
void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
void setDepthMode(CompareMode compare, bool write) override;
+12
View File
@@ -283,6 +283,7 @@ void OpenGL::setupContext()
state.curTextureUnit = 0;
setDepthWrites(state.depthWritesEnabled);
setStencilWriteMask(state.stencilWriteMask);
createDefaultTexture();
@@ -1112,6 +1113,17 @@ bool OpenGL::hasDepthWrites() const
return state.depthWritesEnabled;
}
void OpenGL::setStencilWriteMask(uint32 mask)
{
glStencilMask(mask);
state.stencilWriteMask = mask;
}
uint32 OpenGL::getStencilWriteMask() const
{
return state.stencilWriteMask;
}
void OpenGL::useProgram(GLuint program)
{
glUseProgram(program);
+4
View File
@@ -298,6 +298,9 @@ public:
void setDepthWrites(bool enable);
bool hasDepthWrites() const;
void setStencilWriteMask(uint32 mask);
uint32 getStencilWriteMask() const;
/**
* Calls glUseProgram.
**/
@@ -524,6 +527,7 @@ private:
float pointSize;
bool depthWritesEnabled = true;
uint32 stencilWriteMask = LOVE_UINT32_MAX;
GLuint boundFramebuffers[2];
+7
View File
@@ -86,12 +86,19 @@ static GLenum createFBO(GLuint &framebuffer, TextureType texType, PixelFormat fo
if (!hadDepthWrites) // glDepthMask also affects glClear.
gl.setDepthWrites(true);
uint32 stencilwrite = gl.getStencilWriteMask();
if (stencilwrite != LOVE_UINT32_MAX)
gl.setStencilWriteMask(LOVE_UINT32_MAX);
gl.clearDepth(1.0);
glClearStencil(0);
glClear(GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
if (!hadDepthWrites)
gl.setDepthWrites(hadDepthWrites);
if (stencilwrite != LOVE_UINT32_MAX)
gl.setStencilWriteMask(stencilwrite);
}
else
{
+32 -56
View File
@@ -267,9 +267,6 @@ static Graphics::RenderTarget checkRenderTarget(lua_State *L, int idx)
int w_setCanvas(lua_State *L)
{
// Disable stencil writes.
luax_catchexcept(L, [](){ instance()->stopDrawToStencilBuffer(); });
// called with none -> reset to default buffer
if (lua_isnoneornil(L, 1))
{
@@ -607,77 +604,57 @@ int w_getScissor(lua_State *L)
return 4;
}
int w_stencil(lua_State *L)
int w_setStencilMode(lua_State *L)
{
luaL_checktype(L, 1, LUA_TFUNCTION);
StencilAction action = STENCIL_REPLACE;
if (!lua_isnoneornil(L, 2))
if (lua_gettop(L) <= 1 && lua_isnoneornil(L, 1))
{
const char *actionstr = luaL_checkstring(L, 2);
if (!getConstant(actionstr, action))
return luax_enumerror(L, "stencil draw action", getConstants(action), actionstr);
luax_catchexcept(L, [&](){ instance()->setStencilMode(); });
return 0;
}
int stencilvalue = (int) luaL_optinteger(L, 3, 1);
StencilAction action = STENCIL_KEEP;
const char *actionstr = luaL_checkstring(L, 1);
if (!getConstant(actionstr, action))
return luax_enumerror(L, "stencil draw action", getConstants(action), actionstr);
// Fourth argument: whether to keep the contents of the stencil buffer.
OptionalInt stencilclear;
int argtype = lua_type(L, 4);
if (argtype == LUA_TNONE || argtype == LUA_TNIL || (argtype == LUA_TBOOLEAN && luax_toboolean(L, 4) == false))
stencilclear.set(0);
else if (argtype == LUA_TNUMBER)
stencilclear.set((int) luaL_checkinteger(L, 4));
else if (argtype != LUA_TBOOLEAN)
luaL_checktype(L, 4, LUA_TBOOLEAN);
CompareMode compare = COMPARE_ALWAYS;
const char *comparestr = luaL_checkstring(L, 2);
if (!getConstant(comparestr, compare))
return luax_enumerror(L, "compare mode", getConstants(compare), comparestr);
if (stencilclear.hasValue)
instance()->clear(OptionalColorD(), stencilclear, OptionalDouble());
int value = (int) luaL_optinteger(L, 3, 0);
luax_catchexcept(L, [&](){ instance()->drawToStencilBuffer(action, stencilvalue); });
uint32 readmask = (uint32) luaL_optnumber(L, 4, LOVE_UINT32_MAX);
uint32 writemask = (uint32) luaL_optnumber(L, 5, LOVE_UINT32_MAX);
// Call stencilfunc()
lua_pushvalue(L, 1);
lua_call(L, 0, 0);
luax_catchexcept(L, [&](){ instance()->stopDrawToStencilBuffer(); });
luax_catchexcept(L, [&](){ instance()->setStencilMode(action, compare, value, readmask, writemask); });
return 0;
}
int w_setStencilTest(lua_State *L)
int w_getStencilMode(lua_State *L)
{
// COMPARE_ALWAYS effectively disables stencil testing.
StencilAction action = STENCIL_KEEP;
CompareMode compare = COMPARE_ALWAYS;
int comparevalue = 0;
int value = 1;
uint32 readmask = LOVE_UINT32_MAX;
uint32 writemask = LOVE_UINT32_MAX;
if (!lua_isnoneornil(L, 1))
{
const char *comparestr = luaL_checkstring(L, 1);
if (!getConstant(comparestr, compare))
return luax_enumerror(L, "compare mode", getConstants(compare), comparestr);
instance()->getStencilMode(action, compare, value, readmask, writemask);
comparevalue = (int) luaL_checkinteger(L, 2);
}
luax_catchexcept(L, [&](){ instance()->setStencilTest(compare, comparevalue); });
return 0;
}
int w_getStencilTest(lua_State *L)
{
CompareMode compare = COMPARE_ALWAYS;
int comparevalue = 1;
instance()->getStencilTest(compare, comparevalue);
const char *actionstr;
if (!getConstant(action, actionstr))
return luaL_error(L, "Unknown stencil draw action.");
const char *comparestr;
if (!getConstant(compare, comparestr))
return luaL_error(L, "Unknown compare mode.");
lua_pushstring(L, actionstr);
lua_pushstring(L, comparestr);
lua_pushnumber(L, comparevalue);
return 2;
lua_pushnumber(L, value);
lua_pushnumber(L, readmask);
lua_pushnumber(L, writemask);
return 5;
}
static void parseDPIScale(Data *d, float *dpiscale)
@@ -3712,9 +3689,8 @@ static const luaL_Reg functions[] =
{ "intersectScissor", w_intersectScissor },
{ "getScissor", w_getScissor },
{ "stencil", w_stencil },
{ "setStencilTest", w_setStencilTest },
{ "getStencilTest", w_getStencilTest },
{ "setStencilMode", w_setStencilMode },
{ "getStencilMode", w_getStencilMode },
{ "points", w_points },
{ "line", w_line },