Revamped love.graphics.stencil and love.graphics.setStencilTest:

- Each canvas (and the screen) has an invisible stencil value associated with each pixel. Stencil values are between [0, 255].

- love.graphics.stencil takes a function argument which draws things, and makes the geometry of what's drawn set the stencil values of pixels instead of coloring the pixels.

- Its second argument is a 'stencil action' which determines what happens to the stencil values of pixels. The possible stencil actions are 'replace' (the default), 'invert', 'increment', 'decrement', 'incrementwrap', and 'decrementwrap'. If the stencil action is 'replace', the third argument of love.graphics.stencil is a number between [0, 255] which determines what the stencil value of each pixel that touches drawn objects is replaced with.

The fourth argument is a boolean which determines whether the previous stencil values of all pixels on-screen should be kept (true), or cleared to 0 (false, the default.) love.graphics.clear also clears the stencil values.

- love.graphics.setStencilTest allows for anything drawn to be affected by a comparison between the arguments to setStencilTest and the existing stencil value of the pixels that are touched by what's drawn. Its first argument is a compare mode which can be 'equal', 'notequal', 'less', 'lequal', 'gequal', or 'greater', and the second argument is an integer value (between 0 and 255) used in the comparison with the value in the stencil buffer.

For example, love.graphics.setStencilTest("greater", 0) will cause any geometry drawn afterwards to only appear on pixels whose stencil value is greater than 0.

src/modules/graphics/opengl/wrap_Graphics.cpp

@@ -217,35 +217,63 @@ int w_stencil(lua_State *L)
 {
 	luaL_checktype(L, 1, LUA_TFUNCTION);
 
-	// Second argument: whether to keep the contents of the stencil buffer.
-	if (lua_toboolean(L, 2) == 0)
+	Graphics::StencilAction action = Graphics::STENCIL_REPLACE;
+
+	if (!lua_isnoneornil(L, 2))
+	{
+		const char *actionstr = luaL_checkstring(L, 2);
+		if (!Graphics::getConstant(actionstr, action))
+			return luaL_error(L, "Invalid stencil draw action: %s", actionstr);
+	}
+
+	int stencilvalue = (int) luaL_optnumber(L, 3, 1);
+
+	// Fourth argument: whether to keep the contents of the stencil buffer.
+	if (lua_toboolean(L, 4) == 0)
 		instance()->clearStencil();
 
-	instance()->drawToStencilBuffer(true);
+	instance()->drawToStencilBuffer(action, stencilvalue);
 
 	// Call stencilfunc()
 	lua_pushvalue(L, 1);
 	lua_call(L, 0, 0);
 
-	instance()->drawToStencilBuffer(false);
-
+	instance()->stopDrawToStencilBuffer();
 	return 0;
 }
 
 int w_setStencilTest(lua_State *L)
 {
-	bool enable = luax_toboolean(L, 1);
-	bool invert = luax_toboolean(L, 2);
-	instance()->setStencilTest(enable, invert);
+	// COMPARE_ALWAYS effectively disables stencil testing.
+	Graphics::CompareMode compare = Graphics::COMPARE_ALWAYS;
+	int comparevalue = 0;
+
+	if (!lua_isnoneornil(L, 1))
+	{
+		const char *comparestr = luaL_checkstring(L, 1);
+		if (!Graphics::getConstant(comparestr, compare))
+			return luaL_error(L, "Invalid compare mode: %s", comparestr);
+
+		comparevalue = (int) luaL_checknumber(L, 2);
+	}
+
+	instance()->setStencilTest(compare, comparevalue);
 	return 0;
 }
 
 int w_getStencilTest(lua_State *L)
 {
-	bool enabled, inverted;
-	instance()->getStencilTest(enabled, inverted);
-	luax_pushboolean(L, enabled);
-	luax_pushboolean(L, inverted);
+	Graphics::CompareMode compare = Graphics::COMPARE_ALWAYS;
+	int comparevalue = 1;
+
+	instance()->getStencilTest(compare, comparevalue);
+
+	const char *comparestr;
+	if (!Graphics::getConstant(compare, comparestr))
+		return luaL_error(L, "Unknown compare mode.");
+
+	lua_pushstring(L, comparestr);
+	lua_pushnumber(L, comparevalue);
 	return 2;
 }
 
@@ -1201,7 +1229,7 @@ int w_newScreenshot(lua_State *L)
 int w_setCanvas(lua_State *L)
 {
 	// Disable stencil writes.
-	instance()->drawToStencilBuffer(false);
+	instance()->stopDrawToStencilBuffer();
 
 	// called with none -> reset to default buffer
 	if (lua_isnoneornil(L, 1))