Move current love.graphics.setStencilMode functionality to setStencil…

…State.

Add new higher level love.graphics.setStencilMode function.
- Add love.graphics.setStencilMode(mode [, value = 1])
- Add love.graphics.setStencilMode().
- Add mode, value = love.graphics.getStencilMode().

The possible modes are "off" (same as no parameters), "draw", and "test".
The "draw" mode disables color writes and sets the stencil state to ("replace", "always") using the given value.
The "test" mode enables color writes and sets the stencil state to ("keep", "equal") using the given value to compare to.

changes.txt

@@ -68,7 +68,8 @@ Released: N/A
 * Added APIs to override the default orthographic projection: love.graphics.setOrthoProjection, setPerspectiveProjection, and resetProjection.
 * Added ability to set point size within a vertex shader by setting the 'love_PointSize' variable.
 * Added love.graphics.setBlendState, which gives lower level control over blend operations than setBlendMode.
-* Added love.graphics.setStencilMode and getStencilMode. Replaces love.graphics.stencil as well as setStencilTest.
+* Added high level love.graphics.setStencilMode and getStencilMode functions. Replaces love.graphics.stencil and setStencilTest.
+* Added lower level love.graphics.setStencilState and getStencilState functions.
 * Added a variant of love.graphics.setColorMask which accepts a single boolean.
 * Added new 'clampone' wrap mode.
 * Added 'clampone', 'texelbuffer', 'indexbuffer32bit', 'mipmaprange', and 'indirectdraw' graphics feature enums.
@@ -113,8 +114,8 @@ Released: N/A
 * Deprecated love.graphics.setNewFont (use love.graphics.newFont and love.graphics.setFont instead).
 * Deprecated love.graphics.newText (renamed to love.graphics.newTextBatch).
 * Deprecated love.graphics.getImageFormats and love.graphics.getCanvasFormats (replaced by getTextureFormats).
-* Deprecated love.graphics.stencil (replaced by love.graphics.setStencilMode).
-* Deprecated love.graphics.setStencilTest and love.graphics.getStencilTest (replaced by love.graphics.setStencilMode and getStencilMode).
+* Deprecated love.graphics.stencil (replaced by love.graphics.setStencilMode or love.graphics.setStencilState).
+* Deprecated love.graphics.setStencilTest and love.graphics.getStencilTest (replaced by love.graphics.setStencilMode or setStencilState).
 * Deprecated t.window.highdpi in love.conf and the highdpi flag of love.window.setMode (replaced by t.highdpi in love.conf).
 * Deprecated t.accelerometerjoystick in love.conf (replaced by love.sensor module).
 * Deprecated the variants of Mesh:attachAttribute and SpriteBatch:attachAttribute which accept a Mesh (replaced by variants which accept a Buffer).

src/modules/graphics/Graphics.cpp

@@ -700,7 +700,7 @@ void Graphics::restoreState(const DisplayState &s)
 	setShader(s.shader.get());
 	setRenderTargets(s.renderTargets);
 
-	setStencilMode(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
+	setStencilState(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
 	setDepthMode(s.depthTest, s.depthWrite);
 
 	setColorMask(s.colorMask);
@@ -776,7 +776,7 @@ void Graphics::restoreStateChecked(const DisplayState &s)
 		setRenderTargets(s.renderTargets);
 
 	if (!(s.stencil == cur.stencil))
-		setStencilMode(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
+		setStencilState(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
 
 	if (s.depthTest != cur.depthTest || s.depthWrite != cur.depthWrite)
 		setDepthMode(s.depthTest, s.depthWrite);
@@ -1309,12 +1309,37 @@ bool Graphics::getScissor(Rect &rect) const
 	return state.scissor;
 }
 
+void Graphics::setStencilMode(StencilMode mode, int value)
+{
+	StencilState s = computeStencilState(mode, value);
+	setStencilState(s.action, s.compare, s.value, s.readMask, s.writeMask);
+	if (mode == STENCIL_MODE_DRAW)
+		setColorMask({ false, false, false, false });
+	else
+		setColorMask({ true, true, true, true });
+}
+
 void Graphics::setStencilMode()
 {
-	setStencilMode(STENCIL_KEEP, COMPARE_ALWAYS, 0, LOVE_UINT32_MAX, LOVE_UINT32_MAX);
+	StencilState s = computeStencilState(STENCIL_MODE_OFF, 0);
+	setStencilState(s.action, s.compare, s.value, s.readMask, s.writeMask);
+	setColorMask({ true, true, true, true });
+}
+
+StencilMode Graphics::getStencilMode(int &value) const
+{
+	const DisplayState& state = states.back();
+	StencilMode mode = computeStencilMode(state.stencil);
+	value = state.stencil.value;
+	return mode;
+}
+
+void Graphics::setStencilState()
+{
+	setStencilState(STENCIL_KEEP, COMPARE_ALWAYS, 0, LOVE_UINT32_MAX, LOVE_UINT32_MAX);
 }
 
-void Graphics::getStencilMode(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const
+void Graphics::getStencilState(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const
 {
 	const DisplayState &state = states.back();
 	action = state.stencil.action;

src/modules/graphics/Graphics.h

@@ -613,9 +613,13 @@ class Graphics : public Module
 	 */
 	bool getScissor(Rect &rect) const;
 
-	virtual void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) = 0;
+	void setStencilMode(StencilMode mode, int value);
 	void setStencilMode();
-	void getStencilMode(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const;
+	StencilMode getStencilMode(int &value) const;
+
+	virtual void setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) = 0;
+	void setStencilState();
+	void getStencilState(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const;
 
 	virtual void setDepthMode(CompareMode compare, bool write) = 0;
 	void setDepthMode();

src/modules/graphics/metal/Graphics.h

@@ -96,7 +96,7 @@ class Graphics final : public love::graphics::Graphics
 	void setScissor(const Rect &rect) override;
 	void setScissor() override;
 
-	void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
+	void setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
 
 	void setDepthMode(CompareMode compare, bool write) override;
 

src/modules/graphics/metal/Graphics.mm

@@ -846,7 +846,7 @@ static bool isClampOne(SamplerState::WrapMode w)
 	 * example, if the compare function is GREATER then the stencil test will
 	 * pass if the reference value is greater than the value in the stencil
 	 * buffer. With our API it's more intuitive to assume that
-	 * setStencilMode(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
+	 * setStencilState(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
 	 * stencil buffer has a value greater than 4.
 	 **/
 	stencildesc.stencilCompareFunction = getMTLCompareFunction(getReversedCompareMode(stencil.compare));
@@ -1760,7 +1760,7 @@ static inline void advanceVertexOffsets(const VertexAttributes &attributes, Buff
 	}
 }
 
-void Graphics::setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
+void Graphics::setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
 {
 	DisplayState &state = states.back();
 

src/modules/graphics/opengl/Graphics.cpp

@@ -1434,7 +1434,7 @@ void Graphics::setScissor()
 		gl.setEnableState(OpenGL::ENABLE_SCISSOR_TEST, false);
 }
 
-void Graphics::setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
+void Graphics::setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
 {
 	DisplayState &state = states.back();
 
@@ -1493,7 +1493,7 @@ void Graphics::setStencilMode(StencilAction action, CompareMode compare, int val
 	 * example, if the compare function is GREATER then the stencil test will
 	 * pass if the reference value is greater than the value in the stencil
 	 * buffer. With our API it's more intuitive to assume that
-	 * setStencilMode(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
+	 * setStencilState(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
 	 * stencil buffer has a value greater than 4.
 	 **/
 	GLenum glcompare = OpenGL::getGLCompareMode(getReversedCompareMode(compare));

src/modules/graphics/opengl/Graphics.h

@@ -92,7 +92,7 @@ class Graphics final : public love::graphics::Graphics
 	void setScissor(const Rect &rect) override;
 	void setScissor() override;
 
-	void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
+	void setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
 
 	void setDepthMode(CompareMode compare, bool write) override;
 

src/modules/graphics/renderstate.cpp

@@ -28,7 +28,7 @@ namespace graphics
 
 // These are all with premultiplied alpha. computeBlendState adjusts for
 // alpha-multiply if needed.
-static const BlendState states[BLEND_MAX_ENUM] =
+static const BlendState blendStates[BLEND_MAX_ENUM] =
 {
 	// BLEND_ALPHA
 	{BLENDOP_ADD, BLENDOP_ADD, BLENDFACTOR_ONE, BLENDFACTOR_ONE, BLENDFACTOR_ONE_MINUS_SRC_ALPHA, BLENDFACTOR_ONE_MINUS_SRC_ALPHA},
@@ -63,7 +63,7 @@ static const BlendState states[BLEND_MAX_ENUM] =
 
 BlendState computeBlendState(BlendMode mode, BlendAlpha alphamode)
 {
-	BlendState s = states[mode];
+	BlendState s = blendStates[mode];
 
 	// We can only do alpha-multiplication when srcRGB would have been unmodified.
 	if (s.srcFactorRGB == BLENDFACTOR_ONE && alphamode == BLENDALPHA_MULTIPLY && mode != BLEND_NONE)
@@ -87,7 +87,7 @@ BlendMode computeBlendMode(BlendState s, BlendAlpha &alphamode)
 
 	for (int i = 0; i < (int) BLEND_MAX_ENUM; i++)
 	{
-		if (i != (int) BLEND_CUSTOM && states[i] == s)
+		if (i != (int) BLEND_CUSTOM && blendStates[i] == s)
 		{
 			alphamode = alphamultiply ? BLENDALPHA_MULTIPLY : BLENDALPHA_PREMULTIPLIED;
 			return (BlendMode) i;
@@ -111,6 +111,39 @@ bool isAlphaMultiplyBlendSupported(BlendMode mode)
 	}
 }
 
+static const StencilState stencilStates[STENCIL_MODE_MAX_ENUM] =
+{
+	// STENCIL_MODE_OFF
+	{},
+
+	// STENCIL_MODE_DRAW
+	{COMPARE_ALWAYS, STENCIL_REPLACE},
+
+	// STENCIL_MODE_TEST
+	{COMPARE_EQUAL, STENCIL_KEEP},
+
+	// STENCIL_MODE_CUSTOM - N/A
+	{},
+};
+
+StencilState computeStencilState(StencilMode mode, int value)
+{
+	StencilState s = stencilStates[mode];
+	s.value = value;
+	return s;
+}
+
+StencilMode computeStencilMode(const StencilState &s)
+{
+	for (int i = 0; i < (int)STENCIL_MODE_MAX_ENUM; i++)
+	{
+		if (stencilStates[i].action == s.action && stencilStates[i].compare == s.compare)
+			return (StencilMode) i;
+	}
+
+	return STENCIL_MODE_CUSTOM;
+}
+
 CompareMode getReversedCompareMode(CompareMode mode)
 {
 	switch (mode)
@@ -171,6 +204,15 @@ STRINGMAP_BEGIN(BlendOperation, BLENDOP_MAX_ENUM, blendOperation)
 }
 STRINGMAP_END(BlendOperation, BLENDOP_MAX_ENUM, blendOperation)
 
+STRINGMAP_BEGIN(StencilMode, STENCIL_MODE_MAX_ENUM, stencilMode)
+{
+	{ "off",    STENCIL_MODE_OFF    },
+	{ "draw",   STENCIL_MODE_DRAW   },
+	{ "test",   STENCIL_MODE_TEST   },
+	{ "custom", STENCIL_MODE_CUSTOM },
+}
+STRINGMAP_END(StencilMode, STENCIL_MODE_MAX_ENUM, stencilMode)
+
 STRINGMAP_BEGIN(StencilAction, STENCIL_MAX_ENUM, stencilAction)
 {
 	{ "keep",          STENCIL_KEEP           },

src/modules/graphics/renderstate.h

@@ -83,6 +83,15 @@ enum BlendOperation
 	BLENDOP_MAX_ENUM
 };
 
+enum StencilMode // High level wrappers.
+{
+	STENCIL_MODE_OFF,
+	STENCIL_MODE_DRAW,
+	STENCIL_MODE_TEST,
+	STENCIL_MODE_CUSTOM,
+	STENCIL_MODE_MAX_ENUM
+};
+
 enum StencilAction
 {
 	STENCIL_KEEP,
@@ -194,6 +203,9 @@ BlendState computeBlendState(BlendMode mode, BlendAlpha alphamode);
 BlendMode computeBlendMode(BlendState s, BlendAlpha &alphamode);
 bool isAlphaMultiplyBlendSupported(BlendMode mode);
 
+StencilState computeStencilState(StencilMode mode, int value);
+StencilMode computeStencilMode(const StencilState &s);
+
 /**
  * GPU APIs do the comparison in the opposite way of what makes sense for some
  * of love's APIs. For example in OpenGL if the compare function is GL_GREATER,
@@ -208,6 +220,7 @@ STRINGMAP_DECLARE(BlendMode);
 STRINGMAP_DECLARE(BlendAlpha);
 STRINGMAP_DECLARE(BlendFactor);
 STRINGMAP_DECLARE(BlendOperation);
+STRINGMAP_DECLARE(StencilMode);
 STRINGMAP_DECLARE(StencilAction);
 STRINGMAP_DECLARE(CompareMode);
 

src/modules/graphics/vulkan/Graphics.cpp

@@ -1001,7 +1001,7 @@ void Graphics::setScissor()
 	vkCmdSetScissor(commandBuffers.at(currentFrame), 0, 1, &scissor);
 }
 
-void Graphics::setStencilMode(StencilAction action, CompareMode compare, int value, love::uint32 readmask, love::uint32 writemask)
+void Graphics::setStencilState(StencilAction action, CompareMode compare, int value, love::uint32 readmask, love::uint32 writemask)
 {
 	if (action != STENCIL_KEEP)
 	{

src/modules/graphics/vulkan/Graphics.h

@@ -293,7 +293,7 @@ class Graphics final : public love::graphics::Graphics
 	void setColor(Colorf c) override;
 	void setScissor(const Rect &rect) override;
 	void setScissor() override;
-	void setStencilMode(StencilAction action, CompareMode compare, int value, love::uint32 readmask, love::uint32 writemask) override;
+	void setStencilState(StencilAction action, CompareMode compare, int value, love::uint32 readmask, love::uint32 writemask) override;
 	void setDepthMode(CompareMode compare, bool write) override;
 	void setFrontFaceWinding(Winding winding) override;
 	void setColorMask(ColorChannelMask mask) override;

src/modules/graphics/wrap_Graphics.cpp

@@ -608,7 +608,40 @@ int w_setStencilMode(lua_State *L)
 {
 	if (lua_gettop(L) <= 1 && lua_isnoneornil(L, 1))
 	{
-		luax_catchexcept(L, [&](){ instance()->setStencilMode(); });
+		luax_catchexcept(L, [&]() { instance()->setStencilMode(); });
+		return 0;
+	}
+
+	StencilMode mode = STENCIL_MODE_OFF;
+	const char *modestr = luaL_checkstring(L, 1);
+	if (!getConstant(modestr, mode))
+		return luax_enumerror(L, "stencil mode", getConstants(mode), modestr);
+
+	int value = (int) luaL_optinteger(L, 3, 1);
+
+	luax_catchexcept(L, [&]() { instance()->setStencilMode(mode, value); });
+	return 0;
+}
+
+int w_getStencilMode(lua_State *L)
+{
+	int value = 0;
+	StencilMode mode = instance()->getStencilMode(value);
+
+	const char *modestr;
+	if (!getConstant(mode, modestr))
+		return luaL_error(L, "Unknown stencil mode.");
+
+	lua_pushstring(L, modestr);
+	lua_pushinteger(L, value);
+	return 2;
+}
+
+int w_setStencilState(lua_State *L)
+{
+	if (lua_gettop(L) <= 1 && lua_isnoneornil(L, 1))
+	{
+		luax_catchexcept(L, [&](){ instance()->setStencilState(); });
 		return 0;
 	}
 
@@ -627,19 +660,19 @@ int w_setStencilMode(lua_State *L)
 	uint32 readmask = (uint32) luaL_optnumber(L, 4, LOVE_UINT32_MAX);
 	uint32 writemask = (uint32) luaL_optnumber(L, 5, LOVE_UINT32_MAX);
 
-	luax_catchexcept(L, [&](){ instance()->setStencilMode(action, compare, value, readmask, writemask); });
+	luax_catchexcept(L, [&](){ instance()->setStencilState(action, compare, value, readmask, writemask); });
 	return 0;
 }
 
-int w_getStencilMode(lua_State *L)
+int w_getStencilState(lua_State *L)
 {
 	StencilAction action = STENCIL_KEEP;
 	CompareMode compare = COMPARE_ALWAYS;
 	int value = 1;
 	uint32 readmask = LOVE_UINT32_MAX;
 	uint32 writemask = LOVE_UINT32_MAX;
 
-	instance()->getStencilMode(action, compare, value, readmask, writemask);
+	instance()->getStencilState(action, compare, value, readmask, writemask);
 
 	const char *actionstr;
 	if (!getConstant(action, actionstr))
@@ -651,7 +684,7 @@ int w_getStencilMode(lua_State *L)
 
 	lua_pushstring(L, actionstr);
 	lua_pushstring(L, comparestr);
-	lua_pushnumber(L, value);
+	lua_pushinteger(L, value);
 	lua_pushnumber(L, readmask);
 	lua_pushnumber(L, writemask);
 	return 5;
@@ -3936,6 +3969,8 @@ static const luaL_Reg functions[] =
 
 	{ "setStencilMode", w_setStencilMode },
 	{ "getStencilMode", w_getStencilMode },
+	{ "setStencilState", w_setStencilState },
+	{ "getStencilState", w_getStencilState },
 
 	{ "points", w_points },
 	{ "line", w_line },