Stencil Testing

Stencil testing is another mechanism provided by OpenGL to discard pixels based on some test.

With stencil test, there is another buffer that contains information for discarding pixels. This buffer can be modified our the developer and can be used to create some interesting effects.

First we need to enable it:

glEnable(GL_STENCIL_TEST);
glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);

Should not forget to clear the stencil buffer same as we did with the depth buffer.

void Renderer_ClearBackground(float R, float G, float B, float Alpha) {
    glClearColor(R, G, B, Alpha);
    glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
}

In our case, we will use the stencil buffer to draw an outline around the model. For this We first need to enable writing on the stencil buffer and then draw the object. This will write on the buffer, 1s where the object is. Then we have to disable the stencil buffer write and draw again to the color buffer. We can now use the stencil buffer to draw the outline with a different shader.

void Renderer_DrawModel(const renderer &Renderer, glm::vec<3, float> Position,
                        glm::vec<3, float> Scale, glm::vec<4, float> Rotation,
                        glm::vec<4, float> Color, model EntityModel,
                        bool IsSelected) {
    // 1st render pass
    glStencilFunc(GL_ALWAYS, 1, 0xFF);
    glStencilMask(0xFF);

    glUseProgram(Renderer.ShaderProgram.ID);

    glm::vec3 CubeColor = glm::vec3(Color[0], Color[1], Color[2]);
    glUniform3fv(Renderer.ShaderProgram.Uniforms.EntityColorUniformLoc, 1,
                 glm::value_ptr(CubeColor));

    glm::mat4 Model = glm::mat4(1.0f);
    Model = glm::translate(Model, Position);
    Model = glm::scale(Model, Scale);

    glm::vec3 RotationVec = glm::vec3(Rotation[1], Rotation[2], Rotation[3]);
    Model = glm::rotate(Model, glm::radians(Rotation[0]), RotationVec);
    glUniformMatrix4fv(Renderer.ShaderProgram.Uniforms.ModelUniformLoc, 1,
                       GL_FALSE, glm::value_ptr(Model));

    Model_Draw(Renderer.ShaderProgram.ID, &EntityModel);

    if (IsSelected) {
        // 2st render pass: draws the outline
        glStencilFunc(GL_NOTEQUAL, 1, 0xFF);
        glStencilMask(0xFF);
        glDisable(GL_DEPTH_TEST);

        glUseProgram(Renderer.OutlineShaderProgram.ID);

        Model = glm::mat4(1.0f);
        Model = glm::translate(Model, Position);
        Model = glm::scale(Model, Scale + 0.01f);
        Model = glm::rotate(Model, glm::radians(Rotation[0]), RotationVec);
        glUniformMatrix4fv(
            Renderer.OutlineShaderProgram.Uniforms.ModelUniformLoc, 1, GL_FALSE,
            glm::value_ptr(Model));

        Model_Draw(Renderer.OutlineShaderProgram.ID, &EntityModel);

        glStencilMask(0xFF);
        glStencilFunc(GL_ALWAYS, 1, 0xFF);
        glEnable(GL_DEPTH_TEST);
    }
}