https://www.khronos.org/registry/OpenGL-Refpages/gl4/html/glStencilFunc.xhtml
https://www.khronos.org/registry/OpenGL-Refpages/gl4/html/glStencilOp.xhtml
Controls:
Move your cursor to move the little triangle
Scroll up / down to change the depth of the little triangle
Press "SPACE" to enable / disable stencil testing
The big triangle has a depth value of 0.5f. Move the little triangle over the big one and scroll up / down to see the depth test working. Press SPACE to see what effect the stencil test has at various depth values.
Results:
If the stencil test is disabled, then the little triangle will be covered by the big one due to the depth test if the depth value is bigger than 0.5f. If covered, you can make is appear if you enable the stencil test. The little triangle is drawn first with these settings:
/* little triangle: fill stencil buffer */
GLint stencilvalue = 0x01;
GLuint stencilmask = 0xFF;
glStencilFunc(GL_ALWAYS, stencilvalue, stencilmask);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
glStencilFunc() is set to GL_ALWAYS, that means te stencil test will always pass, regardless of the stencil value.
glStencilOp() is set to:
KEEP the stencil buffers value if stencil test fails
otherwise: KEEP the stencil buffers value if depth test fails
otherwise: REPLACE the stencil buffers value if depth test passes
Because it is drawn first (directly after we cleared the framebuffer), the area covered by the little triangle will be filled with the stencil value (ANDed with mask which is 0xFF, that means all stencil value bits will be used).
Later these funcctions change:
/* big triangle: only check stencil buffer */
glStencilFunc(GL_NOTEQUAL, stencilvalue, stencilmask);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
glStencilFunc() is now set to GL_NOTEQUAL, that means that the stencil test will only pass if stencil value (ANDed with mask) will be NOT EQUAL to the value in the stencil buffer. In addition to that, we wont update the stencil buffer, regardless if the depth test and / or stencil test passes (3x GL_KEEP). That means whereever the little triangle has set the stencil bits to (0x01 AND 0xFF == 0x01) , the stencil test will pass if the new incoming stencil value is NOT EQUAL to 0x01. But we havent changed the stencil value, thats why the stenccil test fails there (where little triangle is located).
If we disable stencil testing, then these 2 functions wont have any effect at all.
Main.cpp
#include <GL/glew.h>
#include <GLFW/glfw3.h>
#include <glm/glm.hpp>
#include <glm/gtc/type_ptr.hpp>
#include <glm/gtx/transform.hpp>
#include <iostream>
#include <vector>
#include "Shader.h"
using namespace std;
using namespace glm;
/* callbacks */
/******************************************************************************************************/
void error_callback(int error, const char* description);
void drop_callback(GLFWwindow* window, int pathcount, const char** paths);
void framebuffer_size_callback(GLFWwindow* window, int width, int height);
void key_callback(GLFWwindow* window, int key, int scancode, int action, int mods);
void cursor_position_callback(GLFWwindow* window, double xpos, double ypos);
void mouse_button_callback(GLFWwindow* window, int button, int action, int mods);
void scroll_callback(GLFWwindow* window, double xoffset, double yoffset);
/******************************************************************************************************/
/* main functions */
/******************************************************************************************************/
#define CheckForGLErrors CheckForGLErrors__(__FILE__, __LINE__);
bool Initialize();
void Render();
void CleanUp();
void CheckForGLErrors__(const char* file, unsigned int line);
/******************************************************************************************************/
/* global variables */
/******************************************************************************************************/
enum Buffers {
Buffer_Vertices,
MAX_Buffers,
};
enum Queries {
/* none */
MAX_Queries,
};
enum Renderbuffers {
/* none */
MAX_Renderbuffers,
};
enum Samplers {
/* none */
MAX_Samplers,
};
enum Textures {
/* none */
MAX_Textures,
};
enum Programs {
Program_Vertices,
MAX_Programs,
};
enum Framebuffers {
/* none */
MAX_Framebuffers,
};
enum TransformFeedbacks {
/* none */
MAX_TransformFeedbacks,
};
enum VertexArrays {
VertexArray_Vertices,
MAX_VertexArrays,
};
/* objects sharable between contexts */
GLuint buffers [MAX_Buffers + 1];
GLuint queries [MAX_Queries + 1];
GLuint renderbuffers [MAX_Renderbuffers + 1];
GLuint samplers [MAX_Samplers + 1];
GLuint textures [MAX_Textures + 1];
GLuint programs [MAX_Programs + 1];
/* container objects */
GLuint framebuffers [MAX_Framebuffers + 1];
GLuint transformfeedbacks [MAX_TransformFeedbacks + 1];
GLuint vertexarrays [MAX_VertexArrays + 1];
/* NOTE: +1 necessary to avoid compile errors if MAX_... = 0 */
/******************************************************************************************************/
/* stencil test illustration */
/******************************************************************************************************/
bool stenciltest = false;
vec3 position_of_little_triangle;
/******************************************************************************************************/
/******************************************************************************************************/
int main(int argc, char* argv[])
{
/* Initialize GLFW */
if (!glfwInit())
return 1;
/* Create a window and its OpenGL context */
bool fullscreen = false;
glfwWindowHint(GLFW_CONTEXT_VERSION_MAJOR, 4);
glfwWindowHint(GLFW_CONTEXT_VERSION_MINOR, 5);
glfwWindowHint(GLFW_STENCIL_BITS, 8);
GLFWmonitor* monitor = fullscreen ? glfwGetPrimaryMonitor() : NULL;
GLFWwindow* window = glfwCreateWindow(800, 600, "OpenGL", monitor, NULL);
if (!window)
{
glfwTerminate();
return 1;
}
/* Make the window's context current */
glfwMakeContextCurrent(window);
/* Set V-sync */
bool vsync = true;
glfwSwapInterval(vsync ? 1 : 0);
/* Set callback functions */
glfwSetErrorCallback(error_callback);
glfwSetDropCallback(window, drop_callback);
glfwSetFramebufferSizeCallback(window, framebuffer_size_callback);
glfwSetKeyCallback(window, key_callback);
glfwSetCursorPosCallback(window, cursor_position_callback);
glfwSetMouseButtonCallback(window, mouse_button_callback);
glfwSetScrollCallback(window, scroll_callback);
/* Initialize GLEW */
if (glewInit() != GLEW_OK)
{
glfwTerminate();
return 2;
}
/* Initialize graphics */
if (!Initialize())
{
glfwTerminate();
return 3;
}
/* Loop until the user closes the window */
while (!glfwWindowShouldClose(window))
{
/* Render here */
Render();
/* Swap front and back buffers */
glfwSwapBuffers(window);
/* Poll for and process events */
glfwPollEvents();
}
/* Clean up graphics */
CleanUp();
glfwTerminate();
return 0;
}
/******************************************************************************************************/
/* input */
/******************************************************************************************************/
void error_callback(int error, const char* description)
{
cout << "GLFW Error: " << description << endl;
cin.get();
}
void drop_callback(GLFWwindow* window, int pathcount, const char** paths)
{
for (unsigned int i = 0; i < pathcount; i++)
cout << "dropped file: " << paths[i] << endl;
}
void framebuffer_size_callback(GLFWwindow* window, int width, int height)
{
glViewport(0, 0, width, height);
}
void key_callback(GLFWwindow* window, int key, int scancode, int action, int mods)
{
if (action == GLFW_PRESS)
{
if (key == GLFW_KEY_ESCAPE)
glfwSetWindowShouldClose(window, GLFW_TRUE);
/* stencil test: ON / OFF */
if (key == GLFW_KEY_SPACE)
{
stenciltest = !stenciltest;
cout << "stencil test: " << (stenciltest ? "on" : "off") << endl;
if (stenciltest)
glEnable(GL_STENCIL_TEST);
else
glDisable(GL_STENCIL_TEST);
}
}
}
void cursor_position_callback(GLFWwindow* window, double xpos, double ypos)
{
/* frame buffer size */
int size_x = 0, size_y = 0;
glfwGetFramebufferSize(window, &size_x, &size_y);
/* move triangle to cursor (screen space) */
position_of_little_triangle.x = -1.0f + 2.0f * xpos / size_x;
position_of_little_triangle.y = -1.0f + 2.0f * (size_y - ypos) / size_y;
}
void mouse_button_callback(GLFWwindow* window, int button, int action, int mods)
{
if (action == GLFW_PRESS)
{
if (button == GLFW_MOUSE_BUTTON_RIGHT)
{}
}
}
void scroll_callback(GLFWwindow* window, double xoffset, double yoffset)
{
/* scroll to change the depth of the little triangle */
position_of_little_triangle.z += (yoffset > 0) ? 0.1f : -0.1f;
if (position_of_little_triangle.z < 0.001f)
position_of_little_triangle.z = 0.001f;
if (position_of_little_triangle.z > 0.999f)
position_of_little_triangle.z = 0.999f;
cout << "little triangle depth: " << position_of_little_triangle.z << endl;
}
/******************************************************************************************************/
/* main functions */
/******************************************************************************************************/
bool Initialize()
{
GLint stencilbits = 0;
glGetNamedFramebufferAttachmentParameteriv(0, GL_STENCIL, GL_FRAMEBUFFER_ATTACHMENT_STENCIL_SIZE, &stencilbits);
if (stencilbits < 8)
{
cout << "error: stencil buffer has not enough bits (" << stencilbits << " bits)" << endl;
return false;
}
cout << "stencil buffer size: " << stencilbits << " bits" << endl;
/* create all objects */
glGenBuffers(MAX_Buffers, buffers);
glGenQueries(MAX_Queries, queries);
glGenRenderbuffers(MAX_Renderbuffers, renderbuffers);
glGenSamplers(MAX_Samplers, samplers);
glGenTextures(MAX_Textures, textures);
for (unsigned int i = 0; i < MAX_Programs; i++)
programs[i] = glCreateProgram();
glGenFramebuffers(MAX_Framebuffers, framebuffers);
glGenTransformFeedbacks(MAX_TransformFeedbacks, transformfeedbacks);
glGenVertexArrays(MAX_VertexArrays, vertexarrays);
/* setup program */
string vertexshader_source = {
"#version 450 core\n"
"layout (location = 0) uniform mat4 Transformation = mat4(1);"
"layout (location = 0) in vec3 in_position;"
"layout (location = 1) in vec4 in_color;"
"smooth out vec4 interpolated_color;"
"void main() {"
"gl_Position = Transformation * vec4(in_position, 1);"
"interpolated_color = in_color;"
"}"
};
string fragmentshader_source = {
"#version 450 core\n"
"smooth in vec4 interpolated_color;"
"layout (location = 0) out vec4 out_color;"
"void main() {"
"out_color = interpolated_color;"
"}"
};
bool success = true;
GLuint vertexshader = glCreateShader(GL_VERTEX_SHADER);
GLuint fragmentshader = glCreateShader(GL_FRAGMENT_SHADER);
if (!CompileShader(vertexshader, vertexshader_source))
success = false;
if (!CompileShader(fragmentshader, fragmentshader_source))
success = false;
if (!LinkProgram(programs[Program_Vertices], { vertexshader, fragmentshader }))
success = false;
glDeleteShader(vertexshader);
glDeleteShader(fragmentshader);
if (!success)
return false;
/* setup vertex array */
struct Vertex {
vec3 Position;
vec4 Color;
};
glBindVertexArray(vertexarrays[VertexArray_Vertices]);
glBindBuffer(GL_ARRAY_BUFFER, buffers[Buffer_Vertices]);
glVertexAttribPointer(0, 3, GL_FLOAT, GL_FALSE, sizeof(Vertex), reinterpret_cast<GLvoid*>(offsetof(Vertex, Position)));
glVertexAttribPointer(1, 4, GL_FLOAT, GL_FALSE, sizeof(Vertex), reinterpret_cast<GLvoid*>(offsetof(Vertex, Color)));
glBindBuffer(GL_ARRAY_BUFFER, 0);
glEnableVertexAttribArray(0);
glEnableVertexAttribArray(1);
glBindVertexArray(0);
/* setup vertex buffer */
vector<Vertex> vertices = {
{ { 0, 0, 0 }, { 1, 0, 0, 1 } },
{ { 1, 0, 0 }, { 0, 1, 0, 1 } },
{ { 0, 1, 0 }, { 0, 0, 1, 1 } },
};
glBindBuffer(GL_ARRAY_BUFFER, buffers[Buffer_Vertices]);
glBufferData(GL_ARRAY_BUFFER, sizeof(Vertex) * vertices.size(), vertices.data(), GL_STATIC_DRAW);
glBindBuffer(GL_ARRAY_BUFFER, 0);
/* enable depth + stencil test */
glEnable(GL_DEPTH_TEST);
glEnable(GL_STENCIL_TEST);
/* successfully initialized */
return true;
}
void Render()
{
/* clear frame buffer */
glClearColor(0, 0, 0.5f, 0);
glClear(GL_COLOR_BUFFER_BIT |GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
/* draw scene */
glUseProgram(programs[Program_Vertices]);
glBindVertexArray(vertexarrays[VertexArray_Vertices]);
/* little triangle: fill stencil buffer */
GLint stencilvalue = 0x01;
GLuint stencilmask = 0xFF;
glStencilFunc(GL_ALWAYS, stencilvalue, stencilmask);
glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
mat4 Transformation = translate(position_of_little_triangle) * scale(vec3(0.2f));
glUniformMatrix4fv(0, 1, GL_FALSE, value_ptr(Transformation));
glDrawArrays(GL_TRIANGLES, 0, 3);
/* big triangle: only check stencil buffer */
glStencilFunc(GL_NOTEQUAL, stencilvalue, stencilmask);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
Transformation = translate(vec3(0, 0, 0.5f));
glUniformMatrix4fv(0, 1, GL_FALSE, value_ptr(Transformation));
glDrawArrays(GL_TRIANGLES, 0, 3);
glBindVertexArray(0);
glUseProgram(0);
CheckForGLErrors
}
void CleanUp()
{
/* destroy all objects */
glDeleteBuffers(MAX_Buffers, buffers);
glDeleteQueries(MAX_Queries, queries);
glDeleteRenderbuffers(MAX_Renderbuffers, renderbuffers);
glDeleteSamplers(MAX_Samplers, samplers);
glDeleteTextures(MAX_Textures, textures);
for (unsigned int i = 0; i < MAX_Programs; i++)
{
glDeleteProgram(programs[i]);
programs[i] = 0;
}
glDeleteFramebuffers(MAX_Framebuffers, framebuffers);
glDeleteTransformFeedbacks(MAX_TransformFeedbacks, transformfeedbacks);
glDeleteVertexArrays(MAX_VertexArrays, vertexarrays);
}
void CheckForGLErrors__(const char* file, unsigned int line)
{
string errorstring;
for (GLenum error; (error = glGetError()) != GL_NO_ERROR;)
{
if (error == GL_INVALID_ENUM)
errorstring += " GL_INVALID_ENUM";
if (error == GL_INVALID_VALUE)
errorstring += " GL_INVALID_VALUE";
if (error == GL_INVALID_OPERATION)
errorstring += " GL_INVALID_OPERATION";
if (error == GL_STACK_OVERFLOW)
errorstring += " GL_STACK_OVERFLOW";
if (error == GL_STACK_UNDERFLOW)
errorstring += " GL_STACK_UNDERFLOW";
if (error == GL_OUT_OF_MEMORY)
errorstring += " GL_OUT_OF_MEMORY";
if (error == GL_INVALID_FRAMEBUFFER_OPERATION)
errorstring += " GL_INVALID_FRAMEBUFFER_OPERATION";
if (error == GL_CONTEXT_LOST)
errorstring += " GL_CONTEXT_LOST";
}
if (!errorstring.empty())
{
cout
<< (char)7
<< "OpenGL Error: " << endl
<< "\tLine: \t" << line << endl
<< "\tFile: \t" << file << endl
<< "\tErrors: \t" << errorstring << endl << endl;
cin.get();
}
}
Shader.h
#ifndef SHADER_H
#define SHADER_H
#include <GL/glew.h>
#include <string>
#include <list>
std::string LoadTextFile(const std::string& filepath);
std::string ShaderTypeName(GLuint shader);
bool CompileShader(GLuint shader, const std::string& sourcecode);
std::string ShaderInfoLog(GLuint shader);
bool LinkProgram(GLuint program, const std::list<GLuint>& shaderlist);
std::string ProgramInfoLog(GLuint program);
#endif // SHADER_H
Shader.cpp
#include "Shader.h"
#include <iostream>
#include <fstream>
std::string LoadTextFile(const std::string & filepath)
{
std::string result(""), line;
std::fstream f(filepath.c_str(), std::ios::in);
while (f.good())
{
std::getline(f, line);
result += line + '\n';
}
return result;
}
std::string ShaderTypeName(GLuint shader)
{
if (glIsShader(shader))
{
GLint type = 0;
glGetShaderiv(shader, GL_SHADER_TYPE, &type);
if (type == GL_VERTEX_SHADER)
return "Vertex Shader";
if (type == GL_TESS_CONTROL_SHADER)
return "Tessellation Control Shader";
if (type == GL_TESS_EVALUATION_SHADER)
return "Tessellation Evaluation Shader";
if (type == GL_GEOMETRY_SHADER)
return "Geometry Shader";
if (type == GL_FRAGMENT_SHADER)
return "Fragment Shader";
if (type == GL_COMPUTE_SHADER)
return "Compute Shader";
}
return "invalid shader";
}
bool CompileShader(GLuint shader, const std::string& sourcecode)
{
if (!glIsShader(shader))
{
std::cout << "ERROR: shader compilation failed, no valid shader specified" << std::endl;
return false;
}
if (sourcecode.empty())
{
std::cout << "ERROR: shader compilation failed, no source code specified (" << ShaderTypeName(shader) << ")" << std::endl;
return false;
}
const char* sourcearray[] = { sourcecode.c_str() };
glShaderSource(shader, 1, sourcearray, NULL);
glCompileShader(shader);
// check compile status
GLint status = 0;
glGetShaderiv(shader, GL_COMPILE_STATUS, &status);
// successfully compiled shader
if (status == GL_TRUE)
return true;
// show compile errors
std::cout << "ERROR: shader compilation failed (" << ShaderTypeName(shader) << ")" << std::endl << ShaderInfoLog(shader) << std::endl;
return false;
}
std::string ShaderInfoLog(GLuint shader)
{
if (glIsShader(shader))
{
GLint logsize = 0;
GLchar infolog[1024] = { 0 };
glGetShaderInfoLog(shader, 1024, &logsize, infolog);
return std::string(infolog);
}
return "invalid shader";
}
bool LinkProgram(GLuint program, const std::list<GLuint>& shaderlist)
{
if (!glIsProgram(program))
{
std::cout << "ERROR: shader linking failed, no valid program specified" << std::endl;
return false;
}
// attach all shaders to the program
for (std::list<GLuint>::const_iterator it = shaderlist.begin(); it != shaderlist.end(); it++)
{
if (glIsShader(*it))
glAttachShader(program, *it);
}
// link program
glLinkProgram(program);
// detach all shaders again
for (std::list<GLuint>::const_iterator it = shaderlist.begin(); it != shaderlist.end(); it++)
{
if (glIsShader(*it))
glDetachShader(program, *it);
}
GLint status = 0;
glGetProgramiv(program, GL_LINK_STATUS, &status);
// successfully linked program
if (status == GL_TRUE)
return true;
// show link errors
std::cout << "ERROR: shader linking failed" << std::endl << ProgramInfoLog(program) << std::endl;
return false;
}
std::string ProgramInfoLog(GLuint program)
{
if (glIsProgram(program))
{
GLint logsize = 0;
GLchar infolog[1024] = { 0 };
glGetProgramInfoLog(program, 1024, &logsize, infolog);
return std::string(infolog);
}
return "invalid program";
}