I'm trying to do this example.
I couldn't understand how it works still and I'm having difficulties in visualizing how the code would look like.
I stumbled upon this post which was the closest I think I could find.
From the redbook link, how can I do a "two-pass algorithm" to get the desired output? How do I use GL_INVERT and check if the pixel is covered even number of times?
I somehow managed to understand how the stencil works for the concave polygon but my problem now is the polygon is being rendered incorrectly.
http://img.photobucket.com/albums/v442/ardo/ScreenShot2013-12-30at24155PM.png
The location of where I am drawing this is in a framebuffer which is set up like this:
//multisample
glGenRenderbuffersEXT(1, &colorBuffer);
glBindRenderbufferEXT(GL_RENDERBUFFER_EXT, colorBuffer);
glRenderbufferStorageMultisampleEXT(GL_RENDERBUFFER_EXT, multisampling, GL_RGBA,800, 600);
//multi sample depth
glGenRenderbuffersEXT(1, &depthBuffer);
glBindRenderbufferEXT(GL_RENDERBUFFER_EXT, depthBuffer);
glRenderbufferStorageMultisampleEXT(GL_RENDERBUFFER_EXT, multisampling, GL_DEPTH24_STENCIL8, 800, 600);
//multisamplefbo
glGenFramebuffersEXT(1, &mFBO);
glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, mFBO);
glFramebufferRenderbufferEXT(GL_FRAMEBUFFER_EXT, GL_COLOR_ATTACHMENT0_EXT, GL_RENDERBUFFER_EXT, colorBuffer);
glFramebufferRenderbufferEXT(GL_FRAMEBUFFER_EXT, GL_DEPTH_STENCIL_ATTACHMENT, GL_RENDERBUFFER_EXT, depthBuffer);
Then my drawing code is:
glEnable(GL_STENCIL_TEST);
glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
glStencilFunc(GL_ALWAYS, 0x1, 0x1);
glStencilOp(GL_KEEP, GL_INVERT, GL_INVERT);
glBegin(GL_TRIANGLE_FAN);
glVertex3f(1.396900,3.130690,0);
glVertex3f(2.034830,2.466900,0);
glVertex3f(2.486338,2.441036,0);
glVertex3f(2.802204,2.437803,0);
glVertex3f(2.910181,2.447098,0);
glVertex3f(2.957240,2.466900,0);
glVertex3f(3.019335,2.552570,0);
glVertex3f(3.130733,2.673799,0);
glVertex3f(3.284830,2.828970,0);
glVertex3f(3.474490,3.087590,0);
glVertex3f(4.319320,2.596210,0);
glVertex3f(4.508980,2.147930,0);
glVertex3f(4.714798,1.866680,0);
glVertex3f(4.907215,1.620586,0);
glVertex3f(5.101651,1.397930,0);
glVertex3f(5.186114,1.317364,0);
glVertex3f(5.254869,1.269024,0);
glVertex3f(5.302510,1.261700,0);
glVertex3f(5.323633,1.304180,0);
glVertex3f(5.312832,1.405254,0);
glVertex3f(5.264703,1.573711,0);
glVertex3f(5.173841,1.818340,0);
glVertex3f(5.034840,2.147930,0);
glVertex3f(4.875695,2.492816,0);
glVertex3f(4.727900,2.780675,0);
glVertex3f(4.591307,3.016304,0);
glVertex3f(4.465765,3.204503,0);
glVertex3f(4.351127,3.350070,0);
glVertex3f(4.247241,3.457803,0);
glVertex3f(4.153958,3.532502,0);
glVertex3f(4.071129,3.578965,0);
glVertex3f(3.998605,3.601990,0);
glVertex3f(3.936235,3.606376,0);
glVertex3f(3.841363,3.578427,0);
glVertex3f(3.766900,3.510000,0);
glVertex3f(2.974490,3.458280,0);
glVertex3f(2.672770,2.958280,0);
glVertex3f(2.379670,3.070350,0);
glVertex3f(1.853810,3.303110,0);
glVertex3f(1.396910,3.303110,0);
glVertex3f(1.396910,3.130690,0);
glVertex3f(1.396900,3.130690,0);
glVertex3f(1.396900,3.130690,0);
glEnd();
glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
glStencilFunc(GL_EQUAL, 0x1, 0x1);
glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
glColor3f(1,0,1);
glBegin(GL_TRIANGLE_FAN);
glVertex3f(1.396900,3.130690,0);
glVertex3f(2.034830,2.466900,0);
glVertex3f(2.486338,2.441036,0);
glVertex3f(2.802204,2.437803,0);
glVertex3f(2.910181,2.447098,0);
glVertex3f(2.957240,2.466900,0);
glVertex3f(3.019335,2.552570,0);
glVertex3f(3.130733,2.673799,0);
glVertex3f(3.284830,2.828970,0);
glVertex3f(3.474490,3.087590,0);
glVertex3f(4.319320,2.596210,0);
glVertex3f(4.508980,2.147930,0);
glVertex3f(4.714798,1.866680,0);
glVertex3f(4.907215,1.620586,0);
glVertex3f(5.101651,1.397930,0);
glVertex3f(5.186114,1.317364,0);
glVertex3f(5.254869,1.269024,0);
glVertex3f(5.302510,1.261700,0);
glVertex3f(5.323633,1.304180,0);
glVertex3f(5.312832,1.405254,0);
glVertex3f(5.264703,1.573711,0);
glVertex3f(5.173841,1.818340,0);
glVertex3f(5.034840,2.147930,0);
glVertex3f(4.875695,2.492816,0);
glVertex3f(4.727900,2.780675,0);
glVertex3f(4.591307,3.016304,0);
glVertex3f(4.465765,3.204503,0);
glVertex3f(4.351127,3.350070,0);
glVertex3f(4.247241,3.457803,0);
glVertex3f(4.153958,3.532502,0);
glVertex3f(4.071129,3.578965,0);
glVertex3f(3.998605,3.601990,0);
glVertex3f(3.936235,3.606376,0);
glVertex3f(3.841363,3.578427,0);
glVertex3f(3.766900,3.510000,0);
glVertex3f(2.974490,3.458280,0);
glVertex3f(2.672770,2.958280,0);
glVertex3f(2.379670,3.070350,0);
glVertex3f(1.853810,3.303110,0);
glVertex3f(1.396910,3.303110,0);
glVertex3f(1.396910,3.130690,0);
glVertex3f(1.396900,3.130690,0);
glVertex3f(1.396900,3.130690,0);
glEnd();
glDisable(GL_STENCIL_TEST);
The way I have my multisampling set up is here (This is before the drawing is written)
[self checkError];
glEnable(GL_DEPTH_TEST);
[self checkError];
glDepthMask(GL_TRUE);
[self checkError];
glDepthFunc(GL_LESS);
[self checkError];
glClearDepth(10000.0);
[self checkError];
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
[self checkError];
if(multisampling != 0){
//Set multisampled framebuffer
glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, mFBO);
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
//Antialiasing functions
glEnable (GL_POLYGON_SMOOTH);
glEnable (GL_BLEND);
glBlendFunc (GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
glHint (GL_POLYGON_SMOOTH_HINT, GL_NICEST);
glLineWidth (1.5);
}
glDisable(GL_CULL_FACE);
And my blit call:
glBindFramebufferEXT(GL_READ_FRAMEBUFFER_EXT, mFBO);
glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER_EXT, currentFramebuffer);
glBlitFramebuffer(0, 0, 800, 600, 0, 0, 800, 600, GL_COLOR_BUFFER_BIT, GL_NEAREST);
glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, mFBO);
glReadBuffer(GL_COLOR_ATTACHMENT0_EXT);
glBindFramebufferEXT(GL_DRAW_FRAMEBUFFER_EXT, currentFramebuffer);
glBindFramebufferEXT(GL_READ_FRAMEBUFFER_EXT, currentFramebuffer);
glBindFramebufferEXT(GL_FRAMEBUFFER_EXT, currentFramebuffer);
What don't you get? The example starts out with a stencil buffer cleared to 0x00 and each time you draw a fragment (whether it passes or fails a depth/stencil test) it performs a bitwise inversion of the stencil buffer (e.g. ~0x00 --> 0xff). If you do this an odd number of times, the stencil buffer will be non-zero, but if you have an even number of fragments it will be zero.
Some things that might be confusing are the use of 1 as the bitmask for the stencil test and the stencil op in the second pass. This effectively limits the test to a single bit. In other words, the bit flipping I mentioned earlier will only happen for bit 1. Thus, you can actually simplify your test for even/odd to a test of a single bit... if the stencil buffer stores a value of 1 then you drew an odd number of fragments. If it stores a value of 0 then you drew an even number.
The second pass in your example actually does the exact stencil test I described. It tests the stencil buffer for 1 and fails the stencil test if the first bit != 1.