From 353a3e8c682ab2151271648b9246c1f509120813 Mon Sep 17 00:00:00 2001 From: SamGondelman Date: Wed, 9 Aug 2017 18:04:23 -0700 Subject: [PATCH] properly fix AA stenciling --- interface/src/ui/overlays/OverlaysPayload.cpp | 11 ++----- .../render-utils/src/AntialiasingEffect.cpp | 6 ++-- libraries/render-utils/src/GeometryCache.cpp | 23 +++++++------- libraries/render-utils/src/GeometryCache.h | 2 ++ .../render-utils/src/RenderDeferredTask.cpp | 31 ++++++------------- .../render-utils/src/RenderDeferredTask.h | 2 +- .../render-utils/src/StencilMaskPass.cpp | 12 ++++--- libraries/render-utils/src/StencilMaskPass.h | 8 +++-- libraries/render-utils/src/simple.slv | 3 +- libraries/render/src/render/FilterTask.cpp | 30 ------------------ libraries/render/src/render/FilterTask.h | 29 ----------------- 11 files changed, 43 insertions(+), 114 deletions(-) diff --git a/interface/src/ui/overlays/OverlaysPayload.cpp b/interface/src/ui/overlays/OverlaysPayload.cpp index 34b7d29b0d..8b7100205a 100644 --- a/interface/src/ui/overlays/OverlaysPayload.cpp +++ b/interface/src/ui/overlays/OverlaysPayload.cpp @@ -50,21 +50,16 @@ namespace render { } template <> int payloadGetLayer(const Overlay::Pointer& overlay) { // Magic number while we are defining the layering mechanism: - const int LAYER_NO_AA = 3; const int LAYER_2D = 2; const int LAYER_3D_FRONT = 1; const int LAYER_3D = 0; if (overlay->is3D()) { auto overlay3D = std::dynamic_pointer_cast(overlay); - if (overlay3D->isAA()) { - if (overlay3D->getDrawInFront()) { - return LAYER_3D_FRONT; - } else { - return LAYER_3D; - } + if (overlay3D->getDrawInFront()) { + return LAYER_3D_FRONT; } else { - return LAYER_NO_AA; + return LAYER_3D; } } else { return LAYER_2D; diff --git a/libraries/render-utils/src/AntialiasingEffect.cpp b/libraries/render-utils/src/AntialiasingEffect.cpp index f4d2bc5e40..3013ad9ebb 100644 --- a/libraries/render-utils/src/AntialiasingEffect.cpp +++ b/libraries/render-utils/src/AntialiasingEffect.cpp @@ -70,7 +70,7 @@ const gpu::PipelinePointer& Antialiasing::getAntialiasingPipeline(RenderArgs* ar gpu::StatePointer state = gpu::StatePointer(new gpu::State()); - PrepareStencil::testMask(*state); + PrepareStencil::testMaskNoAA(*state); state->setDepthTest(false, false, gpu::LESS_EQUAL); @@ -95,7 +95,7 @@ const gpu::PipelinePointer& Antialiasing::getBlendPipeline() { gpu::StatePointer state = gpu::StatePointer(new gpu::State()); state->setDepthTest(false, false, gpu::LESS_EQUAL); - PrepareStencil::testMask(*state); + PrepareStencil::testMaskNoAA(*state); // Good to go add the brand new pipeline _blendPipeline = gpu::Pipeline::create(program, state); @@ -131,7 +131,6 @@ void Antialiasing::run(const render::RenderContextPointer& renderContext, const // FXAA step auto pipeline = getAntialiasingPipeline(renderContext->args); - _antialiasingBuffer->setDepthStencilBuffer(sourceBuffer->getDepthStencilBuffer(), sourceBuffer->getDepthStencilBufferFormat()); batch.setResourceTexture(0, sourceBuffer->getRenderBuffer(0)); batch.setFramebuffer(_antialiasingBuffer); batch.setPipeline(pipeline); @@ -160,7 +159,6 @@ void Antialiasing::run(const render::RenderContextPointer& renderContext, const // Blend step - getBlendPipeline(); batch.setResourceTexture(0, _antialiasingTexture); batch.setFramebuffer(sourceBuffer); batch.setPipeline(getBlendPipeline()); diff --git a/libraries/render-utils/src/GeometryCache.cpp b/libraries/render-utils/src/GeometryCache.cpp index 60b0bd8adb..3f43d3173f 100644 --- a/libraries/render-utils/src/GeometryCache.cpp +++ b/libraries/render-utils/src/GeometryCache.cpp @@ -1718,7 +1718,7 @@ void GeometryCache::useSimpleDrawPipeline(gpu::Batch& batch, bool noBlend) { auto stateNoBlend = std::make_shared(); - PrepareStencil::testMaskDrawShape(*state); + PrepareStencil::testMaskDrawShape(*stateNoBlend); auto noBlendPS = gpu::StandardShaderLib::getDrawTextureOpaquePS(); auto programNoBlend = gpu::Shader::createProgram(vs, noBlendPS); @@ -1827,14 +1827,11 @@ static void buildWebShader(const std::string& vertShaderText, const std::string& state->setBlendFunction(blendEnable, gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA, gpu::State::FACTOR_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::ONE); - if (blendEnable) { - PrepareStencil::testMask(*state); - } else { - PrepareStencil::testMaskDrawShape(*state); - } - if (!isAA) { - PrepareStencil::drawMaskDepth(*state); + if (isAA) { + blendEnable ? PrepareStencil::testMask(*state) : PrepareStencil::testMaskDrawShape(*state); + } else { + PrepareStencil::testMaskDrawShapeNoAA(*state); } pipelinePointerOut = gpu::Pipeline::create(shaderPointerOut, state); @@ -1848,10 +1845,11 @@ gpu::PipelinePointer GeometryCache::getOpaqueWebBrowserProgram(bool isAA) { static std::once_flag once; std::call_once(once, [&]() { const bool BLEND_ENABLE = false; - buildWebShader(simple_vert, simple_opaque_web_browser_frag, BLEND_ENABLE, isAA, _simpleOpaqueWebBrowserShader, _simpleOpaqueWebBrowserPipeline); + buildWebShader(simple_vert, simple_opaque_web_browser_frag, BLEND_ENABLE, true, _simpleOpaqueWebBrowserShader, _simpleOpaqueWebBrowserPipeline); + buildWebShader(simple_vert, simple_opaque_web_browser_frag, BLEND_ENABLE, false, _simpleOpaqueWebBrowserShader, _simpleOpaqueWebBrowserPipelineNoAA); }); - return _simpleOpaqueWebBrowserPipeline; + return isAA ? _simpleOpaqueWebBrowserPipeline : _simpleOpaqueWebBrowserPipelineNoAA; } void GeometryCache::bindTransparentWebBrowserProgram(gpu::Batch& batch, bool isAA) { @@ -1862,10 +1860,11 @@ gpu::PipelinePointer GeometryCache::getTransparentWebBrowserProgram(bool isAA) { static std::once_flag once; std::call_once(once, [&]() { const bool BLEND_ENABLE = true; - buildWebShader(simple_vert, simple_transparent_web_browser_frag, BLEND_ENABLE, isAA, _simpleTransparentWebBrowserShader, _simpleTransparentWebBrowserPipeline); + buildWebShader(simple_vert, simple_transparent_web_browser_frag, BLEND_ENABLE, true, _simpleTransparentWebBrowserShader, _simpleTransparentWebBrowserPipeline); + buildWebShader(simple_vert, simple_transparent_web_browser_frag, BLEND_ENABLE, false, _simpleTransparentWebBrowserShader, _simpleTransparentWebBrowserPipelineNoAA); }); - return _simpleTransparentWebBrowserPipeline; + return isAA ? _simpleTransparentWebBrowserPipeline : _simpleTransparentWebBrowserPipelineNoAA; } void GeometryCache::bindSimpleProgram(gpu::Batch& batch, bool textured, bool transparent, bool culled, bool unlit, bool depthBiased) { diff --git a/libraries/render-utils/src/GeometryCache.h b/libraries/render-utils/src/GeometryCache.h index 69759b311e..77effc8969 100644 --- a/libraries/render-utils/src/GeometryCache.h +++ b/libraries/render-utils/src/GeometryCache.h @@ -428,8 +428,10 @@ private: gpu::ShaderPointer _simpleOpaqueWebBrowserShader; gpu::PipelinePointer _simpleOpaqueWebBrowserPipeline; + gpu::PipelinePointer _simpleOpaqueWebBrowserPipelineNoAA; gpu::ShaderPointer _simpleTransparentWebBrowserShader; gpu::PipelinePointer _simpleTransparentWebBrowserPipeline; + gpu::PipelinePointer _simpleTransparentWebBrowserPipelineNoAA; }; #endif // hifi_GeometryCache_h diff --git a/libraries/render-utils/src/RenderDeferredTask.cpp b/libraries/render-utils/src/RenderDeferredTask.cpp index 547e9de5b8..27d2b54121 100644 --- a/libraries/render-utils/src/RenderDeferredTask.cpp +++ b/libraries/render-utils/src/RenderDeferredTask.cpp @@ -64,13 +64,6 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren const auto background = items[RenderFetchCullSortTask::BACKGROUND]; const auto spatialSelection = items[RenderFetchCullSortTask::SPATIAL_SELECTION]; - // Filter the non antialiaased overlays - const int LAYER_NO_AA = 3; - const auto overlaysOpaqueAA = task.addJob("FilterAAOverlaysOpaque", overlayOpaques, LAYER_NO_AA); - const auto overlaysOpaqueNonAA = task.addJob("FilterNonAAOverlaysOpaque", overlayOpaques, LAYER_NO_AA); - const auto overlaysTransparentAA = task.addJob("FilterAAOverlaysTransparent", overlayTransparents, LAYER_NO_AA); - const auto overlaysTransparentNonAA = task.addJob("FilterNonAAOverlaysTransparent", overlayTransparents, LAYER_NO_AA); - // Prepare deferred, generate the shared Deferred Frame Transform const auto deferredFrameTransform = task.addJob("DeferredFrameTransform"); const auto lightingModel = task.addJob("LightingModel"); @@ -157,7 +150,7 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren const auto toneMappingInputs = render::Varying(ToneMappingDeferred::Inputs(lightingFramebuffer, primaryFramebuffer)); task.addJob("ToneMapping", toneMappingInputs); - { // DEbug the bounds of the rendered items, still look at the zbuffer + { // Debug the bounds of the rendered items, still look at the zbuffer task.addJob("DrawMetaBounds", metas); task.addJob("DrawOpaqueBounds", opaques); task.addJob("DrawTransparentBounds", transparents); @@ -167,22 +160,16 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren } // Overlays - const auto overlaysOpaqueAAInputs = DrawOverlay3D::Inputs(overlaysOpaqueAA, lightingModel).hasVarying(); - const auto overlaysOpaqueNonAAInputs = DrawOverlay3D::Inputs(overlaysOpaqueNonAA, lightingModel).hasVarying(); - const auto overlaysTransparentAAInputs = DrawOverlay3D::Inputs(overlaysTransparentAA, lightingModel).hasVarying(); - const auto overlaysTransparentNonAAInputs = DrawOverlay3D::Inputs(overlaysTransparentNonAA, lightingModel).hasVarying(); - task.addJob("DrawOverlay3DOpaqueAA", overlaysOpaqueAAInputs, true); - task.addJob("DrawOverlay3DOpaqueNonAA", overlaysOpaqueNonAAInputs, true); - task.addJob("DrawOverlay3DTransparentAA", overlaysTransparentAAInputs, false); - task.addJob("DrawOverlay3DTransparentNonAA", overlaysTransparentNonAAInputs, false); + const auto overlayOpaquesInputs = DrawOverlay3D::Inputs(overlayOpaques, lightingModel).hasVarying(); + const auto overlayTransparentsInputs = DrawOverlay3D::Inputs(overlayTransparents, lightingModel).hasVarying(); + task.addJob("DrawOverlay3DOpaque", overlayOpaquesInputs, true); + task.addJob("DrawOverlay3DTransparent", overlayTransparentsInputs, false); - { // DEbug the bounds of the rendered OVERLAY items, still look at the zbuffer - task.addJob("DrawOverlayOpaqueBounds", overlaysOpaqueAA); - task.addJob("DrawOverlayOpaqueNonAABounds", overlaysOpaqueNonAA); - task.addJob("DrawOverlayTransparentAABounds", overlaysTransparentAA); - task.addJob("DrawOverlayTransparentNonAABounds", overlaysTransparentNonAA); + { // Debug the bounds of the rendered Overlay items, still look at the zbuffer + task.addJob("DrawOverlayOpaqueBounds", overlayOpaques); + task.addJob("DrawOverlayTransparentBounds", overlayTransparents); } - + // Debugging stages { // Debugging Deferred buffer job diff --git a/libraries/render-utils/src/RenderDeferredTask.h b/libraries/render-utils/src/RenderDeferredTask.h index fd7c5eb23b..dccd539229 100644 --- a/libraries/render-utils/src/RenderDeferredTask.h +++ b/libraries/render-utils/src/RenderDeferredTask.h @@ -153,7 +153,7 @@ public: protected: render::ShapePlumberPointer _shapePlumber; int _maxDrawn; // initialized by Config - bool _opaquePass{ true }; + bool _opaquePass { true }; }; class Blit { diff --git a/libraries/render-utils/src/StencilMaskPass.cpp b/libraries/render-utils/src/StencilMaskPass.cpp index 59e4cc6e59..295e124ed1 100644 --- a/libraries/render-utils/src/StencilMaskPass.cpp +++ b/libraries/render-utils/src/StencilMaskPass.cpp @@ -108,14 +108,14 @@ void PrepareStencil::drawMask(gpu::State& state) { state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::ALWAYS, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE)); } -void PrepareStencil::drawMaskDepth(gpu::State& state) { - state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::ALWAYS, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_REPLACE)); -} - void PrepareStencil::testMask(gpu::State& state) { state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::NOT_EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP)); } +void PrepareStencil::testMaskNoAA(gpu::State& state) { + state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK | PrepareStencil::STENCIL_NO_AA, 0xFF, gpu::NOT_EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP)); +} + void PrepareStencil::testBackground(gpu::State& state) { state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_BACKGROUND, 0xFF, gpu::EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP)); } @@ -124,6 +124,10 @@ void PrepareStencil::testMaskDrawShape(gpu::State& state) { state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::NOT_EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_ZERO)); } +void PrepareStencil::testMaskDrawShapeNoAA(gpu::State& state) { + state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK | PrepareStencil::STENCIL_NO_AA, 0xFF, gpu::ALWAYS, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_REPLACE)); +} + void PrepareStencil::testShape(gpu::State& state) { state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_SHAPE, 0xFF, gpu::EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP)); } \ No newline at end of file diff --git a/libraries/render-utils/src/StencilMaskPass.h b/libraries/render-utils/src/StencilMaskPass.h index 19224369ff..2c0294c471 100644 --- a/libraries/render-utils/src/StencilMaskPass.h +++ b/libraries/render-utils/src/StencilMaskPass.h @@ -41,16 +41,18 @@ public: void run(const render::RenderContextPointer& renderContext, const gpu::FramebufferPointer& dstFramebuffer); - static const gpu::int8 STENCIL_MASK = 2; - static const gpu::int8 STENCIL_BACKGROUND = 1; static const gpu::int8 STENCIL_SHAPE = 0; + static const gpu::int8 STENCIL_BACKGROUND = 1 << 0; + static const gpu::int8 STENCIL_MASK = 1 << 1; + static const gpu::int8 STENCIL_NO_AA = 1 << 2; static void drawMask(gpu::State& state); - static void drawMaskDepth(gpu::State& state); static void testMask(gpu::State& state); + static void testMaskNoAA(gpu::State& state); static void testBackground(gpu::State& state); static void testMaskDrawShape(gpu::State& state); + static void testMaskDrawShapeNoAA(gpu::State& state); static void testShape(gpu::State& state); diff --git a/libraries/render-utils/src/simple.slv b/libraries/render-utils/src/simple.slv index c9fa761ebc..64d3e24192 100644 --- a/libraries/render-utils/src/simple.slv +++ b/libraries/render-utils/src/simple.slv @@ -27,11 +27,12 @@ out vec4 _position; void main(void) { _color = colorToLinearRGBA(inColor); _texCoord0 = inTexCoord0.st; + _position = inPosition; _modelNormal = inNormal.xyz; // standard transform TransformCamera cam = getTransformCamera(); TransformObject obj = getTransformObject(); - <$transformModelToEyeAndClipPos(cam, obj, inPosition, _position, gl_Position)$> + <$transformModelToClipPos(cam, obj, inPosition, gl_Position)$> <$transformModelToWorldDir(cam, obj, inNormal.xyz, _normal)$> } \ No newline at end of file diff --git a/libraries/render/src/render/FilterTask.cpp b/libraries/render/src/render/FilterTask.cpp index fe3aea94f6..252006c6e7 100644 --- a/libraries/render/src/render/FilterTask.cpp +++ b/libraries/render/src/render/FilterTask.cpp @@ -21,36 +21,6 @@ using namespace render; -void FilterLayeredItems::run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems) { - auto& scene = renderContext->_scene; - - // Clear previous values - outItems.clear(); - - // Filter matches into one bucket - for (auto itemBound : inItems) { - auto& item = scene->getItem(itemBound.id); - if (item.getLayer() == _keepLayer) { - outItems.emplace_back(itemBound); - } - } -} - -void FilterOutLayeredItems::run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems) { - auto& scene = renderContext->_scene; - - // Clear previous values - outItems.clear(); - - // Filter non-matches into one bucket - for (auto itemBound : inItems) { - auto& item = scene->getItem(itemBound.id); - if (item.getLayer() != _removeLayer) { - outItems.emplace_back(itemBound); - } - } -} - void SliceItems::run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems) { outItems.clear(); std::static_pointer_cast(renderContext->jobConfig)->setNumItems((int)inItems.size()); diff --git a/libraries/render/src/render/FilterTask.h b/libraries/render/src/render/FilterTask.h index 19883e03cd..6d30f46485 100644 --- a/libraries/render/src/render/FilterTask.h +++ b/libraries/render/src/render/FilterTask.h @@ -62,35 +62,6 @@ namespace render { } }; - // Filter the items belonging to the job's keep layer - class FilterLayeredItems { - public: - using JobModel = Job::ModelIO; - - FilterLayeredItems() {} - FilterLayeredItems(int keepLayer) : - _keepLayer(keepLayer) {} - - int _keepLayer { 0 }; - - void run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems); - }; - - // Filter the items not belonging to the job's remove layer - class FilterOutLayeredItems { - public: - using JobModel = Job::ModelIO; - - FilterOutLayeredItems() {} - FilterOutLayeredItems(int removeLayer) : - _removeLayer(removeLayer) { - } - - int _removeLayer { 0 }; - - void run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems); - }; - // SliceItems job config defining the slice range class SliceItemsConfig : public Job::Config { Q_OBJECT