working on fixing non-AA overlays

interface/src/ui/overlays/OverlaysPayload.cpp

@@ -37,9 +37,6 @@ namespace render {
             if (std::static_pointer_cast<Base3DOverlay>(overlay)->getDrawInFront()) {
                 builder.withLayered();
             }
-            if (!std::static_pointer_cast<Base3DOverlay>(overlay)->isAA()) {
-                builder.withLayered();
-            }
             if (overlay->getAlphaPulse() != 0.0f || overlay->getAlpha() != 1.0f) {
                 builder.withTransparent();
             }
@@ -60,14 +57,15 @@ namespace render {
 
         if (overlay->is3D()) {
             auto overlay3D = std::dynamic_pointer_cast<Base3DOverlay>(overlay);
-            if (overlay3D->isAA())
+            if (overlay3D->isAA()) {
                 if (overlay3D->getDrawInFront()) {
                     return LAYER_3D_FRONT;
                 } else {
                     return LAYER_3D;
                 }
-            else
+            } else {
                 return LAYER_NO_AA;
+            }
         } else {
             return LAYER_2D;
         }

libraries/render-utils/src/AntialiasingEffect.cpp

@@ -131,6 +131,7 @@ void Antialiasing::run(const render::RenderContextPointer& renderContext, const
 
         // FXAA step
         auto pipeline = getAntialiasingPipeline(renderContext->args);
+        _antialiasingBuffer->setDepthStencilBuffer(sourceBuffer->getDepthStencilBuffer(), sourceBuffer->getDepthStencilBufferFormat());
         batch.setResourceTexture(0, sourceBuffer->getRenderBuffer(0));
         batch.setFramebuffer(_antialiasingBuffer);
         batch.setPipeline(pipeline);

libraries/render-utils/src/GeometryCache.cpp

@@ -38,9 +38,7 @@
 #include "simple_textured_frag.h"
 #include "simple_textured_unlit_frag.h"
 #include "simple_opaque_web_browser_frag.h"
-#include "simple_opaque_web_browser_overlay_frag.h"
 #include "simple_transparent_web_browser_frag.h"
-#include "simple_transparent_web_browser_overlay_frag.h"
 #include "glowLine_vert.h"
 #include "glowLine_frag.h"
 
@@ -1814,7 +1812,7 @@ inline bool operator==(const SimpleProgramKey& a, const SimpleProgramKey& b) {
     return a.getRaw() == b.getRaw();
 }
 
-static void buildWebShader(const std::string& vertShaderText, const std::string& fragShaderText, bool blendEnable,
+static void buildWebShader(const std::string& vertShaderText, const std::string& fragShaderText, bool blendEnable, bool isAA,
                            gpu::ShaderPointer& shaderPointerOut, gpu::PipelinePointer& pipelinePointerOut) {
     auto VS = gpu::Shader::createVertex(vertShaderText);
     auto PS = gpu::Shader::createPixel(fragShaderText);
@@ -1835,6 +1833,10 @@ static void buildWebShader(const std::string& vertShaderText, const std::string&
         PrepareStencil::testMaskDrawShape(*state);
     }
 
+    if (!isAA) {
+        PrepareStencil::drawMaskDepth(*state);
+    }
+
     pipelinePointerOut = gpu::Pipeline::create(shaderPointerOut, state);
 }
 
@@ -1846,11 +1848,10 @@ gpu::PipelinePointer GeometryCache::getOpaqueWebBrowserProgram(bool isAA) {
     static std::once_flag once;
     std::call_once(once, [&]() {
         const bool BLEND_ENABLE = false;
-        buildWebShader(simple_vert, simple_opaque_web_browser_frag, BLEND_ENABLE, _simpleOpaqueWebBrowserShader, _simpleOpaqueWebBrowserPipeline);
+        buildWebShader(simple_vert, simple_opaque_web_browser_frag, BLEND_ENABLE, isAA, _simpleOpaqueWebBrowserShader, _simpleOpaqueWebBrowserPipeline);
-        buildWebShader(simple_vert, simple_opaque_web_browser_overlay_frag, BLEND_ENABLE, _simpleOpaqueWebBrowserOverlayShader, _simpleOpaqueWebBrowserOverlayPipeline);
     });
 
-    return isAA ? _simpleOpaqueWebBrowserPipeline : _simpleOpaqueWebBrowserOverlayPipeline;
+    return _simpleOpaqueWebBrowserPipeline;
 }
 
 void GeometryCache::bindTransparentWebBrowserProgram(gpu::Batch& batch, bool isAA) {
@@ -1860,13 +1861,11 @@ void GeometryCache::bindTransparentWebBrowserProgram(gpu::Batch& batch, bool isA
 gpu::PipelinePointer GeometryCache::getTransparentWebBrowserProgram(bool isAA) {
     static std::once_flag once;
     std::call_once(once, [&]() {
-
         const bool BLEND_ENABLE = true;
-        buildWebShader(simple_vert, simple_transparent_web_browser_frag, BLEND_ENABLE, _simpleTransparentWebBrowserShader, _simpleTransparentWebBrowserPipeline);
+        buildWebShader(simple_vert, simple_transparent_web_browser_frag, BLEND_ENABLE, isAA, _simpleTransparentWebBrowserShader, _simpleTransparentWebBrowserPipeline);
-        buildWebShader(simple_vert, simple_transparent_web_browser_overlay_frag, BLEND_ENABLE, _simpleTransparentWebBrowserOverlayShader, _simpleTransparentWebBrowserOverlayPipeline);
     });
 
-    return isAA ? _simpleTransparentWebBrowserPipeline : _simpleTransparentWebBrowserOverlayPipeline;
+    return _simpleTransparentWebBrowserPipeline;
 }
 
 void GeometryCache::bindSimpleProgram(gpu::Batch& batch, bool textured, bool transparent, bool culled, bool unlit, bool depthBiased) {

libraries/render-utils/src/GeometryCache.h

@@ -430,11 +430,6 @@ private:
     gpu::PipelinePointer _simpleOpaqueWebBrowserPipeline;
     gpu::ShaderPointer _simpleTransparentWebBrowserShader;
     gpu::PipelinePointer _simpleTransparentWebBrowserPipeline;
-
-    gpu::ShaderPointer _simpleOpaqueWebBrowserOverlayShader;
-    gpu::PipelinePointer _simpleOpaqueWebBrowserOverlayPipeline;
-    gpu::ShaderPointer _simpleTransparentWebBrowserOverlayShader;
-    gpu::PipelinePointer _simpleTransparentWebBrowserOverlayPipeline;
 };
 
 #endif // hifi_GeometryCache_h

libraries/render-utils/src/RenderDeferredTask.cpp

@@ -66,7 +66,10 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren
 
     // Filter the non antialiaased overlays
     const int LAYER_NO_AA = 3;
-    const auto nonAAOverlays = task.addJob<FilterLayeredItems>("Filter2DWebOverlays", overlayOpaques, LAYER_NO_AA);
+    const auto overlaysOpaqueAA = task.addJob<FilterOutLayeredItems>("FilterAAOverlaysOpaque", overlayOpaques, LAYER_NO_AA);
+    const auto overlaysOpaqueNonAA = task.addJob<FilterLayeredItems>("FilterNonAAOverlaysOpaque", overlayOpaques, LAYER_NO_AA);
+    const auto overlaysTransparentAA = task.addJob<FilterOutLayeredItems>("FilterAAOverlaysTransparent", overlayTransparents, LAYER_NO_AA);
+    const auto overlaysTransparentNonAA = task.addJob<FilterLayeredItems>("FilterNonAAOverlaysTransparent", overlayTransparents, LAYER_NO_AA);
 
     // Prepare deferred, generate the shared Deferred Frame Transform
     const auto deferredFrameTransform = task.addJob<GenerateDeferredFrameTransform>("DeferredFrameTransform");
@@ -164,14 +167,20 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren
     }
 
     // Overlays
-    const auto overlayOpaquesInputs = DrawOverlay3D::Inputs(overlayOpaques, lightingModel).hasVarying();
+    const auto overlaysOpaqueAAInputs = DrawOverlay3D::Inputs(overlaysOpaqueAA, lightingModel).hasVarying();
-    const auto overlayTransparentsInputs = DrawOverlay3D::Inputs(overlayTransparents, lightingModel).hasVarying();
+    const auto overlaysOpaqueNonAAInputs = DrawOverlay3D::Inputs(overlaysOpaqueNonAA, lightingModel).hasVarying();
-    task.addJob<DrawOverlay3D>("DrawOverlay3DOpaque", overlayOpaquesInputs, true);
+    const auto overlaysTransparentAAInputs = DrawOverlay3D::Inputs(overlaysTransparentAA, lightingModel).hasVarying();
-    task.addJob<DrawOverlay3D>("DrawOverlay3DTransparent", overlayTransparentsInputs, false);
+    const auto overlaysTransparentNonAAInputs = DrawOverlay3D::Inputs(overlaysTransparentNonAA, lightingModel).hasVarying();
+    task.addJob<DrawOverlay3D>("DrawOverlay3DOpaqueAA", overlaysOpaqueAAInputs, true);
+    task.addJob<DrawOverlay3D>("DrawOverlay3DOpaqueNonAA", overlaysOpaqueNonAAInputs, true);
+    task.addJob<DrawOverlay3D>("DrawOverlay3DTransparentAA", overlaysTransparentAAInputs, false);
+    task.addJob<DrawOverlay3D>("DrawOverlay3DTransparentNonAA", overlaysTransparentNonAAInputs, false);
 
     { // DEbug the bounds of the rendered OVERLAY items, still look at the zbuffer
-        task.addJob<DrawBounds>("DrawOverlayOpaqueBounds", overlayOpaques);
+        task.addJob<DrawBounds>("DrawOverlayOpaqueBounds", overlaysOpaqueAA);
-        task.addJob<DrawBounds>("DrawOverlayTransparentBounds", overlayTransparents);
+        task.addJob<DrawBounds>("DrawOverlayOpaqueNonAABounds", overlaysOpaqueNonAA);
+        task.addJob<DrawBounds>("DrawOverlayTransparentAABounds", overlaysTransparentAA);
+        task.addJob<DrawBounds>("DrawOverlayTransparentNonAABounds", overlaysTransparentNonAA);
     }
     
      // Debugging stages
@@ -204,14 +213,9 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren
         task.addJob<DebugZoneLighting>("DrawZoneStack", deferredFrameTransform);
     }
 
-
     // AA job to be revisited
     task.addJob<Antialiasing>("Antialiasing", primaryFramebuffer);
 
-    // Draw 2DWeb non AA
-    const auto nonAAOverlaysInputs = DrawOverlay3D::Inputs(nonAAOverlays, lightingModel).hasVarying();
-    task.addJob<DrawOverlay3D>("Draw2DWebSurfaces", nonAAOverlaysInputs, false);
-
     task.addJob<EndGPURangeTimer>("ToneAndPostRangeTimer", toneAndPostRangeTimer);
 
     // Blit!

libraries/render-utils/src/StencilMaskPass.cpp

@@ -108,6 +108,10 @@ void PrepareStencil::drawMask(gpu::State& state) {
     state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::ALWAYS, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE));
 }
 
+void PrepareStencil::drawMaskDepth(gpu::State& state) {
+    state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::ALWAYS, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_REPLACE));
+}
+
 void PrepareStencil::testMask(gpu::State& state) {
     state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::NOT_EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
 }

libraries/render-utils/src/StencilMaskPass.h

@@ -47,6 +47,7 @@ public:
 
 
     static void drawMask(gpu::State& state);
+    static void drawMaskDepth(gpu::State& state);
     static void testMask(gpu::State& state);
     static void testBackground(gpu::State& state);
     static void testMaskDrawShape(gpu::State& state);

libraries/render-utils/src/simple.slv

@@ -27,12 +27,11 @@ out vec4 _position;
 void main(void) {
     _color = colorToLinearRGBA(inColor);
     _texCoord0 = inTexCoord0.st;
-    _position = inPosition;
     _modelNormal = inNormal.xyz;
 
     // standard transform
     TransformCamera cam = getTransformCamera();
     TransformObject obj = getTransformObject();
-    <$transformModelToClipPos(cam, obj, inPosition, gl_Position)$>
+    <$transformModelToEyeAndClipPos(cam, obj, inPosition, _position, gl_Position)$>
     <$transformModelToWorldDir(cam, obj, inNormal.xyz, _normal)$>
 }

libraries/render-utils/src/simple_opaque_web_browser_overlay.slf

@@ -1,30 +0,0 @@
-<@include gpu/Config.slh@>
-<$VERSION_HEADER$>
-//  Generated on <$_SCRIBE_DATE$>
-//
-//  simple_opaque_web_browser_overlay.slf
-//  fragment shader
-//
-//  Created by Anthony Thibault on 1/30/17.
-//  Copyright 2017 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-//  Same as simple_opaque_web_browser.slf except frame buffer is sRGB, so colorToLinearRGBA is not necessary.
-
-<@include gpu/Color.slh@>
-<@include DeferredBufferWrite.slh@>
-
-// the albedo texture
-uniform sampler2D originalTexture;
-
-// the interpolated normal
-in vec3 _normal;
-in vec4 _color;
-in vec2 _texCoord0;
-
-void main(void) {
-    vec4 texel = texture(originalTexture, _texCoord0.st);
-    _fragColor0 = vec4(_color.rgb * texel.rgb, 1.0);
-}

libraries/render-utils/src/simple_transparent_web_browser_overlay.slf

@@ -1,31 +0,0 @@
-<@include gpu/Config.slh@>
-<$VERSION_HEADER$>
-//  Generated on <$_SCRIBE_DATE$>
-//
-//  simple_transparent_web_browser_overlay.slf
-//  fragment shader
-//
-//  Created by Anthony Thibault on 1/30/17.
-//  Copyright 2017 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-//  Same as simple_transparent_web_browser.slf except frame buffer is sRGB, So colorToLinearRGBA is not necessary.
-//
-
-<@include gpu/Color.slh@>
-<@include DeferredBufferWrite.slh@>
-
-// the albedo texture
-uniform sampler2D originalTexture;
-
-// the interpolated normal
-in vec3 _normal;
-in vec4 _color;
-in vec2 _texCoord0;
-
-void main(void) {
-    vec4 texel = texture(originalTexture, _texCoord0.st);
-    _fragColor0 = vec4(_color.rgb * texel.rgb, _color.a);
-}

libraries/render/src/render/FilterTask.cpp

@@ -27,7 +27,7 @@ void FilterLayeredItems::run(const RenderContextPointer& renderContext, const It
     // Clear previous values
     outItems.clear();
 
-    // For each item, filter it into one bucket
+    // Filter matches into one bucket
     for (auto itemBound : inItems) {
         auto& item = scene->getItem(itemBound.id);
         if (item.getLayer() == _keepLayer) {
@@ -36,6 +36,21 @@ void FilterLayeredItems::run(const RenderContextPointer& renderContext, const It
     }
 }
 
+void FilterOutLayeredItems::run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems) {
+    auto& scene = renderContext->_scene;
+
+    // Clear previous values
+    outItems.clear();
+
+    // Filter non-matches into one bucket
+    for (auto itemBound : inItems) {
+        auto& item = scene->getItem(itemBound.id);
+        if (item.getLayer() != _removeLayer) {
+            outItems.emplace_back(itemBound);
+        }
+    }
+}
+
 void SliceItems::run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems) {
     outItems.clear();
     std::static_pointer_cast<Config>(renderContext->jobConfig)->setNumItems((int)inItems.size());

libraries/render/src/render/FilterTask.h

@@ -76,6 +76,21 @@ namespace render {
         void run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems);
     };
 
+    // Filter the items not belonging to the job's remove layer
+    class FilterOutLayeredItems {
+    public:
+        using JobModel = Job::ModelIO<FilterOutLayeredItems, ItemBounds, ItemBounds>;
+
+        FilterOutLayeredItems() {}
+        FilterOutLayeredItems(int removeLayer) :
+            _removeLayer(removeLayer) {
+        }
+
+        int _removeLayer { 0 };
+
+        void run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems);
+    };
+
     // SliceItems job config defining the slice range
     class SliceItemsConfig : public Job::Config {
         Q_OBJECT