Merge pull request #11345 from SamGondelman/textAA

Text3D Overlays and Entities fixes and no-AA

interface/src/ui/overlays/Text3DOverlay.cpp

@@ -17,6 +17,8 @@
 #include <RenderDeferredTask.h>
 #include <TextRenderer3D.h>
 
+#include <AbstractViewStateInterface.h>
+
 const int FIXED_FONT_POINT_SIZE = 40;
 const int FIXED_FONT_SCALING_RATIO = FIXED_FONT_POINT_SIZE * 80.0f; // this is a ratio determined through experimentation
 const float LINE_SCALE_RATIO = 1.2f;
@@ -114,6 +116,7 @@ void Text3DOverlay::render(RenderArgs* args) {
 
     glm::vec3 topLeft(-halfDimensions.x, -halfDimensions.y, SLIGHTLY_BEHIND);
     glm::vec3 bottomRight(halfDimensions.x, halfDimensions.y, SLIGHTLY_BEHIND);
+    DependencyManager::get<GeometryCache>()->bindSimpleProgram(batch, false, quadColor.a < 1.0f, false, false, false, false);
     DependencyManager::get<GeometryCache>()->renderQuad(batch, topLeft, bottomRight, quadColor, _geometryId);
 
     // Same font properties as textSize()
@@ -133,13 +136,8 @@ void Text3DOverlay::render(RenderArgs* args) {
     glm::vec4 textColor = { _color.red / MAX_COLOR, _color.green / MAX_COLOR,
                             _color.blue / MAX_COLOR, getTextAlpha() };
 
-    // FIXME: Factor out textRenderer so that Text3DOverlay overlay parts can be grouped by pipeline
+    // FIXME: Factor out textRenderer so that Text3DOverlay overlay parts can be grouped by pipeline for a gpu performance increase.
-    //        for a gpu performance increase. Currently,
+    _textRenderer->draw(batch, 0, 0, getText(), textColor, glm::vec2(-1.0f), true);
-    //        Text renderer sets its own pipeline,
-    _textRenderer->draw(batch, 0, 0, getText(), textColor, glm::vec2(-1.0f), getDrawInFront());
-    //        so before we continue, we must reset the pipeline
-    batch.setPipeline(args->_shapePipeline->pipeline);
-    args->_shapePipeline->prepare(batch, args);
 }
 
 const render::ShapeKey Text3DOverlay::getShapeKey() {
@@ -160,7 +158,18 @@ void Text3DOverlay::setProperties(const QVariantMap& properties) {
 
     auto textAlpha = properties["textAlpha"];
     if (textAlpha.isValid()) {
+        float prevTextAlpha = getTextAlpha();
         setTextAlpha(textAlpha.toFloat());
+        // Update our payload key if necessary to handle transparency
+        if ((prevTextAlpha < 1.0f && _textAlpha >= 1.0f) || (prevTextAlpha >= 1.0f && _textAlpha < 1.0f)) {
+            auto itemID = getRenderItemID();
+            if (render::Item::isValidID(itemID)) {
+                render::ScenePointer scene = AbstractViewStateInterface::instance()->getMain3DScene();
+                render::Transaction transaction;
+                transaction.updateItem(itemID);
+                scene->enqueueTransaction(transaction);
+            }
+        }
     }
 
     bool valid;

interface/src/ui/overlays/Text3DOverlay.h

@@ -43,6 +43,7 @@ public:
     xColor getBackgroundColor();
     float getTextAlpha() { return _textAlpha; }
     float getBackgroundAlpha() { return getAlpha(); }
+    bool isTransparent() override { return Overlay::isTransparent() || _textAlpha < 1.0f; }
 
     // setters
     void setText(const QString& text);

libraries/entities-renderer/src/RenderableTextEntityItem.cpp

@@ -110,7 +110,7 @@ void TextEntityRenderer::doRender(RenderArgs* args) {
     if (!_geometryID) {
         _geometryID = geometryCache->allocateID();
     }
-    geometryCache->bindSimpleProgram(batch, false, transparent, false, false, true);
+    geometryCache->bindSimpleProgram(batch, false, transparent, false, false, false, false);
     geometryCache->renderQuad(batch, minCorner, maxCorner, backgroundColor, _geometryID);
     
     float scale = _lineHeight / _textRenderer->getFontSize();

libraries/entities-renderer/src/RenderableWebEntityItem.cpp

@@ -176,8 +176,6 @@ void WebEntityRenderer::doRender(RenderArgs* args) {
         }
     }
 
-        //_timer.singleShot
-
     PerformanceTimer perfTimer("WebEntityRenderer::render");
     static const glm::vec2 texMin(0.0f), texMax(1.0f), topLeft(-0.5f), bottomRight(0.5f);
 
@@ -187,11 +185,7 @@ void WebEntityRenderer::doRender(RenderArgs* args) {
     float fadeRatio = _isFading ? Interpolate::calculateFadeRatio(_fadeStartTime) : 1.0f;
     batch._glColor4f(1.0f, 1.0f, 1.0f, fadeRatio);
 
-    if (fadeRatio < OPAQUE_ALPHA_THRESHOLD) {
+    DependencyManager::get<GeometryCache>()->bindWebBrowserProgram(batch, fadeRatio < OPAQUE_ALPHA_THRESHOLD);
-        DependencyManager::get<GeometryCache>()->bindWebBrowserProgram(batch, true);
-    } else {
-        DependencyManager::get<GeometryCache>()->bindWebBrowserProgram(batch);
-    }
     DependencyManager::get<GeometryCache>()->renderQuad(batch, topLeft, bottomRight, texMin, texMax, glm::vec4(1.0f, 1.0f, 1.0f, fadeRatio), _geometryId);
 }
 

libraries/model/src/model/Skybox.cpp

@@ -97,7 +97,10 @@ void Skybox::render(gpu::Batch& batch, const ViewFrustum& viewFrustum, const Sky
             }
 
             auto skyState = std::make_shared<gpu::State>();
-            skyState->setStencilTest(true, 0xFF, gpu::State::StencilTest(1, 0xFF, gpu::EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
+            // Must match PrepareStencil::STENCIL_BACKGROUND
+            const int8_t STENCIL_BACKGROUND = 0;
+            skyState->setStencilTest(true, 0xFF, gpu::State::StencilTest(STENCIL_BACKGROUND, 0xFF, gpu::EQUAL,
+                gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
 
             thePipeline = gpu::Pipeline::create(skyShader, skyState);
         }

libraries/procedural/src/procedural/ProceduralSkybox.cpp

@@ -23,7 +23,10 @@ ProceduralSkybox::ProceduralSkybox() : model::Skybox() {
     _procedural._fragmentSource = skybox_frag;
     // Adjust the pipeline state for background using the stencil test
     _procedural.setDoesFade(false);
-    _procedural._opaqueState->setStencilTest(true, 0xFF, gpu::State::StencilTest(1, 0xFF, gpu::EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
+    // Must match PrepareStencil::STENCIL_BACKGROUND
+    const int8_t STENCIL_BACKGROUND = 0;
+    _procedural._opaqueState->setStencilTest(true, 0xFF, gpu::State::StencilTest(STENCIL_BACKGROUND, 0xFF, gpu::EQUAL,
+        gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
 }
 
 bool ProceduralSkybox::empty() {

libraries/render-utils/src/AntialiasingEffect.cpp

@@ -70,9 +70,8 @@ const gpu::PipelinePointer& Antialiasing::getAntialiasingPipeline(RenderArgs* ar
 
         gpu::StatePointer state = gpu::StatePointer(new gpu::State());
 
-        PrepareStencil::testMaskNoAA(*state);
-
         state->setDepthTest(false, false, gpu::LESS_EQUAL);
+        PrepareStencil::testNoAA(*state);
 
         // Good to go add the brand new pipeline
         _antialiasingPipeline = gpu::Pipeline::create(program, state);
@@ -95,7 +94,7 @@ const gpu::PipelinePointer& Antialiasing::getBlendPipeline() {
         gpu::StatePointer state = gpu::StatePointer(new gpu::State());
 
         state->setDepthTest(false, false, gpu::LESS_EQUAL);
-        PrepareStencil::testMaskNoAA(*state);
+        PrepareStencil::testNoAA(*state);
 
         // Good to go add the brand new pipeline
         _blendPipeline = gpu::Pipeline::create(program, state);

libraries/render-utils/src/DeferredLightingEffect.cpp

@@ -420,7 +420,7 @@ void PrepareDeferred::run(const RenderContextPointer& renderContext, const Input
             gpu::Framebuffer::BUFFER_COLOR0 | gpu::Framebuffer::BUFFER_COLOR1 | gpu::Framebuffer::BUFFER_COLOR2 | gpu::Framebuffer::BUFFER_COLOR3 |
             gpu::Framebuffer::BUFFER_DEPTH |
             gpu::Framebuffer::BUFFER_STENCIL,
-            vec4(vec3(0), 0), 1.0, 1, true);
+            vec4(vec3(0), 0), 1.0, 0, true);
 
         // For the rest of the rendering, bind the lighting model
         batch.setUniformBuffer(LIGHTING_MODEL_BUFFER_SLOT, lightingModel->getParametersBuffer());

libraries/render-utils/src/GeometryCache.cpp

@@ -561,7 +561,7 @@ void GeometryCache::initializeShapePipelines() {
 render::ShapePipelinePointer GeometryCache::getShapePipeline(bool textured, bool transparent, bool culled,
     bool unlit, bool depthBias) {
 
-    return std::make_shared<render::ShapePipeline>(getSimplePipeline(textured, transparent, culled, unlit, depthBias, false), nullptr,
+    return std::make_shared<render::ShapePipeline>(getSimplePipeline(textured, transparent, culled, unlit, depthBias, false, true), nullptr,
         [](const render::ShapePipeline& , gpu::Batch& batch, render::Args*) {
         batch.setResourceTexture(render::ShapePipeline::Slot::MAP::ALBEDO, DependencyManager::get<TextureCache>()->getWhiteTexture());
     }
@@ -1877,6 +1877,7 @@ public:
         IS_UNLIT_FLAG,
         HAS_DEPTH_BIAS_FLAG,
         IS_FADING_FLAG,
+        IS_ANTIALIASED_FLAG,
 
         NUM_FLAGS,
     };
@@ -1888,6 +1889,7 @@ public:
         IS_UNLIT = (1 << IS_UNLIT_FLAG),
         HAS_DEPTH_BIAS = (1 << HAS_DEPTH_BIAS_FLAG),
         IS_FADING = (1 << IS_FADING_FLAG),
+        IS_ANTIALIASED = (1 << IS_ANTIALIASED_FLAG),
     };
     typedef unsigned short Flags;
 
@@ -1899,6 +1901,7 @@ public:
     bool isUnlit() const { return isFlag(IS_UNLIT); }
     bool hasDepthBias() const { return isFlag(HAS_DEPTH_BIAS); }
     bool isFading() const { return isFlag(IS_FADING); }
+    bool isAntiAliased() const { return isFlag(IS_ANTIALIASED); }
 
     Flags _flags = 0;
     short _spare = 0;
@@ -1907,9 +1910,9 @@ public:
 
 
     SimpleProgramKey(bool textured = false, bool transparent = false, bool culled = true,
-        bool unlit = false, bool depthBias = false, bool fading = false) {
+        bool unlit = false, bool depthBias = false, bool fading = false, bool isAntiAliased = true) {
         _flags = (textured ? IS_TEXTURED : 0) | (transparent ? IS_TRANSPARENT : 0) | (culled ? IS_CULLED : 0) |
-            (unlit ? IS_UNLIT : 0) | (depthBias ? HAS_DEPTH_BIAS : 0) | (fading ? IS_FADING : 0);
+            (unlit ? IS_UNLIT : 0) | (depthBias ? HAS_DEPTH_BIAS : 0) | (fading ? IS_FADING : 0) | (isAntiAliased ? IS_ANTIALIASED : 0);
     }
 
     SimpleProgramKey(int bitmask) : _flags(bitmask) {}
@@ -1958,8 +1961,8 @@ gpu::PipelinePointer GeometryCache::getWebBrowserProgram(bool transparent) {
     return transparent ? _simpleTransparentWebBrowserPipelineNoAA : _simpleOpaqueWebBrowserPipelineNoAA;
 }
 
-void GeometryCache::bindSimpleProgram(gpu::Batch& batch, bool textured, bool transparent, bool culled, bool unlit, bool depthBiased) {
+void GeometryCache::bindSimpleProgram(gpu::Batch& batch, bool textured, bool transparent, bool culled, bool unlit, bool depthBiased, bool isAntiAliased) {
-    batch.setPipeline(getSimplePipeline(textured, transparent, culled, unlit, depthBiased));
+    batch.setPipeline(getSimplePipeline(textured, transparent, culled, unlit, depthBiased, false, isAntiAliased));
 
     // If not textured, set a default albedo map
     if (!textured) {
@@ -1968,8 +1971,8 @@ void GeometryCache::bindSimpleProgram(gpu::Batch& batch, bool textured, bool tra
     }
 }
 
-gpu::PipelinePointer GeometryCache::getSimplePipeline(bool textured, bool transparent, bool culled, bool unlit, bool depthBiased, bool fading) {
+gpu::PipelinePointer GeometryCache::getSimplePipeline(bool textured, bool transparent, bool culled, bool unlit, bool depthBiased, bool fading, bool isAntiAliased) {
-    SimpleProgramKey config { textured, transparent, culled, unlit, depthBiased, fading };
+    SimpleProgramKey config { textured, transparent, culled, unlit, depthBiased, fading, isAntiAliased };
 
     // If the pipeline already exists, return it
     auto it = _simplePrograms.find(config);
@@ -2027,11 +2030,10 @@ gpu::PipelinePointer GeometryCache::getSimplePipeline(bool textured, bool transp
         gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA,
         gpu::State::FACTOR_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::ONE);
 
-    if (config.isTransparent()) {
+    if (config.isAntiAliased()) {
-        PrepareStencil::testMask(*state);
+        config.isTransparent() ? PrepareStencil::testMask(*state) : PrepareStencil::testMaskDrawShape(*state);
-    }
+    } else {
-    else {
+        PrepareStencil::testMaskDrawShapeNoAA(*state);
-        PrepareStencil::testMaskDrawShape(*state);
     }
 
     gpu::ShaderPointer program = (config.isUnlit()) ? (config.isFading() ? _unlitFadeShader : _unlitShader) : (config.isFading() ? _simpleFadeShader : _simpleShader);

libraries/render-utils/src/GeometryCache.h

@@ -161,10 +161,10 @@ public:
 
     // Bind the pipeline and get the state to render static geometry
     void bindSimpleProgram(gpu::Batch& batch, bool textured = false, bool transparent = false, bool culled = true,
-                                          bool unlit = false, bool depthBias = false);
+                                          bool unlit = false, bool depthBias = false, bool isAntiAliased = true);
     // Get the pipeline to render static geometry
     static gpu::PipelinePointer getSimplePipeline(bool textured = false, bool transparent = false, bool culled = true,
-                                          bool unlit = false, bool depthBias = false, bool fading = false);
+                                          bool unlit = false, bool depthBias = false, bool fading = false, bool isAntiAliased = true);
 
     void bindWebBrowserProgram(gpu::Batch& batch, bool transparent = false);
     gpu::PipelinePointer getWebBrowserProgram(bool transparent);

libraries/render-utils/src/RenderForwardTask.cpp

@@ -17,6 +17,7 @@
 #include <PathUtils.h>
 #include <ViewFrustum.h>
 #include <gpu/Context.h>
+#include "StencilMaskPass.h"
 
 #include "FramebufferCache.h"
 #include "TextureCache.h"
@@ -93,7 +94,7 @@ void PrepareFramebuffer::run(const RenderContextPointer& renderContext,
             gpu::Framebuffer::BUFFER_COLOR0 |
             gpu::Framebuffer::BUFFER_DEPTH |
             gpu::Framebuffer::BUFFER_STENCIL,
-            vec4(vec3(0), 1), 1.0, 0.0, true);
+            vec4(vec3(0), 1), 1.0, 0, true);
     });
 
     framebuffer = _framebuffer;
@@ -130,11 +131,7 @@ const gpu::PipelinePointer Stencil::getPipeline() {
 
         auto state = std::make_shared<gpu::State>();
         state->setDepthTest(true, false, gpu::LESS_EQUAL);
-        const gpu::int8 STENCIL_OPAQUE = 1;
+        PrepareStencil::drawBackground(*state);
-        state->setStencilTest(true, 0xFF, gpu::State::StencilTest(STENCIL_OPAQUE, 0xFF, gpu::ALWAYS,
-                    gpu::State::STENCIL_OP_REPLACE,
-                    gpu::State::STENCIL_OP_REPLACE,
-                    gpu::State::STENCIL_OP_KEEP));
 
         _stencilPipeline = gpu::Pipeline::create(program, state);
     }

libraries/render-utils/src/StencilMaskPass.cpp

@@ -104,30 +104,51 @@ void PrepareStencil::run(const RenderContextPointer& renderContext, const gpu::F
     });
 }
 
+// Always draw MASK to the stencil buffer (used to always prevent drawing in certain areas later)
 void PrepareStencil::drawMask(gpu::State& state) {
-    state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::ALWAYS, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE));
+    state.setStencilTest(true, 0xFF, gpu::State::StencilTest(STENCIL_MASK, 0xFF, gpu::ALWAYS,
+        gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE));
 }
 
+// Draw BACKGROUND to the stencil buffer behind everything else
+void PrepareStencil::drawBackground(gpu::State& state) {
+    state.setStencilTest(true, 0xFF, gpu::State::StencilTest(STENCIL_BACKGROUND, 0xFF, gpu::ALWAYS,
+        gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_KEEP));
+}
+
+// Pass if this area has NOT been marked as MASK
 void PrepareStencil::testMask(gpu::State& state) {
-    state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::NOT_EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
+    state.setStencilTest(true, 0x00, gpu::State::StencilTest(STENCIL_MASK, 0xFF, gpu::NOT_EQUAL,
+        gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
 }
 
-void PrepareStencil::testMaskNoAA(gpu::State& state) {
+// Pass if this area has NOT been marked as NO_AA or anything containing NO_AA
-    state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK | PrepareStencil::STENCIL_NO_AA, 0xFF, gpu::NOT_EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
+void PrepareStencil::testNoAA(gpu::State& state) {
+    state.setStencilTest(true, 0x00, gpu::State::StencilTest(STENCIL_NO_AA, STENCIL_NO_AA, gpu::NOT_EQUAL,
+        gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
 }
 
+// Pass if this area WAS marked as BACKGROUND
+// (see: model/src/Skybox.cpp, procedural/src/ProceduralSkybox.cpp)
 void PrepareStencil::testBackground(gpu::State& state) {
-    state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_BACKGROUND, 0xFF, gpu::EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
+    state.setStencilTest(true, 0x00, gpu::State::StencilTest(STENCIL_BACKGROUND, 0xFF, gpu::EQUAL,
-}
+        gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
-
-void PrepareStencil::testMaskDrawShape(gpu::State& state) {
-    state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK, 0xFF, gpu::NOT_EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_ZERO));
-}
-
-void PrepareStencil::testMaskDrawShapeNoAA(gpu::State& state) {
-    state.setStencilTest(true, 0xFF, gpu::State::StencilTest(PrepareStencil::STENCIL_MASK | PrepareStencil::STENCIL_NO_AA, 0xFF, gpu::ALWAYS, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_REPLACE));
 }
 
+// Pass if this area WAS marked as SHAPE or anything containing SHAPE
 void PrepareStencil::testShape(gpu::State& state) {
-    state.setStencilTest(true, 0x00, gpu::State::StencilTest(PrepareStencil::STENCIL_SHAPE, 0xFF, gpu::EQUAL, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
+    state.setStencilTest(true, 0x00, gpu::State::StencilTest(STENCIL_SHAPE, STENCIL_SHAPE, gpu::EQUAL,
+        gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP));
+}
+
+// Pass if this area was NOT marked as MASK, write to SHAPE if it passes
+void PrepareStencil::testMaskDrawShape(gpu::State& state) {
+    state.setStencilTest(true, STENCIL_SHAPE, gpu::State::StencilTest(STENCIL_MASK | STENCIL_SHAPE, STENCIL_MASK, gpu::NOT_EQUAL,
+        gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_REPLACE));
+}
+
+// Pass if this area was NOT marked as MASK, write to SHAPE and NO_AA if it passes
+void PrepareStencil::testMaskDrawShapeNoAA(gpu::State& state) {
+    state.setStencilTest(true, STENCIL_SHAPE | STENCIL_NO_AA, gpu::State::StencilTest(STENCIL_MASK | STENCIL_SHAPE | STENCIL_NO_AA, STENCIL_MASK, gpu::NOT_EQUAL,
+        gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_KEEP, gpu::State::STENCIL_OP_REPLACE));
 }

libraries/render-utils/src/StencilMaskPass.h

@@ -41,20 +41,20 @@ public:
 
     void run(const render::RenderContextPointer& renderContext, const gpu::FramebufferPointer& dstFramebuffer);
 
-    static const gpu::int8 STENCIL_SHAPE = 0;
+    // Always use 0 to clear the stencil buffer to set it to background
-    static const gpu::int8 STENCIL_BACKGROUND = 1 << 0;
+    static const gpu::int8 STENCIL_BACKGROUND = 0; // must match values in Skybox.cpp and ProceduralSkybox.cpp
-    static const gpu::int8 STENCIL_MASK = 1 << 1;
+    static const gpu::int8 STENCIL_MASK =       1 << 0;
-    static const gpu::int8 STENCIL_NO_AA = 1 << 2;
+    static const gpu::int8 STENCIL_SHAPE =      1 << 1;
-
+    static const gpu::int8 STENCIL_NO_AA =      1 << 2;
 
     static void drawMask(gpu::State& state);
+    static void drawBackground(gpu::State& state);
+    static void testNoAA(gpu::State& state);
     static void testMask(gpu::State& state);
-    static void testMaskNoAA(gpu::State& state);
     static void testBackground(gpu::State& state);
+    static void testShape(gpu::State& state);
     static void testMaskDrawShape(gpu::State& state);
     static void testMaskDrawShapeNoAA(gpu::State& state);
-    static void testShape(gpu::State& state);
-
 
 private:
     gpu::PipelinePointer _meshStencilPipeline;

libraries/render-utils/src/sdf_text3D.slf

@@ -1,7 +1,7 @@
 <@include gpu/Config.slh@>
 <$VERSION_HEADER$>
 //  Generated on <$_SCRIBE_DATE$>
-//  sdf_text.frag
+//  sdf_text3D.frag
 //  fragment shader
 //
 //  Created by Bradley Austin Davis on 2015-02-04
@@ -38,18 +38,21 @@ void main() {
     // perform adaptive anti-aliasing of the edges
     // The larger we're rendering, the less anti-aliasing we need
     float s = smoothing * length(fwidth(_texCoord0));
-    float w = clamp( s, 0.0, 0.5);
+    float w = clamp(s, 0.0, 0.5);
     float a = smoothstep(0.5 - w, 0.5 + w, sdf);
     
-    // discard if unvisible
+    // discard if invisible
     if (a < 0.01) {
         discard;
     }
-    
+
-    packDeferredFragmentTranslucent(
+    packDeferredFragment(
         normalize(_normal),
         a * Color.a,
         Color.rgb,
-        DEFAULT_FRESNEL,
+        DEFAULT_ROUGHNESS,
-        DEFAULT_ROUGHNESS);
+        DEFAULT_METALLIC,
+        DEFAULT_EMISSIVE,
+        DEFAULT_OCCLUSION,
+        DEFAULT_SCATTERING);
 }

libraries/render-utils/src/sdf_text3D.slv

@@ -1,7 +1,7 @@
 <@include gpu/Config.slh@>
 <$VERSION_HEADER$>
 //  Generated on <$_SCRIBE_DATE$>
-//  sdf_text.vert
+//  sdf_text3D.vert
 //  vertex shader
 //
 //  Created by Brad Davis on 10/14/13.
@@ -27,5 +27,6 @@ void main() {
     TransformCamera cam = getTransformCamera();
     TransformObject obj = getTransformObject();
     <$transformModelToClipPos(cam, obj, inPosition, gl_Position)$>
-    <$transformModelToWorldDir(cam, obj, inNormal.xyz, _normal.xyz)$>
+    const vec3 normal = vec3(0, 0, 1);
+    <$transformModelToWorldDir(cam, obj, normal, _normal)$>
 }

libraries/render-utils/src/sdf_text3D_transparent.slf

@@ -1,7 +1,7 @@
 <@include gpu/Config.slh@>
 <$VERSION_HEADER$>
 //  Generated on <$_SCRIBE_DATE$>
-//  sdf_text.frag
+//  sdf_text3D_transparent.frag
 //  fragment shader
 //
 //  Created by Bradley Austin Davis on 2015-02-04
@@ -10,6 +10,8 @@
 //  Distributed under the Apache License, Version 2.0.
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 
+<@include DeferredBufferWrite.slh@>
+
 uniform sampler2D Font;
 uniform bool Outline;
 uniform vec4 Color;
@@ -18,8 +20,6 @@ uniform vec4 Color;
 in vec3 _normal;
 in vec2 _texCoord0;
 
-layout(location = 0) out vec4 _fragColor0;
-
 const float gamma = 2.2;
 const float smoothing = 32.0;
 const float interiorCutoff = 0.8;
@@ -38,15 +38,18 @@ void main() {
     // perform adaptive anti-aliasing of the edges
     // The larger we're rendering, the less anti-aliasing we need
     float s = smoothing * length(fwidth(_texCoord0));
-    float w = clamp( s, 0.0, 0.5);
+    float w = clamp(s, 0.0, 0.5);
     float a = smoothstep(0.5 - w, 0.5 + w, sdf);
     
-    // gamma correction for linear attenuation
+    // discard if invisible
-    a = pow(a, 1.0 / gamma);
-    
-    // discard if unvisible
     if (a < 0.01) {
         discard;
     }
-    _fragColor0 = vec4(Color.rgb, a);
+
+    packDeferredFragmentTranslucent(
+        normalize(_normal),
+        a * Color.a,
+        Color.rgb,
+        DEFAULT_FRESNEL,
+        DEFAULT_ROUGHNESS);
 }

libraries/render-utils/src/text/Font.cpp

@@ -9,10 +9,11 @@
 
 #include "sdf_text3D_vert.h"
 #include "sdf_text3D_frag.h"
-#include "sdf_text3D_overlay_frag.h"
+#include "sdf_text3D_transparent_frag.h"
 
 #include "../RenderUtilsLogging.h"
 #include "FontFamilies.h"
+#include "../StencilMaskPass.h"
 
 static std::mutex fontMutex;
 
@@ -224,13 +225,13 @@ void Font::setupGPU() {
         {
             auto vertexShader = gpu::Shader::createVertex(std::string(sdf_text3D_vert));
             auto pixelShader = gpu::Shader::createPixel(std::string(sdf_text3D_frag));
-            auto pixelShaderOverlay = gpu::Shader::createPixel(std::string(sdf_text3D_overlay_frag));
+            auto pixelShaderTransparent = gpu::Shader::createPixel(std::string(sdf_text3D_transparent_frag));
             gpu::ShaderPointer program = gpu::Shader::createProgram(vertexShader, pixelShader);
-            gpu::ShaderPointer programOverlay = gpu::Shader::createProgram(vertexShader, pixelShaderOverlay);
+            gpu::ShaderPointer programTransparent = gpu::Shader::createProgram(vertexShader, pixelShaderTransparent);
 
             gpu::Shader::BindingSet slotBindings;
             gpu::Shader::makeProgram(*program, slotBindings);
-            gpu::Shader::makeProgram(*programOverlay, slotBindings);
+            gpu::Shader::makeProgram(*programTransparent, slotBindings);
 
             _fontLoc = program->getTextures().findLocation("Font");
             _outlineLoc = program->getUniforms().findLocation("Outline");
@@ -239,15 +240,20 @@ void Font::setupGPU() {
             auto state = std::make_shared<gpu::State>();
             state->setCullMode(gpu::State::CULL_BACK);
             state->setDepthTest(true, true, gpu::LESS_EQUAL);
-            state->setBlendFunction(true,
+            state->setBlendFunction(false,
                 gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA,
                 gpu::State::FACTOR_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::ONE);
+            PrepareStencil::testMaskDrawShapeNoAA(*state);
             _pipeline = gpu::Pipeline::create(program, state);
 
-            auto layeredState = std::make_shared<gpu::State>();
+            auto transparentState = std::make_shared<gpu::State>();
-            layeredState->setCullMode(gpu::State::CULL_BACK);
+            transparentState->setCullMode(gpu::State::CULL_BACK);
-            layeredState->setDepthTest(true, true, gpu::LESS_EQUAL);
+            transparentState->setDepthTest(true, true, gpu::LESS_EQUAL);
-            _layeredPipeline = gpu::Pipeline::create(programOverlay, layeredState);
+            transparentState->setBlendFunction(true,
+                gpu::State::SRC_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::INV_SRC_ALPHA,
+                gpu::State::FACTOR_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::ONE);
+            PrepareStencil::testMaskDrawShapeNoAA(*transparentState);
+            _transparentPipeline = gpu::Pipeline::create(programTransparent, transparentState);
         }
 
         // Sanity checks
@@ -361,7 +367,7 @@ void Font::drawString(gpu::Batch& batch, float x, float y, const QString& str, c
 
     setupGPU();
 
-    batch.setPipeline(layered ? _layeredPipeline : _pipeline);
+    batch.setPipeline(((*color).a < 1.0f || layered) ? _transparentPipeline : _pipeline);
     batch.setResourceTexture(_fontLoc, _texture);
     batch._glUniform1i(_outlineLoc, (effectType == OUTLINE_EFFECT));
     

libraries/render-utils/src/text/Font.h

@@ -63,7 +63,7 @@ private:
 
     // gpu structures
     gpu::PipelinePointer _pipeline;
-    gpu::PipelinePointer _layeredPipeline;
+    gpu::PipelinePointer _transparentPipeline;
     gpu::TexturePointer _texture;
     gpu::Stream::FormatPointer _format;
     gpu::BufferPointer _verticesBuffer;