From 0bbd1edc0aff10cb275c11bfae23c211dcf9603f Mon Sep 17 00:00:00 2001
From: Olivier Prat <olivier@zvork.fr>
Date: Tue, 7 Nov 2017 18:26:15 +0100
Subject: [PATCH] Fixed fluttering highlight bug by attaching only stencil when
 drawing outline

---
 libraries/gpu/src/gpu/Framebuffer.cpp         | 63 +++++++++++++++----
 libraries/gpu/src/gpu/Framebuffer.h           |  3 +
 .../render-utils/src/HighlightEffect.cpp      |  2 +-
 3 files changed, 55 insertions(+), 13 deletions(-)

diff --git a/libraries/gpu/src/gpu/Framebuffer.cpp b/libraries/gpu/src/gpu/Framebuffer.cpp
index f1257e7c83..4fcc9ab0f9 100755
--- a/libraries/gpu/src/gpu/Framebuffer.cpp
+++ b/libraries/gpu/src/gpu/Framebuffer.cpp
@@ -220,7 +220,7 @@ uint32 Framebuffer::getRenderBufferSubresource(uint32 slot) const {
     }
 }
 
-bool Framebuffer::setDepthStencilBuffer(const TexturePointer& texture, const Format& format, uint32 subresource) {
+bool Framebuffer::assignDepthStencilBuffer(const TexturePointer& texture, const Format& format, uint32 subresource) {
     if (isSwapchain()) {
         return false;
     }
@@ -244,20 +244,59 @@ bool Framebuffer::setDepthStencilBuffer(const TexturePointer& texture, const For
     // assign the new one
     _depthStencilBuffer = TextureView(texture, subresource, format);
 
-    _bufferMask = ( _bufferMask & ~BUFFER_DEPTHSTENCIL);
-    if (texture) {
-        if (format.getSemantic() == gpu::DEPTH) {
-            _bufferMask |= BUFFER_DEPTH;
-        } else if (format.getSemantic() == gpu::STENCIL) {
-            _bufferMask |= BUFFER_STENCIL;
-        } else if (format.getSemantic() == gpu::DEPTH_STENCIL) {
-            _bufferMask |= BUFFER_DEPTHSTENCIL;
-        }
-    }
-
     return true;
 }
 
+bool Framebuffer::setDepthBuffer(const TexturePointer& texture, const Format& format, uint32 subresource) {
+    if (assignDepthStencilBuffer(texture, format, subresource)) {
+        _bufferMask = (_bufferMask & ~BUFFER_DEPTHSTENCIL);
+        if (texture) {
+            if (format.getSemantic() == gpu::DEPTH || format.getSemantic() == gpu::DEPTH_STENCIL) {
+                _bufferMask |= BUFFER_DEPTH;
+            } else {
+                return false;
+            }
+        }
+
+        return true;
+    }
+    return false;
+}
+
+bool Framebuffer::setStencilBuffer(const TexturePointer& texture, const Format& format, uint32 subresource) {
+    if (assignDepthStencilBuffer(texture, format, subresource)) {
+        _bufferMask = (_bufferMask & ~BUFFER_DEPTHSTENCIL);
+        if (texture) {
+            if (format.getSemantic() == gpu::STENCIL || format.getSemantic() == gpu::DEPTH_STENCIL) {
+                _bufferMask |= BUFFER_STENCIL;
+            } else  {
+                return false;
+            }
+        }
+
+        return true;
+    }
+    return false;
+}
+
+bool Framebuffer::setDepthStencilBuffer(const TexturePointer& texture, const Format& format, uint32 subresource) {
+    if (assignDepthStencilBuffer(texture, format, subresource)) {
+        _bufferMask = (_bufferMask & ~BUFFER_DEPTHSTENCIL);
+        if (texture) {
+            if (format.getSemantic() == gpu::DEPTH) {
+                _bufferMask |= BUFFER_DEPTH;
+            } else if (format.getSemantic() == gpu::STENCIL) {
+                _bufferMask |= BUFFER_STENCIL;
+            } else if (format.getSemantic() == gpu::DEPTH_STENCIL) {
+                _bufferMask |= BUFFER_DEPTHSTENCIL;
+            }
+        }
+
+        return true;
+    }
+    return false;
+}
+
 TexturePointer Framebuffer::getDepthStencilBuffer() const {
     if (isSwapchain()) {
         return TexturePointer();
diff --git a/libraries/gpu/src/gpu/Framebuffer.h b/libraries/gpu/src/gpu/Framebuffer.h
index b3a500d68f..b3cf0fbba3 100755
--- a/libraries/gpu/src/gpu/Framebuffer.h
+++ b/libraries/gpu/src/gpu/Framebuffer.h
@@ -107,6 +107,8 @@ public:
     TexturePointer getRenderBuffer(uint32 slot) const;
     uint32 getRenderBufferSubresource(uint32 slot) const;
 
+    bool setDepthBuffer(const TexturePointer& texture, const Format& format, uint32 subresource = 0);
+    bool setStencilBuffer(const TexturePointer& texture, const Format& format, uint32 subresource = 0);
     bool setDepthStencilBuffer(const TexturePointer& texture, const Format& format, uint32 subresource = 0);
     TexturePointer getDepthStencilBuffer() const;
     uint32 getDepthStencilBufferSubresource() const;
@@ -168,6 +170,7 @@ protected:
     uint16 _numSamples = 0;
 
     void updateSize(const TexturePointer& texture);
+    bool assignDepthStencilBuffer(const TexturePointer& texture, const Format& format, uint32 subresource);
 
     // Non exposed
     Framebuffer(const Framebuffer& framebuffer) = delete;
diff --git a/libraries/render-utils/src/HighlightEffect.cpp b/libraries/render-utils/src/HighlightEffect.cpp
index 98a8835409..7c58e5ba66 100644
--- a/libraries/render-utils/src/HighlightEffect.cpp
+++ b/libraries/render-utils/src/HighlightEffect.cpp
@@ -59,7 +59,7 @@ void HighlightRessources::update(const gpu::FramebufferPointer& primaryFrameBuff
 void HighlightRessources::allocateColorBuffer(const gpu::FramebufferPointer& primaryFrameBuffer) {
     _colorFrameBuffer = gpu::FramebufferPointer(gpu::Framebuffer::create("primaryWithStencil"));
     _colorFrameBuffer->setRenderBuffer(0, primaryFrameBuffer->getRenderBuffer(0));
-    _colorFrameBuffer->setDepthStencilBuffer(_depthStencilTexture, _depthStencilTexture->getTexelFormat());
+    _colorFrameBuffer->setStencilBuffer(_depthStencilTexture, _depthStencilTexture->getTexelFormat());
 }
 
 void HighlightRessources::allocateDepthBuffer(const gpu::FramebufferPointer& primaryFrameBuffer) {