Starting to add the stencil mask for hmd

libraries/gpu/src/gpu/DrawTransformVertexPosition.slv

@@ -0,0 +1,28 @@
+<@include gpu/Config.slh@>
+<$VERSION_HEADER$>
+//  Generated on <$_SCRIBE_DATE$>
+//
+//  Draw and transform the fed vertex position with the standard MVP stack
+// Output the clip position
+//
+//  Created by Sam Gateau on 5/30/2017
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+<@include gpu/Transform.slh@>
+
+<$declareStandardTransform()$>
+
+layout(location = 0) in vec4 inPosition;
+
+out vec3 varWorldPos;
+
+void main(void) {
+    // standard transform
+    TransformCamera cam = getTransformCamera();
+    TransformObject obj = getTransformObject();
+    <$transformModelToClipPos(cam, obj, inPosition, gl_Position)$>
+}

libraries/gpu/src/gpu/DrawWhite.slf

@@ -0,0 +1,18 @@
+<@include gpu/Config.slh@>
+<$VERSION_HEADER$>
+//  Generated on <$_SCRIBE_DATE$>
+//
+//  Draw white
+//
+//  Created by Sam Gateau on 5/30/2017
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+out vec4 outFragColor;
+
+void main(void) {
+    outFragColor = vec4(1.0);
+}

libraries/gpu/src/gpu/StandardShaderLib.cpp

@@ -16,6 +16,8 @@
 #include "DrawTransformUnitQuad_vert.h"
 #include "DrawTexcoordRectTransformUnitQuad_vert.h"
 #include "DrawViewportQuadTransformTexcoord_vert.h"
+#include "DrawTransformVertexPosition_vert.h"
+#include "DrawWhite_frag.h"
 #include "DrawTexture_frag.h"
 #include "DrawTextureOpaque_frag.h"
 #include "DrawColoredTexture_frag.h"
@@ -26,6 +28,8 @@ ShaderPointer StandardShaderLib::_drawUnitQuadTexcoordVS;
 ShaderPointer StandardShaderLib::_drawTransformUnitQuadVS;
 ShaderPointer StandardShaderLib::_drawTexcoordRectTransformUnitQuadVS;
 ShaderPointer StandardShaderLib::_drawViewportQuadTransformTexcoordVS;
+ShaderPointer StandardShaderLib::_drawTransformVertexPositionVS;
+ShaderPointer StandardShaderLib::_drawWhitePS;
 ShaderPointer StandardShaderLib::_drawTexturePS;
 ShaderPointer StandardShaderLib::_drawTextureOpaquePS;
 ShaderPointer StandardShaderLib::_drawColoredTexturePS;
@@ -84,6 +88,19 @@ ShaderPointer StandardShaderLib::getDrawViewportQuadTransformTexcoordVS() {
     }
     return _drawViewportQuadTransformTexcoordVS;
 }
+ShaderPointer StandardShaderLib::getDrawTransformVertexPositionVS() {
+    if (!_drawTransformVertexPositionVS) {
+        _drawTransformVertexPositionVS = gpu::Shader::createVertex(std::string(DrawTransformVertexPosition_vert));
+    }
+    return _drawTransformVertexPositionVS;
+}
+
+ShaderPointer StandardShaderLib::getDrawWhitePS() {
+    if (!_drawWhitePS) {
+        _drawWhitePS = gpu::Shader::createPixel(std::string(DrawWhite_frag));
+    }
+    return _drawWhitePS;
+}
 
 ShaderPointer StandardShaderLib::getDrawTexturePS() {
     if (!_drawTexturePS) {

libraries/gpu/src/gpu/StandardShaderLib.h

@@ -37,6 +37,11 @@ public:
     // Shader draws the unit quad in the full viewport clipPos = ([(-1,-1),(1,1)]) and transform the texcoord = [(0,0),(1,1)] by the model transform.
     static ShaderPointer getDrawViewportQuadTransformTexcoordVS();
 
+    // Shader draw the fed vertex position and transform it by the full model transform stack (Model, View, Proj).
+    // simply output the world pos and the clip pos to the next stage
+    static ShaderPointer getDrawTransformVertexPositionVS();
+
+    static ShaderPointer getDrawWhitePS();
     static ShaderPointer getDrawTexturePS();
     static ShaderPointer getDrawTextureOpaquePS();
     static ShaderPointer getDrawColoredTexturePS();
@@ -51,6 +56,9 @@ protected:
     static ShaderPointer _drawTransformUnitQuadVS;
     static ShaderPointer _drawTexcoordRectTransformUnitQuadVS;
     static ShaderPointer _drawViewportQuadTransformTexcoordVS;
+    static ShaderPointer _drawTransformVertexPositionVS;
+
+    static ShaderPointer _drawWhitePS;
     static ShaderPointer _drawTexturePS;
     static ShaderPointer _drawTextureOpaquePS;
     static ShaderPointer _drawColoredTexturePS;

libraries/model/src/model/Geometry.cpp

@@ -241,6 +241,42 @@ void Mesh::forEach(std::function<void(glm::vec3)> vertexFunc,
     }
 }
 
+MeshPointer Mesh::createIndexedTriangles_P3F(uint32_t numVertices, uint32_t numIndices, const glm::vec3* vertices, const uint32_t* indices) {
+    MeshPointer mesh;
+    if (numVertices == 0) { return mesh; }
+    if (numIndices < 3) { return mesh; }
+
+    mesh = std::make_shared<Mesh>();
+
+    // Vertex buffer
+    mesh->setVertexBuffer(gpu::BufferView(new gpu::Buffer(numVertices * sizeof(glm::vec3), (gpu::Byte*) vertices), gpu::Element::VEC3F_XYZ));
+
+    // trim down the indices to shorts if possible
+    if (numIndices < std::numeric_limits<uint16_t>::max()) {
+        Indices16 shortIndicesVector;
+        int16_t* shortIndices = nullptr;
+        if (indices) {
+            shortIndicesVector.resize(numIndices);
+            for (int i = 0; i < numIndices; i++) {
+                shortIndicesVector[i] = indices[i];
+            }
+            shortIndices = shortIndicesVector.data();
+        }
+
+        mesh->setIndexBuffer(gpu::BufferView(new gpu::Buffer(numIndices * sizeof(uint16_t), (gpu::Byte*) shortIndices), gpu::Element::INDEX_UINT16));
+    } else {
+
+        mesh->setIndexBuffer(gpu::BufferView(new gpu::Buffer(numIndices * sizeof(uint32_t), (gpu::Byte*) indices), gpu::Element::INDEX_INT32));
+    }
+
+    
+    std::vector<model::Mesh::Part> parts;
+    parts.push_back(model::Mesh::Part(0, numIndices, 0, model::Mesh::TRIANGLES));
+    mesh->setPartBuffer(gpu::BufferView(new gpu::Buffer(parts.size() * sizeof(model::Mesh::Part), (gpu::Byte*) parts.data()), gpu::Element::PART_DRAWCALL));
+
+    return mesh;
+}
+
 
 Geometry::Geometry() {
 }
@@ -256,3 +292,5 @@ Geometry::~Geometry() {
 void Geometry::setMesh(const MeshPointer& mesh) {
     _mesh = mesh;
 }
+
+

libraries/model/src/model/Geometry.h

@@ -65,6 +65,9 @@ public:
     const gpu::BufferStream& getVertexStream() const { return _vertexStream; }
 
     // Index Buffer
+    using Indices16 = std::vector<int16_t>;
+    using Indices32 = std::vector<int32_t>;
+
     void setIndexBuffer(const BufferView& buffer);
     const BufferView& getIndexBuffer() const { return _indexBuffer; }
     size_t getNumIndices() const { return _indexBuffer.getNumElements(); }
@@ -127,6 +130,9 @@ public:
                  std::function<void(glm::vec3)> normalFunc,
                  std::function<void(uint32_t)> indexFunc);
 
+
+    static MeshPointer createIndexedTriangles_P3F(uint32_t numVertices, uint32_t numTriangles, const glm::vec3* vertices = nullptr, const uint32_t* indices = nullptr);
+
 protected:
 
     gpu::Stream::FormatPointer _vertexFormat;

libraries/render-utils/src/DeferredLightingEffect.cpp

@@ -160,11 +160,11 @@ void DeferredLightingEffect::init() {
 
 
     lp->setAmbientIntensity(0.5f);
-	lp->setAmbientMap(_defaultSkyboxAmbientTexture);
-	auto irradianceSH = _defaultSkyboxAmbientTexture->getIrradiance();
-	if (irradianceSH) {
-		lp->setAmbientSphere((*irradianceSH));
-	}
+    lp->setAmbientMap(_defaultSkyboxAmbientTexture);
+    auto irradianceSH = _defaultSkyboxAmbientTexture->getIrradiance();
+    if (irradianceSH) {
+        lp->setAmbientSphere((*irradianceSH));
+    }
 }
 
 void DeferredLightingEffect::setupKeyLightBatch(gpu::Batch& batch, int lightBufferUnit, int ambientBufferUnit, int skyboxCubemapUnit) {

libraries/render-utils/src/RenderDeferredTask.cpp

@@ -85,12 +85,15 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren
     const auto deferredFramebuffer = prepareDeferredOutputs.getN<PrepareDeferred::Outputs>(0);
     const auto lightingFramebuffer = prepareDeferredOutputs.getN<PrepareDeferred::Outputs>(1);
 
+    // draw a stencil mask in hidden regions of the frmaebuffer.
+
     // Render opaque objects in DeferredBuffer
     const auto opaqueInputs = DrawStateSortDeferred::Inputs(opaques, lightingModel).hasVarying();
     task.addJob<DrawStateSortDeferred>("DrawOpaqueDeferred", opaqueInputs, shapePlumber);
 
     // Once opaque is all rendered create stencil background
     task.addJob<DrawStencilDeferred>("DrawOpaqueStencil", deferredFramebuffer);
+    task.addJob<PrepareStencil>("PrepareStencil", primaryFramebuffer);
 
     task.addJob<EndGPURangeTimer>("OpaqueRangeTimer", opaqueRangeTimer);
 
@@ -469,6 +472,71 @@ void DrawBackgroundDeferred::run(const RenderContextPointer& renderContext, cons
    // std::static_pointer_cast<Config>(renderContext->jobConfig)->gpuTime = _gpuTimer.getAverage();
 }
 
+gpu::PipelinePointer PrepareStencil::getDrawStencilPipeline() {
+    if (!_drawStencilPipeline) {
+        const gpu::int8 STENCIL_OPAQUE = 1;
+        auto vs = gpu::StandardShaderLib::getDrawTransformVertexPositionVS();
+        auto ps = gpu::StandardShaderLib::getDrawWhitePS();
+        auto program = gpu::Shader::createProgram(vs, ps);
+        gpu::Shader::makeProgram((*program));
+
+        auto state = std::make_shared<gpu::State>();
+        state->setStencilTest(true, 0xFF, gpu::State::StencilTest(0, 0xFF, gpu::ALWAYS, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_REPLACE, gpu::State::STENCIL_OP_KEEP));
+        state->setColorWriteMask(0);
+
+        _drawStencilPipeline = gpu::Pipeline::create(program, state);
+    }
+    return _drawStencilPipeline;
+}
+
+model::MeshPointer PrepareStencil::getMesh() {
+    if (!_mesh) {
+
+        std::vector<glm::vec3> vertices {
+            { -1.0f, -1.0f, 0.0f }, { -1.0f, 0.0f, 0.0f },
+            { -1.0f, 1.0f, 0.0f }, { 0.0f, 1.0f, 0.0f },
+            { 1.0f, 1.0f, 0.0f }, { 1.0f, 0.0f, 0.0f },
+            { 1.0f, -1.0f, 0.0f }, { 0.0f, -1.0f, 0.0f } };
+
+        std::vector<uint32_t> indices { 0, 7, 1, 1, 3, 2, 3, 5, 4, 5, 7, 6 };
+
+        _mesh = model::Mesh::createIndexedTriangles_P3F((uint32_t) vertices.size(), (uint32_t) indices.size(), vertices.data(), indices.data());
+
+    }
+    return _mesh;
+}
+
+
+void PrepareStencil::run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& srcFramebuffer) {
+    assert(renderContext->args);
+    assert(renderContext->args->_context);
+
+    RenderArgs* args = renderContext->args;
+    doInBatch(args->_context, [&](gpu::Batch& batch) {
+        args->_batch = &batch;
+
+        batch.setViewportTransform(args->_viewport);
+        batch.setStateScissorRect(args->_viewport);
+
+        batch.resetViewTransform();
+        batch.setModelTransform(Transform());
+        glm::mat4 proj;
+        batch.setProjectionTransform(proj);
+
+        batch.setPipeline(getDrawStencilPipeline());
+
+        auto mesh = getMesh();
+        batch.setIndexBuffer(gpu::UINT32, (mesh->getIndexBuffer()._buffer), 0);
+        batch.setInputFormat((mesh->getVertexFormat()));
+        batch.setInputStream(0, mesh->getVertexStream());
+
+        // Draw
+        auto part = mesh->getPartBuffer().get<model::Mesh::Part>(0);
+        batch.drawIndexed(gpu::TRIANGLES, part._numIndices, part._startIndex);
+    });
+    args->_batch = nullptr;
+}
+
 void Blit::run(const RenderContextPointer& renderContext, const gpu::FramebufferPointer& srcFramebuffer) {
     assert(renderContext->args);
     assert(renderContext->args->_context);
@@ -538,3 +606,4 @@ void Blit::run(const RenderContextPointer& renderContext, const gpu::Framebuffer
         }
     });
 }
+

libraries/render-utils/src/RenderDeferredTask.h

@@ -13,6 +13,7 @@
 #define hifi_RenderDeferredTask_h
 
 #include <gpu/Pipeline.h>
+#include <model/Geometry.h>
 #include <render/RenderFetchCullSortTask.h>
 #include "LightingModel.h"
 
@@ -185,6 +186,21 @@ protected:
     bool _opaquePass{ true };
 };
 
+class PrepareStencil {
+public:
+    using JobModel = render::Job::ModelI<PrepareStencil, gpu::FramebufferPointer>;
+
+    void run(const render::RenderContextPointer& renderContext, const gpu::FramebufferPointer& dstFramebuffer);
+private:
+    gpu::PipelinePointer _drawStencilPipeline;
+    gpu::PipelinePointer getDrawStencilPipeline();
+
+    model::MeshPointer _mesh;
+    model::MeshPointer getMesh();
+
+};
+
+
 class Blit {
 public:
     using JobModel = render::Job::ModelI<Blit, gpu::FramebufferPointer>;