fix custom shape key shadows + render layers

libraries/entities-renderer/src/RenderableParticleEffectEntityItem.cpp

@@ -21,20 +21,21 @@ using namespace render::entities;
 
 static uint8_t CUSTOM_PIPELINE_NUMBER = 0;
 static gpu::Stream::FormatPointer _vertexFormat;
-static std::map<std::tuple<bool, bool, bool>, gpu::PipelinePointer> _pipelines;
+// forward, transparent, shadow, wireframe
+static std::map<std::tuple<bool, bool, bool, bool>, gpu::PipelinePointer> _pipelines;
 
 static ShapePipelinePointer shapePipelineFactory(const ShapePlumber& plumber, const ShapeKey& key, RenderArgs* args) {
-    // FIXME: custom pipelines like this don't handle shadows or renderLayers correctly
-
     if (_pipelines.empty()) {
         using namespace shader::entities_renderer::program;
 
-        // forward, translucent
+        // forward, translucent, shadow
-        static const std::vector<std::tuple<bool, bool, uint32_t>> keys = {
+        static const std::vector<std::tuple<bool, bool, bool, uint32_t>> keys = {
-            std::make_tuple(false, false, textured_particle),
+            std::make_tuple(false, false, false, textured_particle),
-            std::make_tuple(true, false, textured_particle_forward),
+            std::make_tuple(true, false, false, textured_particle_forward),
-            std::make_tuple(false, true, textured_particle_translucent),
+            std::make_tuple(false, true, false, textured_particle_translucent),
-            std::make_tuple(true, true, textured_particle_forward)
+            std::make_tuple(true, true, false, textured_particle_translucent_forward),
+            std::make_tuple(false, false, true, textured_particle_shadow),
+            // no such thing as shadow and forward/translucent
         };
 
         for (auto& key : keys) {
@@ -54,13 +55,14 @@ static ShapePipelinePointer shapePipelineFactory(const ShapePlumber& plumber, co
                     gpu::State::FACTOR_ALPHA, gpu::State::BLEND_OP_ADD, gpu::State::ONE);
                 transparent ? PrepareStencil::testMask(*state) : PrepareStencil::testMaskDrawShape(*state);
 
-                auto program = gpu::Shader::createProgram(std::get<2>(key));
+                auto program = gpu::Shader::createProgram(std::get<3>(key));
-                _pipelines[std::make_tuple(std::get<0>(key), transparent, wireframe)] = gpu::Pipeline::create(program, state);
+                _pipelines[std::make_tuple(std::get<0>(key), transparent, std::get<2>(key), wireframe)] = gpu::Pipeline::create(program, state);
             }
         }
     }
 
-    return std::make_shared<render::ShapePipeline>(_pipelines[std::make_tuple(args->_renderMethod == Args::RenderMethod::FORWARD, key.isTranslucent(), key.isWireframe())], nullptr, nullptr, nullptr);
+    return std::make_shared<render::ShapePipeline>(_pipelines[std::make_tuple(args->_renderMethod == Args::RenderMethod::FORWARD, key.isTranslucent(),
+        args->_renderMode == Args::RenderMode::SHADOW_RENDER_MODE, key.isWireframe())], nullptr, nullptr, nullptr);
 }
 
 struct GpuParticle {

libraries/entities-renderer/src/RenderablePolyVoxEntityItem.cpp

@@ -1719,21 +1719,28 @@ using namespace render;
 using namespace render::entities;
 
 static uint8_t CUSTOM_PIPELINE_NUMBER;
-static std::map<std::tuple<bool, bool, bool>, ShapePipelinePointer> _pipelines;
+// forward, shadow, fade, wireframe
+static std::map<std::tuple<bool, bool, bool, bool>, ShapePipelinePointer> _pipelines;
 static gpu::Stream::FormatPointer _vertexFormat;
 
 ShapePipelinePointer shapePipelineFactory(const ShapePlumber& plumber, const ShapeKey& key, RenderArgs* args) {
-    // FIXME: custom pipelines like this don't handle shadows or renderLayers correctly
-
     if (_pipelines.empty()) {
         using namespace shader::entities_renderer::program;
 
-        static const std::vector<std::tuple<bool, bool, uint32_t>> keys = {
+        // forward, shadow, fade
-            std::make_tuple(false, false, polyvox), std::make_tuple(true, false, polyvox_forward)
+        static const std::vector<std::tuple<bool, bool, bool, uint32_t>> keys = {
+            std::make_tuple(false, false, false, polyvox),
+            std::make_tuple(true, false, false, polyvox_forward),
+            std::make_tuple(false, true, false, polyvox_shadow),
+            // no such thing as forward + shadow
 #ifdef POLYVOX_ENTITY_USE_FADE_EFFECT
-            , std::make_tuple(false, true, polyvox_fade), std::make_tuple(true, true, polyvox_forward_fade)
+            std::make_tuple(false, false, true, polyvox_fade),
+            std::make_tuple(false, true, true, polyvox_shadow_fade),
+            // no such thing as forward + fade/shadow
 #else
-            , std::make_tuple(false, true, polyvox), std::make_tuple(true, true, polyvox_forward)
+            std::make_tuple(false, false, true, polyvox),
+            std::make_tuple(false, true, true, polyvox_shadow),
+            // no such thing as forward + fade/shadow
 #endif
         };
         for (auto& key : keys) {
@@ -1749,19 +1756,19 @@ ShapePipelinePointer shapePipelineFactory(const ShapePlumber& plumber, const Sha
                     state->setFillMode(gpu::State::FILL_LINE);
                 }
 
-                auto pipeline = gpu::Pipeline::create(gpu::Shader::createProgram(std::get<2>(key)), state);
+                auto pipeline = gpu::Pipeline::create(gpu::Shader::createProgram(std::get<3>(key)), state);
-                if (std::get<1>(key)) {
+                if (!std::get<2>(key)) {
-                    _pipelines[std::make_tuple(std::get<0>(key), std::get<1>(key), wireframe)] = std::make_shared<render::ShapePipeline>(pipeline, nullptr, nullptr, nullptr);
+                    _pipelines[std::make_tuple(std::get<0>(key), std::get<1>(key), std::get<2>(key), wireframe)] = std::make_shared<render::ShapePipeline>(pipeline, nullptr, nullptr, nullptr);
                 } else {
                     const auto& fadeEffect = DependencyManager::get<FadeEffect>();
-                    _pipelines[std::make_tuple(std::get<0>(key), std::get<1>(key), wireframe)] = std::make_shared<render::ShapePipeline>(pipeline, nullptr,
+                    _pipelines[std::make_tuple(std::get<0>(key), std::get<1>(key), std::get<2>(key), wireframe)] = std::make_shared<render::ShapePipeline>(pipeline, nullptr,
                         fadeEffect->getBatchSetter(), fadeEffect->getItemUniformSetter());
                 }
             }
         }
     }
 
-    return _pipelines[std::make_tuple(args->_renderMethod == Args::RenderMethod::FORWARD, key.isFaded(), key.isWireframe())];
+    return _pipelines[std::make_tuple(args->_renderMethod == Args::RenderMethod::FORWARD, args->_renderMode == Args::RenderMode::SHADOW_RENDER_MODE, key.isFaded(), key.isWireframe())];
 }
 
 PolyVoxEntityRenderer::PolyVoxEntityRenderer(const EntityItemPointer& entity) : Parent(entity) {
@@ -1775,16 +1782,6 @@ PolyVoxEntityRenderer::PolyVoxEntityRenderer(const EntityItemPointer& entity) :
     _params = std::make_shared<gpu::Buffer>(sizeof(glm::vec4), nullptr);
 }
 
-ItemKey PolyVoxEntityRenderer::getKey() {
-    auto builder = ItemKey::Builder::opaqueShape().withTagBits(getTagMask()).withLayer(getHifiRenderLayer());
-
-    if (_cullWithParent) {
-        builder.withSubMetaCulled();
-    }
-
-    return builder.build();
-}
-
 ShapeKey PolyVoxEntityRenderer::getShapeKey() {
     auto builder = ShapeKey::Builder().withCustom(CUSTOM_PIPELINE_NUMBER);
     if (_primitiveMode == PrimitiveMode::LINES) {
@@ -1866,13 +1863,7 @@ void PolyVoxEntityRenderer::doRender(RenderArgs* args) {
     batch.setModelTransform(transform);
 
     batch.setInputFormat(_vertexFormat);
-    batch.setInputBuffer(gpu::Stream::POSITION, _mesh->getVertexBuffer()._buffer, 0,
+    batch.setInputBuffer(gpu::Stream::POSITION, _mesh->getVertexBuffer()._buffer, 0, sizeof(PolyVox::PositionMaterialNormal));
-        sizeof(PolyVox::PositionMaterialNormal));
-
-    // TODO -- should we be setting this?
-    // batch.setInputBuffer(gpu::Stream::NORMAL, mesh->getVertexBuffer()._buffer,
-    //                      12,
-    //                      sizeof(PolyVox::PositionMaterialNormal));
     batch.setIndexBuffer(gpu::UINT32, _mesh->getIndexBuffer()._buffer, 0);
 
     for (size_t i = 0; i < _xyzTextures.size(); ++i) {

libraries/entities-renderer/src/RenderablePolyVoxEntityItem.h

@@ -203,7 +203,6 @@ public:
     }
 
 protected:
-    virtual ItemKey getKey() override;
     virtual ShapeKey getShapeKey() override;
     virtual bool needsRenderUpdateFromTypedEntity(const TypedEntityPointer& entity) const override;
     virtual void doRenderUpdateSynchronousTyped(const ScenePointer& scene, Transaction& transaction, const TypedEntityPointer& entity) override;

libraries/entities-renderer/src/entities-renderer/textured_particle.slp

@@ -1 +1 @@
-DEFINES translucent:f/forward:f
+DEFINES (translucent:f forward:f)/shadow:f

libraries/entities-renderer/src/textured_particle.slf

@@ -15,23 +15,34 @@ LAYOUT(binding=0) uniform sampler2D colorMap;
 layout(location=0) in vec4 varColor;
 layout(location=1) in vec2 varTexcoord;
 
-<@if not HIFI_USE_FORWARD@>
+<@if HIFI_USE_FORWARD or HIFI_USE_SHADOW@>
-    <@include DeferredBufferWrite.slh@>
-<@else@>
     layout(location=0) out vec4 _fragColor0;
+<@else@>
+    <@include DeferredBufferWrite.slh@>
 <@endif@>
 
 void main(void) {
     vec4 albedo = texture(colorMap, varTexcoord.xy) * varColor;
 
-<@if not HIFI_USE_FORWARD@>
+<@if HIFI_USE_FORWARD or HIFI_USE_SHADOW@>
+    <@if not HIFI_USE_TRANSLUCENT@>
+        // to reduce texel flickering for floating point error we discard when alpha is "almost one"
+        if (albedo.a < 0.999999) {
+            discard;
+        }
+    <@endif@>
+
+<@if HIFI_USE_FORWARD@>
+    _fragColor0 = albedo;
+<@else@>
+    _fragColor0 = vec4(1.0);
+<@endif@>
+<@else@>
     vec3 NORMAL = vec3(1.0, 0.0, 0.0);
     <@if not HIFI_USE_TRANSLUCENT@>
         packDeferredFragmentUnlit(NORMAL, albedo.a, albedo.rgb);
     <@else@>
         packDeferredFragmentTranslucent(NORMAL, albedo.a, albedo.rgb, DEFAULT_ROUGHNESS);
     <@endif@>
-<@else@>
-    _fragColor0 = albedo;
 <@endif@>
 }

libraries/render-utils/src/RenderCommonTask.cpp

@@ -85,6 +85,8 @@ void DrawLayered3D::run(const RenderContextPointer& renderContext, const Inputs&
     }
 
     if (!inItems.empty()) {
+        auto deferredLightingEffect = DependencyManager::get<DeferredLightingEffect>();
+
         // Render the items
         gpu::doInBatch("DrawLayered3D::main", args->_context, [&](gpu::Batch& batch) {
             args->_batch = &batch;
@@ -108,11 +110,20 @@ void DrawLayered3D::run(const RenderContextPointer& renderContext, const Inputs&
                 batch.setUniformBuffer(graphics::slot::buffer::Buffer::HazeParams, haze->getHazeParametersBuffer());
             }
 
+            // Set the light
+            deferredLightingEffect->setupKeyLightBatch(args, batch);
+
+            auto renderMethod = args->_renderMethod;
+            args->_renderMethod = Args::RenderMethod::FORWARD;
             if (_opaquePass) {
                 renderStateSortShapes(renderContext, _shapePlumber, inItems, _maxDrawn);
             } else {
                 renderShapes(renderContext, _shapePlumber, inItems, _maxDrawn);
             }
+
+            deferredLightingEffect->unsetLocalLightsBatch(batch);
+
+            args->_renderMethod = renderMethod;
             args->_batch = nullptr;
         });
     }

libraries/render-utils/src/RenderPipelines.cpp

@@ -403,6 +403,7 @@ void initZPassPipelines(ShapePlumber& plumber, gpu::StatePointer state, const re
 
 void sortAndRenderZPassShapes(const ShapePlumberPointer& shapePlumber, const render::RenderContextPointer& renderContext, const render::ShapeBounds& inShapes, render::ItemBounds &itemBounds) {
     std::unordered_map<ShapeKey, std::vector<ShapeKey>, ShapeKey::Hash, ShapeKey::KeyEqual> sortedShapeKeys;
+    std::unordered_map<uint8_t, std::unordered_map<ShapeKey, std::vector<ShapeKey>, ShapeKey::Hash, ShapeKey::KeyEqual>> sortedCustomShapeKeys;
     std::unordered_map<ShapeKey, std::vector<ShapeKey>, ShapeKey::Hash, ShapeKey::KeyEqual> sortedOwnPipelineShapeKeys;
 
     for (const auto& items : inShapes) {
@@ -444,14 +445,25 @@ void sortAndRenderZPassShapes(const ShapePlumberPointer& shapePlumber, const ren
 
         if (items.first.hasOwnPipeline()) {
             sortedOwnPipelineShapeKeys[variantKey.build()].push_back(items.first);
+        } else if (items.first.isCustom()) {
+            const uint8_t custom = items.first.getCustom();
+            variantKey.withCustom(custom);
+            sortedCustomShapeKeys[custom][variantKey.build()].push_back(items.first);
         } else {
             sortedShapeKeys[variantKey.build()].push_back(items.first);
         }
     }
 
-    // Render non-withOwnPipeline things
+    // Render non-withCustom, non-withOwnPipeline things
-    for (auto& variantAndKeys : sortedShapeKeys) {
+    for (const auto& variantAndKeys : sortedShapeKeys) {
-        if (variantAndKeys.second.size() > 0) {
+        for (const auto& key : variantAndKeys.second) {
+            renderShapes(renderContext, shapePlumber, inShapes.at(key));
+        }
+    }
+
+    // Render withCustom things
+    for (const auto& customAndSortedCustomKeys : sortedCustomShapeKeys) {
+        for (const auto& variantAndKeys : customAndSortedCustomKeys.second) {
             for (const auto& key : variantAndKeys.second) {
                 renderShapes(renderContext, shapePlumber, inShapes.at(key));
             }
@@ -459,13 +471,11 @@ void sortAndRenderZPassShapes(const ShapePlumberPointer& shapePlumber, const ren
     }
 
     // Render withOwnPipeline things
-    for (auto& variantAndKeys : sortedOwnPipelineShapeKeys) {
+    for (const auto& variantAndKeys : sortedOwnPipelineShapeKeys) {
-        if (variantAndKeys.second.size() > 0) {
         for (const auto& key : variantAndKeys.second) {
             renderShapes(renderContext, shapePlumber, inShapes.at(key));
         }
     }
-    }
 
     renderContext->args->_shapePipeline = nullptr;
     renderContext->args->_batch = nullptr;

libraries/render/src/render/ShapePipeline.cpp

@@ -60,8 +60,8 @@ ShapeKey::Filter::Builder::Builder() {
 }
 
 void ShapePlumber::addPipelineHelper(const Filter& filter, ShapeKey key, int bit, const PipelinePointer& pipeline) const {
-    // Iterate over all keys
+    // Iterate over all non-custom keys
-    if (bit < (int)ShapeKey::FlagBit::NUM_FLAGS) {
+    if (bit < (int)ShapeKey::FlagBit::NUM_NON_CUSTOM - 1) {
         addPipelineHelper(filter, key, bit + 1, pipeline);
         if (!filter._mask[bit]) {
             // Toggle bits set as insignificant in filter._mask 

libraries/render/src/render/ShapePipeline.h

@@ -54,6 +54,7 @@ public:
         CUSTOM_7,
 
         NUM_FLAGS, // Not a valid flag
+        NUM_NON_CUSTOM = INVALID,
 
         CUSTOM_MASK = (0xFF << CUSTOM_0),