wip hud layering and fix crashes

interface/src/ui/overlays/ModelOverlay.cpp

@@ -88,6 +88,10 @@ void ModelOverlay::update(float deltatime) {
         _drawInFrontDirty = false;
         _model->setLayeredInFront(getDrawInFront(), scene);
     }
+    if (_drawInHUDDirty) {
+        _drawInHUDDirty = false;
+        _model->setLayeredInHUD(getDrawHUDLayer(), scene);
+    }
     scene->enqueueTransaction(transaction);
 }
 
@@ -112,6 +116,11 @@ void ModelOverlay::setDrawInFront(bool drawInFront) {
     _drawInFrontDirty = true;
 }
 
+void ModelOverlay::setDrawHUDLayer(bool drawHUDLayer) {
+    Base3DOverlay::setDrawHUDLayer(drawHUDLayer);
+    _drawInHUDDirty = true;
+}
+
 void ModelOverlay::setProperties(const QVariantMap& properties) {
     auto origPosition = getPosition();
     auto origRotation = getRotation();

interface/src/ui/overlays/ModelOverlay.h

@@ -51,6 +51,7 @@ public:
 
     void setVisible(bool visible) override;
     void setDrawInFront(bool drawInFront) override;
+    void setDrawHUDLayer(bool drawHUDLayer) override;
 
 protected:
     Transform evalRenderTransform() override;
@@ -98,6 +99,7 @@ private:
 
     bool _visibleDirty { false };
     bool _drawInFrontDirty { false };
+    bool _drawInHUDDirty { false };
 
 };
 

interface/src/ui/overlays/OverlaysPayload.cpp

@@ -50,20 +50,17 @@ namespace render {
         return overlay->getBounds();
     }
     template <> int payloadGetLayer(const Overlay::Pointer& overlay) {
-        // Magic number while we are defining the layering mechanism:
-        const int LAYER_2D = 2;
-        const int LAYER_3D_FRONT = 1;
-        const int LAYER_3D = 0;
-
         if (overlay->is3D()) {
             auto overlay3D = std::dynamic_pointer_cast<Base3DOverlay>(overlay);
             if (overlay3D->getDrawInFront()) {
-                return LAYER_3D_FRONT;
+                return Item::LAYER_3D_FRONT;
+            } else if (overlay3D->getDrawHUDLayer()) {
+                return Item::LAYER_3D_HUD;
             } else {
-                return LAYER_3D;
+                return Item::LAYER_3D;
             }
         } else {
-            return LAYER_2D;
+            return Item::LAYER_2D;
         }
     }
     template <> void payloadRender(const Overlay::Pointer& overlay, RenderArgs* args) {

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.cpp

@@ -569,11 +569,11 @@ std::function<void(gpu::Batch&, const gpu::TexturePointer&)> OpenGLDisplayPlugin
         batch.setResourceTexture(0, hudTexture);
         if (isStereo()) {
             for_each_eye([&](Eye eye) {
-                batch.setViewportTransform(eyeViewport(eye));
+                batch.setViewportTransform(eyeViewport(eye, getRecommendedRenderSize()));
                 batch.draw(gpu::TRIANGLE_STRIP, 4);
             });
         } else {
-            batch.setViewportTransform(ivec4(uvec2(0), _compositeFramebuffer->getSize()));
+            batch.setViewportTransform(ivec4(uvec2(0), getRecommendedRenderSize()));
             batch.draw(gpu::TRIANGLE_STRIP, 4);
         }
     };
@@ -838,6 +838,15 @@ ivec4 OpenGLDisplayPlugin::eyeViewport(Eye eye) const {
     return ivec4(vpPos, vpSize);
 }
 
+ivec4 OpenGLDisplayPlugin::eyeViewport(Eye eye, uvec2 vpSize) const {
+    vpSize.x /= 2;
+    uvec2 vpPos;
+    if (eye == Eye::Right) {
+        vpPos.x = vpSize.x;
+    }
+    return ivec4(vpPos, vpSize);
+}
+
 gpu::gl::GLBackend* OpenGLDisplayPlugin::getGLBackend() {
     if (!_gpuContext || !_gpuContext->getBackend()) {
         return nullptr;

libraries/display-plugins/src/display-plugins/OpenGLDisplayPlugin.h

@@ -125,6 +125,7 @@ protected:
     void present();
     virtual void swapBuffers();
     ivec4 eyeViewport(Eye eye) const;
+    ivec4 eyeViewport(Eye eye, uvec2 vpSize) const;
 
     void render(std::function<void(gpu::Batch& batch)> f);
 

libraries/display-plugins/src/display-plugins/hmd/HmdDisplayPlugin.cpp

@@ -430,23 +430,25 @@ void HmdDisplayPlugin::HUDRenderer::updatePipeline() {
 std::function<void(gpu::Batch&, const gpu::TexturePointer&)> HmdDisplayPlugin::HUDRenderer::render(HmdDisplayPlugin& plugin) {
     updatePipeline();
     return [this](gpu::Batch& batch, const gpu::TexturePointer& hudTexture) {
-        batch.setPipeline(pipeline);
+        if (pipeline) {
 
-        batch.setInputFormat(format);
+            batch.setPipeline(pipeline);
-        gpu::BufferView posView(vertices, VERTEX_OFFSET, vertices->getSize(), VERTEX_STRIDE, format->getAttributes().at(gpu::Stream::POSITION)._element);
+            batch.setInputFormat(format);
-        gpu::BufferView uvView(vertices, TEXTURE_OFFSET, vertices->getSize(), VERTEX_STRIDE, format->getAttributes().at(gpu::Stream::TEXCOORD)._element);
+            gpu::BufferView posView(vertices, VERTEX_OFFSET, vertices->getSize(), VERTEX_STRIDE, format->getAttributes().at(gpu::Stream::POSITION)._element);
-        batch.setInputBuffer(gpu::Stream::POSITION, posView);
+            gpu::BufferView uvView(vertices, TEXTURE_OFFSET, vertices->getSize(), VERTEX_STRIDE, format->getAttributes().at(gpu::Stream::TEXCOORD)._element);
-        batch.setInputBuffer(gpu::Stream::TEXCOORD, uvView);
+            batch.setInputBuffer(gpu::Stream::POSITION, posView);
-        batch.setIndexBuffer(gpu::UINT16, indices, 0);
+            batch.setInputBuffer(gpu::Stream::TEXCOORD, uvView);
+            batch.setIndexBuffer(gpu::UINT16, indices, 0);
 
-        uniformsBuffer->setSubData(0, uniforms);
+            uniformsBuffer->setSubData(0, uniforms);
-        batch.setUniformBuffer(uniformsLocation, uniformsBuffer);
+            batch.setUniformBuffer(uniformsLocation, uniformsBuffer);
 
-        auto compositorHelper = DependencyManager::get<CompositorHelper>();
+            auto compositorHelper = DependencyManager::get<CompositorHelper>();
-        batch.setModelTransform(compositorHelper->getUiTransform());
+            batch.setModelTransform(compositorHelper->getUiTransform());
-        batch.setResourceTexture(0, hudTexture);
+            batch.setResourceTexture(0, hudTexture);
 
-        batch.drawIndexed(gpu::TRIANGLES, indexCount);
+            batch.drawIndexed(gpu::TRIANGLES, indexCount);
+        }
     };
 }
 

libraries/render-utils/src/MeshPartPayload.cpp

@@ -404,15 +404,15 @@ ItemKey ModelMeshPartPayload::getKey() const {
 }
 
 int ModelMeshPartPayload::getLayer() const {
-    // MAgic number while we are defining the layering mechanism:
-    const int LAYER_3D_FRONT = 1;
-    const int LAYER_3D = 0;
     ModelPointer model = _model.lock();
-    if (model && model->isLayeredInFront()) {
+    if (model) {
-        return LAYER_3D_FRONT;
+        if (model->isLayeredInFront()) {
-    } else {
+            return Item::LAYER_3D_FRONT;
-        return LAYER_3D;
+        } else if (model->isLayeredInHUD()) {
+            return Item::LAYER_3D_HUD;
+        }
     }
+    return Item::LAYER_3D;
 }
 
 ShapeKey ModelMeshPartPayload::getShapeKey() const {

libraries/render-utils/src/Model.cpp

@@ -598,6 +598,21 @@ void Model::setLayeredInFront(bool layered, const render::ScenePointer& scene) {
     }
 }
 
+void Model::setLayeredInHUD(bool layered, const render::ScenePointer& scene) {
+    if (_isLayeredInHUD != layered) {
+        _isLayeredInHUD = layered;
+
+        render::Transaction transaction;
+        foreach(auto item, _modelMeshRenderItemsMap.keys()) {
+            transaction.resetItem(item, _modelMeshRenderItemsMap[item]);
+        }
+        foreach(auto item, _collisionRenderItemsMap.keys()) {
+            transaction.resetItem(item, _collisionRenderItemsMap[item]);
+        }
+        scene->enqueueTransaction(transaction);
+    }
+}
+
 bool Model::addToScene(const render::ScenePointer& scene,
                        render::Transaction& transaction,
                        render::Item::Status::Getters& statusGetters) {

libraries/render-utils/src/Model.h

@@ -84,6 +84,7 @@ public:
     // new Scene/Engine rendering support
     void setVisibleInScene(bool newValue, const render::ScenePointer& scene);
     void setLayeredInFront(bool layered, const render::ScenePointer& scene);
+    void setLayeredInHUD(bool layered, const render::ScenePointer& scene);
     bool needsFixupInScene() const;
 
     bool needsReload() const { return _needsReload; }

libraries/render-utils/src/RenderDeferredTask.cpp

@@ -187,14 +187,19 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren
     }
 
     // Overlays
-    const auto overlayOpaquesInputs = DrawOverlay3D::Inputs(overlayOpaques, lightingModel).asVarying();
+    const auto filteredOverlaysOpaque = task.addJob<FilterLayeredItems>("FilterOverlaysLayeredOpaque", overlayOpaques, Item::LAYER_3D_FRONT);
-    const auto overlayTransparentsInputs = DrawOverlay3D::Inputs(overlayTransparents, lightingModel).asVarying();
+    const auto filteredOverlaysTransparent = task.addJob<FilterLayeredItems>("FilterOverlaysLayeredTransparent", overlayTransparents, Item::LAYER_3D_FRONT);
-    task.addJob<DrawOverlay3D>("DrawOverlay3DOpaque", overlayOpaquesInputs, true);
+    const auto overlaysInFrontOpaque =  filteredOverlaysOpaque.getN<FilterLayeredItems::Outputs>(0);
-    task.addJob<DrawOverlay3D>("DrawOverlay3DTransparent", overlayTransparentsInputs, false);
+    const auto overlaysInFrontTransparent = filteredOverlaysTransparent.getN<FilterLayeredItems::Outputs>(0);
 
-    { // Debug the bounds of the rendered Overlay items, still look at the zbuffer
+    const auto overlayInFrontOpaquesInputs = DrawOverlay3D::Inputs(overlaysInFrontOpaque, lightingModel).asVarying();
-        task.addJob<DrawBounds>("DrawOverlayOpaqueBounds", overlayOpaques);
+    const auto overlayInFrontTransparentsInputs = DrawOverlay3D::Inputs(overlaysInFrontTransparent, lightingModel).asVarying();
-        task.addJob<DrawBounds>("DrawOverlayTransparentBounds", overlayTransparents);
+    task.addJob<DrawOverlay3D>("DrawOverlayInFrontOpaque", overlayInFrontOpaquesInputs, true);
+    task.addJob<DrawOverlay3D>("DrawOverlayInFrontTransparent", overlayInFrontTransparentsInputs, false);
+
+    { // Debug the bounds of the rendered Overlay items that are marked drawInFront, still look at the zbuffer
+        task.addJob<DrawBounds>("DrawOverlayInFrontOpaqueBounds", overlaysInFrontOpaque);
+        task.addJob<DrawBounds>("DrawOverlayInFrontTransparentBounds", overlaysInFrontTransparent);
     }
 
      // Debugging stages
@@ -233,9 +238,22 @@ void RenderDeferredTask::build(JobModel& task, const render::Varying& input, ren
     // AA job to be revisited
     task.addJob<Antialiasing>("Antialiasing", primaryFramebuffer);
 
-    // Composite the HUD
+    // Composite the HUD and HUD overlays
     task.addJob<CompositeHUD>("HUD");
 
+    const auto overlaysHUDOpaque = filteredOverlaysOpaque.getN<FilterLayeredItems::Outputs>(1);
+    const auto overlaysHUDTransparent = filteredOverlaysTransparent.getN<FilterLayeredItems::Outputs>(1);
+
+    const auto overlayHUDOpaquesInputs = DrawOverlay3D::Inputs(overlaysHUDOpaque, lightingModel).asVarying();
+    const auto overlayHUDTransparentsInputs = DrawOverlay3D::Inputs(overlaysHUDTransparent, lightingModel).asVarying();
+    task.addJob<DrawOverlay3D>("DrawOverlayHUDOpaque", overlayHUDOpaquesInputs, true);
+    task.addJob<DrawOverlay3D>("DrawOverlayHUDTransparent", overlayHUDTransparentsInputs, false);
+
+    { // Debug the bounds of the rendered Overlay items that are marked drawHUDLayer, still look at the zbuffer
+        task.addJob<DrawBounds>("DrawOverlayHUDOpaqueBounds", overlaysHUDOpaque);
+        task.addJob<DrawBounds>("DrawOverlayHUDOpaqueBounds", overlaysHUDTransparent);
+    }
+
     task.addJob<EndGPURangeTimer>("ToneAndPostRangeTimer", toneAndPostRangeTimer);
 
     // Blit!

libraries/render/src/render/FilterTask.cpp

@@ -21,19 +21,24 @@
 
 using namespace render;
 
-void FilterLayeredItems::run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems) {
+void FilterLayeredItems::run(const RenderContextPointer& renderContext, const ItemBounds& inItems, Outputs& outputs) {
     auto& scene = renderContext->_scene;
 
-    // Clear previous values
+    ItemBounds matchedItems;
-    outItems.clear();
+    ItemBounds nonMatchItems;
 
     // For each item, filter it into one bucket
     for (auto& itemBound : inItems) {
         auto& item = scene->getItem(itemBound.id);
         if (item.getLayer() == _keepLayer) {
-            outItems.emplace_back(itemBound);
+            matchedItems.emplace_back(itemBound);
+        } else {
+            nonMatchItems.emplace_back(itemBound);
         }
     }
+
+    outputs.edit0() = matchedItems;
+    outputs.edit1() = nonMatchItems;
 }
 
 void SliceItems::run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems) {

libraries/render/src/render/FilterTask.h

@@ -65,15 +65,15 @@ namespace render {
     // Filter the items belonging to the job's keep layer
     class FilterLayeredItems {
     public:
-        using JobModel = Job::ModelIO<FilterLayeredItems, ItemBounds, ItemBounds>;
+        using Outputs = render::VaryingSet2<ItemBounds, ItemBounds>;
+        using JobModel = Job::ModelIO<FilterLayeredItems, ItemBounds, Outputs>;
 
-        FilterLayeredItems() {}
         FilterLayeredItems(int keepLayer) :
             _keepLayer(keepLayer) {}
 
-        int _keepLayer { 0 };
+        int _keepLayer;
 
-        void run(const RenderContextPointer& renderContext, const ItemBounds& inItems, ItemBounds& outItems);
+        void run(const RenderContextPointer& renderContext, const ItemBounds& inItems, Outputs& outputs);
     };
 
     // SliceItems job config defining the slice range

libraries/render/src/render/Item.h

@@ -356,9 +356,14 @@ public:
     // Get the bound of the item expressed in world space (or eye space depending on the key.isWorldSpace())
     const Bound getBound() const { return _payload->getBound(); }
 
-    // Get the layer where the item belongs. 0 by default meaning NOT LAYERED
+    // Get the layer where the item belongs.
     int getLayer() const { return _payload->getLayer(); }
 
+    static const int LAYER_2D = 0;
+    static const int LAYER_3D = 1;
+    static const int LAYER_3D_FRONT = 2;
+    static const int LAYER_3D_HUD = 3;
+
     // Render call for the item
     void render(RenderArgs* args) const { _payload->render(args); }