Merge pull request #12990 from Atlante45/fix/spec-cam-av

Add support for multifrustum avatar queries

assignment-client/src/Agent.cpp

@@ -548,16 +548,21 @@ void Agent::setIsAvatar(bool isAvatar) {
     if (_isAvatar && !_avatarIdentityTimer) {
         // set up the avatar timers
         _avatarIdentityTimer = new QTimer(this);
+        _avatarViewTimer = new QTimer(this);
 
         // connect our slot
         connect(_avatarIdentityTimer, &QTimer::timeout, this, &Agent::sendAvatarIdentityPacket);
+        connect(_avatarViewTimer, &QTimer::timeout, this, &Agent::sendAvatarViewFrustum);
+
+        static const int AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS = 1000;
+        static const int AVATAR_VIEW_PACKET_SEND_INTERVAL_MSECS = 1000;
 
         // start the timers
         _avatarIdentityTimer->start(AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS);  // FIXME - we shouldn't really need to constantly send identity packets
+        _avatarViewTimer->start(AVATAR_VIEW_PACKET_SEND_INTERVAL_MSECS);
 
         // tell the avatarAudioTimer to start ticking
         QMetaObject::invokeMethod(&_avatarAudioTimer, "start");
-
     }
 
     if (!_isAvatar) {
@@ -567,6 +572,10 @@ void Agent::setIsAvatar(bool isAvatar) {
             delete _avatarIdentityTimer;
             _avatarIdentityTimer = nullptr;
 
+            _avatarViewTimer->stop();
+            delete _avatarViewTimer;
+            _avatarViewTimer = nullptr;
+
             // The avatar mixer never times out a connection (e.g., based on identity or data packets)
             // but rather keeps avatars in its list as long as "connected". As a result, clients timeout
             // when we stop sending identity, but then get woken up again by the mixer itself, which sends
@@ -585,6 +594,7 @@ void Agent::setIsAvatar(bool isAvatar) {
                 nodeList->sendPacket(std::move(packet), *node);
             });
         }
+
         QMetaObject::invokeMethod(&_avatarAudioTimer, "stop");
     }
 }
@@ -597,6 +607,25 @@ void Agent::sendAvatarIdentityPacket() {
     }
 }
 
+void Agent::sendAvatarViewFrustum() {
+    auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();
+
+    ViewFrustum view;
+    view.setPosition(scriptedAvatar->getWorldPosition());
+    view.setOrientation(scriptedAvatar->getHeadOrientation());
+    view.calculate();
+
+    uint8_t numFrustums = 1;
+    auto viewFrustumByteArray = view.toByteArray();
+
+    auto avatarPacket = NLPacket::create(PacketType::ViewFrustum, viewFrustumByteArray.size() + sizeof(numFrustums));
+    avatarPacket->writePrimitive(numFrustums);
+    avatarPacket->write(viewFrustumByteArray);
+
+    DependencyManager::get<NodeList>()->broadcastToNodes(std::move(avatarPacket),
+                                                         { NodeType::AvatarMixer });
+}
+
 void Agent::processAgentAvatar() {
     if (!_scriptEngine->isFinished() && _isAvatar) {
         auto scriptedAvatar = DependencyManager::get<ScriptableAvatar>();

assignment-client/src/Agent.h

@@ -97,6 +97,7 @@ private:
     void setAvatarSound(SharedSoundPointer avatarSound) { _avatarSound = avatarSound; }
 
     void sendAvatarIdentityPacket();
+    void sendAvatarViewFrustum();
 
     QString _scriptContents;
     QTimer* _scriptRequestTimeout { nullptr };
@@ -106,6 +107,7 @@ private:
     int _numAvatarSoundSentBytes = 0;
     bool _isAvatar = false;
     QTimer* _avatarIdentityTimer = nullptr;
+    QTimer* _avatarViewTimer = nullptr;
     QHash<QUuid, quint16> _outgoingScriptAudioSequenceNumbers;
 
     AudioGate _audioGate;

assignment-client/src/avatars/AvatarMixer.cpp

@@ -521,11 +521,9 @@ void AvatarMixer::handleViewFrustumPacket(QSharedPointer<ReceivedMessage> messag
     auto start = usecTimestampNow();
     getOrCreateClientData(senderNode);
 
-    if (senderNode->getLinkedData()) {
+    AvatarMixerClientData* nodeData = dynamic_cast<AvatarMixerClientData*>(senderNode->getLinkedData());
-        AvatarMixerClientData* nodeData = dynamic_cast<AvatarMixerClientData*>(senderNode->getLinkedData());
+    if (nodeData) {
-        if (nodeData != nullptr) {
+        nodeData->readViewFrustumPacket(message->getMessage());
-            nodeData->readViewFrustumPacket(message->getMessage());
-        }
     }
 
     auto end = usecTimestampNow();

assignment-client/src/avatars/AvatarMixerClientData.cpp

@@ -19,8 +19,6 @@
 AvatarMixerClientData::AvatarMixerClientData(const QUuid& nodeID) :
     NodeData(nodeID)
 {
-    _currentViewFrustum.invalidate();
-
     // in case somebody calls getSessionUUID on the AvatarData instance, make sure it has the right ID
     _avatar->setID(nodeID);
 }
@@ -128,12 +126,27 @@ void AvatarMixerClientData::removeFromRadiusIgnoringSet(SharedNodePointer self,
     }
 }
 
-void AvatarMixerClientData::readViewFrustumPacket(const QByteArray& message) {
+void AvatarMixerClientData::readViewFrustumPacket(QByteArray message) {
-    _currentViewFrustum.fromByteArray(message);
+    _currentViewFrustums.clear();
+    
+    uint8_t numFrustums = 0;
+    memcpy(&numFrustums, message.constData(), sizeof(numFrustums));
+    message.remove(0, sizeof(numFrustums));
+
+    for (uint8_t i = 0; i < numFrustums; ++i) {
+        ViewFrustum frustum;
+        auto bytesRead = frustum.fromByteArray(message);
+        message.remove(0, bytesRead);
+
+        _currentViewFrustums.push_back(frustum);
+    }
 }
 
 bool AvatarMixerClientData::otherAvatarInView(const AABox& otherAvatarBox) {
-    return _currentViewFrustum.boxIntersectsKeyhole(otherAvatarBox);
+    return std::any_of(std::begin(_currentViewFrustums), std::end(_currentViewFrustums),
+                       [&](const ViewFrustum& viewFrustum) {
+        return viewFrustum.boxIntersectsKeyhole(otherAvatarBox);
+    });
 }
 
 void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const {

assignment-client/src/avatars/AvatarMixerClientData.h

@@ -98,7 +98,7 @@ public:
     void removeFromRadiusIgnoringSet(SharedNodePointer self, const QUuid& other);
     void ignoreOther(SharedNodePointer self, SharedNodePointer other);
 
-    void readViewFrustumPacket(const QByteArray& message);
+    void readViewFrustumPacket(QByteArray message);
 
     bool otherAvatarInView(const AABox& otherAvatarBox);
 
@@ -110,7 +110,7 @@ public:
     bool getRequestsDomainListData() { return _requestsDomainListData; }
     void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; }
 
-    ViewFrustum getViewFrustum() const { return _currentViewFrustum; }
+    const ViewFrustums& getViewFrustums() const { return _currentViewFrustums; }
 
     uint64_t getLastOtherAvatarEncodeTime(QUuid otherAvatar) const;
     void setLastOtherAvatarEncodeTime(const QUuid& otherAvatar, uint64_t time);
@@ -150,7 +150,7 @@ private:
 
     SimpleMovingAverage _avgOtherAvatarDataRate;
     std::unordered_set<QUuid> _radiusIgnoredOthers;
-    ViewFrustum _currentViewFrustum;
+    ViewFrustums _currentViewFrustums;
 
     int _recentOtherAvatarsInView { 0 };
     int _recentOtherAvatarsOutOfView { 0 };

assignment-client/src/avatars/AvatarMixerSlave.cpp

@@ -222,8 +222,8 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node)
     };
 
     // prepare to sort
-    ViewFrustum cameraView = nodeData->getViewFrustum();
+    const auto& cameraViews = nodeData->getViewFrustums();
-    PrioritySortUtil::PriorityQueue<SortableAvatar> sortedAvatars(cameraView,
+    PrioritySortUtil::PriorityQueue<SortableAvatar> sortedAvatars(cameraViews,
             AvatarData::_avatarSortCoefficientSize,
             AvatarData::_avatarSortCoefficientCenter,
             AvatarData::_avatarSortCoefficientAge);

interface/src/Application.cpp

@@ -5844,8 +5844,16 @@ void Application::update(float deltaTime) {
 }
 
 void Application::sendAvatarViewFrustum() {
+    uint8_t numFrustums = 1;
     QByteArray viewFrustumByteArray = _viewFrustum.toByteArray();
-    auto avatarPacket = NLPacket::create(PacketType::ViewFrustum, viewFrustumByteArray.size());
+
+    if (_hasSecondaryViewFrustum) {
+        ++numFrustums;
+        viewFrustumByteArray += _secondaryViewFrustum.toByteArray();
+    }
+
+    auto avatarPacket = NLPacket::create(PacketType::ViewFrustum, viewFrustumByteArray.size() + sizeof(numFrustums));
+    avatarPacket->writePrimitive(numFrustums);
     avatarPacket->write(viewFrustumByteArray);
 
     DependencyManager::get<NodeList>()->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer);

interface/src/Application.h

@@ -175,13 +175,13 @@ public:
     Camera& getCamera() { return _myCamera; }
     const Camera& getCamera() const { return _myCamera; }
     // Represents the current view frustum of the avatar.
-    void copyViewFrustum(ViewFrustum& viewOut) const;
+    void copyViewFrustum(ViewFrustum& viewOut) const override;
+    void copySecondaryViewFrustum(ViewFrustum& viewOut) const override;
+    bool hasSecondaryViewFrustum() const override { return _hasSecondaryViewFrustum; }
     // Represents the view frustum of the current rendering pass,
     // which might be different from the viewFrustum, i.e. shadowmap
     // passes, mirror window passes, etc
     void copyDisplayViewFrustum(ViewFrustum& viewOut) const;
-    void copySecondaryViewFrustum(ViewFrustum& viewOut) const;
-    bool hasSecondaryViewFrustum() const { return _hasSecondaryViewFrustum; }
 
     const OctreePacketProcessor& getOctreePacketProcessor() const { return _octreeProcessor; }
     QSharedPointer<EntityTreeRenderer> getEntities() const { return DependencyManager::get<EntityTreeRenderer>(); }

interface/src/avatar/AvatarManager.cpp

@@ -155,9 +155,19 @@ void AvatarManager::updateOtherAvatars(float deltaTime) {
         AvatarSharedPointer _avatar;
     };
 
-    ViewFrustum cameraView;
+
-    qApp->copyDisplayViewFrustum(cameraView);
+    ViewFrustums views;
-    PrioritySortUtil::PriorityQueue<SortableAvatar> sortedAvatars(cameraView,
+
+    ViewFrustum view;
+    qApp->copyCurrentViewFrustum(view);
+    views.push_back(view);
+
+    if (qApp->hasSecondaryViewFrustum()) {
+        qApp->copySecondaryViewFrustum(view);
+        views.push_back(view);
+    }
+
+    PrioritySortUtil::PriorityQueue<SortableAvatar> sortedAvatars(views,
             AvatarData::_avatarSortCoefficientSize,
             AvatarData::_avatarSortCoefficientCenter,
             AvatarData::_avatarSortCoefficientAge);

libraries/avatars/src/AvatarData.h

@@ -277,8 +277,6 @@ namespace AvatarDataPacket {
 
 const float MAX_AUDIO_LOUDNESS = 1000.0f; // close enough for mouth animation
 
-const int AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS = 1000;
-
 // See also static AvatarData::defaultFullAvatarModelUrl().
 const QString DEFAULT_FULL_AVATAR_MODEL_NAME = QString("Default");
 

libraries/entities-renderer/src/EntityTreeRenderer.cpp

@@ -296,7 +296,8 @@ void EntityTreeRenderer::addPendingEntities(const render::ScenePointer& scene, r
     }
 }
 
-void EntityTreeRenderer::updateChangedEntities(const render::ScenePointer& scene, const ViewFrustum& view, render::Transaction& transaction) {
+void EntityTreeRenderer::updateChangedEntities(const render::ScenePointer& scene, const ViewFrustums& views,
+                                               render::Transaction& transaction) {
     PROFILE_RANGE_EX(simulation_physics, "ChangeInScene", 0xffff00ff, (uint64_t)_changedEntities.size());
     PerformanceTimer pt("change");
     std::unordered_set<EntityItemID> changedEntities;
@@ -357,7 +358,7 @@ void EntityTreeRenderer::updateChangedEntities(const render::ScenePointer& scene
 
         // prioritize and sort the renderables
         uint64_t sortStart = usecTimestampNow();
-        PrioritySortUtil::PriorityQueue<SortableRenderer> sortedRenderables(view);
+        PrioritySortUtil::PriorityQueue<SortableRenderer> sortedRenderables(views);
         {
             PROFILE_RANGE_EX(simulation_physics, "SortRenderables", 0xffff00ff, (uint64_t)_renderablesToUpdate.size());
             std::unordered_map<EntityItemID, EntityRendererPointer>::iterator itr = _renderablesToUpdate.begin();
@@ -415,9 +416,20 @@ void EntityTreeRenderer::update(bool simulate) {
             if (scene) {
                 render::Transaction transaction;
                 addPendingEntities(scene, transaction);
+
+                ViewFrustums views;
+
                 ViewFrustum view;
-                _viewState->copyCurrentViewFrustum(view);
+                _viewState->copyViewFrustum(view);
-                updateChangedEntities(scene, view, transaction);
+                views.push_back(view);
+
+                if (_viewState->hasSecondaryViewFrustum()) {
+                    _viewState->copySecondaryViewFrustum(view);
+                    views.push_back(view);
+                }
+
+
+                updateChangedEntities(scene, views, transaction);
                 scene->enqueueTransaction(transaction);
             }
         }

libraries/entities-renderer/src/EntityTreeRenderer.h

@@ -148,7 +148,8 @@ protected:
 
 private:
     void addPendingEntities(const render::ScenePointer& scene, render::Transaction& transaction);
-    void updateChangedEntities(const render::ScenePointer& scene, const ViewFrustum& view, render::Transaction& transaction);
+    void updateChangedEntities(const render::ScenePointer& scene, const ViewFrustums& views,
+                               render::Transaction& transaction);
     EntityRendererPointer renderableForEntity(const EntityItemPointer& entity) const { return renderableForEntityId(entity->getID()); }
     render::ItemID renderableIdForEntity(const EntityItemPointer& entity) const { return renderableIdForEntityId(entity->getID()); }
 

libraries/entities/src/DiffTraversal.h

@@ -36,7 +36,7 @@ public:
         bool isVerySimilar(const View& view) const;
         ViewFrustum::intersection calculateIntersection(const AACube& cube) const;
 
-        std::vector<ViewFrustum> viewFrustums;
+        ViewFrustums viewFrustums;
         uint64_t startTime { 0 };
         float lodScaleFactor { 1.0f };
     };

libraries/networking/src/udt/PacketHeaders.cpp

@@ -90,6 +90,8 @@ PacketVersion versionForPacketType(PacketType packetType) {
             return 18;  // replace min_avatar_scale and max_avatar_scale with min_avatar_height and max_avatar_height
         case PacketType::Ping:
             return static_cast<PacketVersion>(PingVersion::IncludeConnectionID);
+        case PacketType::ViewFrustum:
+            return static_cast<PacketVersion>(ViewFrustumVersion::SendMultipleFrustums);
         default:
             return 20;
     }

libraries/networking/src/udt/PacketHeaders.h

@@ -328,4 +328,8 @@ enum class PingVersion : PacketVersion {
     IncludeConnectionID = 18
 };
 
+enum class ViewFrustumVersion : PacketVersion {
+    SendMultipleFrustums = 21
+};
+
 #endif // hifi_PacketHeaders_h

libraries/render-utils/src/AbstractViewStateInterface.h

@@ -31,6 +31,10 @@ public:
     /// copies the current view frustum for rendering the view state
     virtual void copyCurrentViewFrustum(ViewFrustum& viewOut) const = 0;
 
+    virtual void copyViewFrustum(ViewFrustum& viewOut) const = 0;
+    virtual void copySecondaryViewFrustum(ViewFrustum& viewOut) const = 0;
+    virtual bool hasSecondaryViewFrustum() const = 0;
+
     virtual QThread* getMainThread() = 0;
 
     virtual PickRay computePickRay(float x, float y) const = 0;

libraries/shared/src/PrioritySortUtil.h

@@ -84,14 +84,12 @@ namespace PrioritySortUtil {
     class PriorityQueue {
     public:
         PriorityQueue() = delete;
-
+        PriorityQueue(const ViewFrustums& views) : _views(views) { }
-        PriorityQueue(const ViewFrustum& view) : _view(view) { }
+        PriorityQueue(const ViewFrustums& views, float angularWeight, float centerWeight, float ageWeight)
-
+                : _views(views), _angularWeight(angularWeight), _centerWeight(centerWeight), _ageWeight(ageWeight)
-        PriorityQueue(const ViewFrustum& view, float angularWeight, float centerWeight, float ageWeight)
-                : _view(view), _angularWeight(angularWeight), _centerWeight(centerWeight), _ageWeight(ageWeight)
         { }
 
-        void setView(const ViewFrustum& view) { _view = view; }
+        void setViews(const ViewFrustums& views) { _views = views; }
 
         void setWeights(float angularWeight, float centerWeight, float ageWeight) {
             _angularWeight = angularWeight;
@@ -109,7 +107,18 @@ namespace PrioritySortUtil {
         bool empty() const { return _queue.empty(); }
 
     private:
+
         float computePriority(const T& thing) const {
+            float priority = std::numeric_limits<float>::min();
+
+            for (const auto& view : _views) {
+                priority = std::max(priority, computePriority(view, thing));
+            }
+
+            return priority;
+        }
+
+        float computePriority(const ViewFrustum& view, const T& thing) const {
             // priority = weighted linear combination of multiple values:
             //   (a) angular size
             //   (b) proximity to center of view
@@ -117,11 +126,11 @@ namespace PrioritySortUtil {
             // where the relative "weights" are tuned to scale the contributing values into units of "priority".
 
             glm::vec3 position = thing.getPosition();
-            glm::vec3 offset = position - _view.getPosition();
+            glm::vec3 offset = position - view.getPosition();
             float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero
             const float MIN_RADIUS = 0.1f; // WORKAROUND for zero size objects (we still want them to sort by distance)
             float radius = glm::min(thing.getRadius(), MIN_RADIUS);
-            float cosineAngle = (glm::dot(offset, _view.getDirection()) / distance);
+            float cosineAngle = (glm::dot(offset, view.getDirection()) / distance);
             float age = (float)(usecTimestampNow() - thing.getTimestamp());
 
             // we modulatate "age" drift rate by the cosineAngle term to make periphrial objects sort forward
@@ -134,8 +143,8 @@ namespace PrioritySortUtil {
                 + _ageWeight * cosineAngleFactor * age;
 
             // decrement priority of things outside keyhole
-            if (distance - radius > _view.getCenterRadius()) {
+            if (distance - radius > view.getCenterRadius()) {
-                if (!_view.sphereIntersectsFrustum(position, radius)) {
+                if (!view.sphereIntersectsFrustum(position, radius)) {
                     constexpr float OUT_OF_VIEW_PENALTY = -10.0f;
                     priority += OUT_OF_VIEW_PENALTY;
                 }
@@ -143,7 +152,7 @@ namespace PrioritySortUtil {
             return priority;
         }
 
-        ViewFrustum _view;
+        ViewFrustums _views;
         std::priority_queue<T> _queue;
         float _angularWeight { DEFAULT_ANGULAR_COEF };
         float _centerWeight { DEFAULT_CENTER_COEF };

libraries/shared/src/ViewFrustum.h

@@ -189,5 +189,6 @@ private:
 
 };
 using ViewFrustumPointer = std::shared_ptr<ViewFrustum>;
+using ViewFrustums = std::vector<ViewFrustum>;
 
 #endif // hifi_ViewFrustum_h

tests/render-perf/src/main.cpp

@@ -441,6 +441,16 @@ protected:
         viewOut = _viewFrustum;
     }
 
+    void copyViewFrustum(ViewFrustum& viewOut) const override {
+        viewOut = _viewFrustum;
+    }
+
+    void copySecondaryViewFrustum(ViewFrustum& viewOut) const override {}
+
+    bool hasSecondaryViewFrustum() const override {
+        return false;
+    }
+
     QThread* getMainThread() override {
         return QThread::currentThread();
     }