diff --git a/assignment-client/src/Agent.cpp b/assignment-client/src/Agent.cpp index 1df901dd98..0b373e511b 100644 --- a/assignment-client/src/Agent.cpp +++ b/assignment-client/src/Agent.cpp @@ -548,16 +548,21 @@ void Agent::setIsAvatar(bool isAvatar) { if (_isAvatar && !_avatarIdentityTimer) { // set up the avatar timers _avatarIdentityTimer = new QTimer(this); + _avatarViewTimer = new QTimer(this); // connect our slot connect(_avatarIdentityTimer, &QTimer::timeout, this, &Agent::sendAvatarIdentityPacket); + connect(_avatarViewTimer, &QTimer::timeout, this, &Agent::sendAvatarViewFrustum); + + static const int AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS = 1000; + static const int AVATAR_VIEW_PACKET_SEND_INTERVAL_MSECS = 1000; // start the timers _avatarIdentityTimer->start(AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS); // FIXME - we shouldn't really need to constantly send identity packets + _avatarViewTimer->start(AVATAR_VIEW_PACKET_SEND_INTERVAL_MSECS); // tell the avatarAudioTimer to start ticking QMetaObject::invokeMethod(&_avatarAudioTimer, "start"); - } if (!_isAvatar) { @@ -567,6 +572,10 @@ void Agent::setIsAvatar(bool isAvatar) { delete _avatarIdentityTimer; _avatarIdentityTimer = nullptr; + _avatarViewTimer->stop(); + delete _avatarViewTimer; + _avatarViewTimer = nullptr; + // The avatar mixer never times out a connection (e.g., based on identity or data packets) // but rather keeps avatars in its list as long as "connected". As a result, clients timeout // when we stop sending identity, but then get woken up again by the mixer itself, which sends @@ -585,6 +594,7 @@ void Agent::setIsAvatar(bool isAvatar) { nodeList->sendPacket(std::move(packet), *node); }); } + QMetaObject::invokeMethod(&_avatarAudioTimer, "stop"); } } @@ -597,6 +607,25 @@ void Agent::sendAvatarIdentityPacket() { } } +void Agent::sendAvatarViewFrustum() { + auto scriptedAvatar = DependencyManager::get(); + + ViewFrustum view; + view.setPosition(scriptedAvatar->getWorldPosition()); + view.setOrientation(scriptedAvatar->getHeadOrientation()); + view.calculate(); + + uint8_t numFrustums = 1; + auto viewFrustumByteArray = view.toByteArray(); + + auto avatarPacket = NLPacket::create(PacketType::ViewFrustum, viewFrustumByteArray.size() + sizeof(numFrustums)); + avatarPacket->writePrimitive(numFrustums); + avatarPacket->write(viewFrustumByteArray); + + DependencyManager::get()->broadcastToNodes(std::move(avatarPacket), + { NodeType::AvatarMixer }); +} + void Agent::processAgentAvatar() { if (!_scriptEngine->isFinished() && _isAvatar) { auto scriptedAvatar = DependencyManager::get(); diff --git a/assignment-client/src/Agent.h b/assignment-client/src/Agent.h index 1229f06276..d144f0bc01 100644 --- a/assignment-client/src/Agent.h +++ b/assignment-client/src/Agent.h @@ -97,6 +97,7 @@ private: void setAvatarSound(SharedSoundPointer avatarSound) { _avatarSound = avatarSound; } void sendAvatarIdentityPacket(); + void sendAvatarViewFrustum(); QString _scriptContents; QTimer* _scriptRequestTimeout { nullptr }; @@ -106,6 +107,7 @@ private: int _numAvatarSoundSentBytes = 0; bool _isAvatar = false; QTimer* _avatarIdentityTimer = nullptr; + QTimer* _avatarViewTimer = nullptr; QHash _outgoingScriptAudioSequenceNumbers; AudioGate _audioGate; diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 29340f6474..6353a1664f 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -521,11 +521,9 @@ void AvatarMixer::handleViewFrustumPacket(QSharedPointer messag auto start = usecTimestampNow(); getOrCreateClientData(senderNode); - if (senderNode->getLinkedData()) { - AvatarMixerClientData* nodeData = dynamic_cast(senderNode->getLinkedData()); - if (nodeData != nullptr) { - nodeData->readViewFrustumPacket(message->getMessage()); - } + AvatarMixerClientData* nodeData = dynamic_cast(senderNode->getLinkedData()); + if (nodeData) { + nodeData->readViewFrustumPacket(message->getMessage()); } auto end = usecTimestampNow(); diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 268aba62d6..a8e5a7c541 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -19,8 +19,6 @@ AvatarMixerClientData::AvatarMixerClientData(const QUuid& nodeID) : NodeData(nodeID) { - _currentViewFrustum.invalidate(); - // in case somebody calls getSessionUUID on the AvatarData instance, make sure it has the right ID _avatar->setID(nodeID); } @@ -128,12 +126,27 @@ void AvatarMixerClientData::removeFromRadiusIgnoringSet(SharedNodePointer self, } } -void AvatarMixerClientData::readViewFrustumPacket(const QByteArray& message) { - _currentViewFrustum.fromByteArray(message); +void AvatarMixerClientData::readViewFrustumPacket(QByteArray message) { + _currentViewFrustums.clear(); + + uint8_t numFrustums = 0; + memcpy(&numFrustums, message.constData(), sizeof(numFrustums)); + message.remove(0, sizeof(numFrustums)); + + for (uint8_t i = 0; i < numFrustums; ++i) { + ViewFrustum frustum; + auto bytesRead = frustum.fromByteArray(message); + message.remove(0, bytesRead); + + _currentViewFrustums.push_back(frustum); + } } bool AvatarMixerClientData::otherAvatarInView(const AABox& otherAvatarBox) { - return _currentViewFrustum.boxIntersectsKeyhole(otherAvatarBox); + return std::any_of(std::begin(_currentViewFrustums), std::end(_currentViewFrustums), + [&](const ViewFrustum& viewFrustum) { + return viewFrustum.boxIntersectsKeyhole(otherAvatarBox); + }); } void AvatarMixerClientData::loadJSONStats(QJsonObject& jsonObject) const { diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index 6963f4df0d..f17404b79f 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -98,7 +98,7 @@ public: void removeFromRadiusIgnoringSet(SharedNodePointer self, const QUuid& other); void ignoreOther(SharedNodePointer self, SharedNodePointer other); - void readViewFrustumPacket(const QByteArray& message); + void readViewFrustumPacket(QByteArray message); bool otherAvatarInView(const AABox& otherAvatarBox); @@ -110,7 +110,7 @@ public: bool getRequestsDomainListData() { return _requestsDomainListData; } void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; } - ViewFrustum getViewFrustum() const { return _currentViewFrustum; } + const ViewFrustums& getViewFrustums() const { return _currentViewFrustums; } uint64_t getLastOtherAvatarEncodeTime(QUuid otherAvatar) const; void setLastOtherAvatarEncodeTime(const QUuid& otherAvatar, uint64_t time); @@ -150,7 +150,7 @@ private: SimpleMovingAverage _avgOtherAvatarDataRate; std::unordered_set _radiusIgnoredOthers; - ViewFrustum _currentViewFrustum; + ViewFrustums _currentViewFrustums; int _recentOtherAvatarsInView { 0 }; int _recentOtherAvatarsOutOfView { 0 }; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 6f19b73cc5..30d94ed772 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -222,8 +222,8 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) }; // prepare to sort - ViewFrustum cameraView = nodeData->getViewFrustum(); - PrioritySortUtil::PriorityQueue sortedAvatars(cameraView, + const auto& cameraViews = nodeData->getViewFrustums(); + PrioritySortUtil::PriorityQueue sortedAvatars(cameraViews, AvatarData::_avatarSortCoefficientSize, AvatarData::_avatarSortCoefficientCenter, AvatarData::_avatarSortCoefficientAge); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index d97b14fa1c..5e10d29505 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -5844,8 +5844,16 @@ void Application::update(float deltaTime) { } void Application::sendAvatarViewFrustum() { + uint8_t numFrustums = 1; QByteArray viewFrustumByteArray = _viewFrustum.toByteArray(); - auto avatarPacket = NLPacket::create(PacketType::ViewFrustum, viewFrustumByteArray.size()); + + if (_hasSecondaryViewFrustum) { + ++numFrustums; + viewFrustumByteArray += _secondaryViewFrustum.toByteArray(); + } + + auto avatarPacket = NLPacket::create(PacketType::ViewFrustum, viewFrustumByteArray.size() + sizeof(numFrustums)); + avatarPacket->writePrimitive(numFrustums); avatarPacket->write(viewFrustumByteArray); DependencyManager::get()->broadcastToNodes(std::move(avatarPacket), NodeSet() << NodeType::AvatarMixer); diff --git a/interface/src/Application.h b/interface/src/Application.h index baaa886910..83e719333d 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -175,13 +175,13 @@ public: Camera& getCamera() { return _myCamera; } const Camera& getCamera() const { return _myCamera; } // Represents the current view frustum of the avatar. - void copyViewFrustum(ViewFrustum& viewOut) const; + void copyViewFrustum(ViewFrustum& viewOut) const override; + void copySecondaryViewFrustum(ViewFrustum& viewOut) const override; + bool hasSecondaryViewFrustum() const override { return _hasSecondaryViewFrustum; } // Represents the view frustum of the current rendering pass, // which might be different from the viewFrustum, i.e. shadowmap // passes, mirror window passes, etc void copyDisplayViewFrustum(ViewFrustum& viewOut) const; - void copySecondaryViewFrustum(ViewFrustum& viewOut) const; - bool hasSecondaryViewFrustum() const { return _hasSecondaryViewFrustum; } const OctreePacketProcessor& getOctreePacketProcessor() const { return _octreeProcessor; } QSharedPointer getEntities() const { return DependencyManager::get(); } diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index b71c060465..087e23a933 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -155,9 +155,19 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { AvatarSharedPointer _avatar; }; - ViewFrustum cameraView; - qApp->copyDisplayViewFrustum(cameraView); - PrioritySortUtil::PriorityQueue sortedAvatars(cameraView, + + ViewFrustums views; + + ViewFrustum view; + qApp->copyCurrentViewFrustum(view); + views.push_back(view); + + if (qApp->hasSecondaryViewFrustum()) { + qApp->copySecondaryViewFrustum(view); + views.push_back(view); + } + + PrioritySortUtil::PriorityQueue sortedAvatars(views, AvatarData::_avatarSortCoefficientSize, AvatarData::_avatarSortCoefficientCenter, AvatarData::_avatarSortCoefficientAge); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 888c3bfb24..bbcdd3693d 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -277,8 +277,6 @@ namespace AvatarDataPacket { const float MAX_AUDIO_LOUDNESS = 1000.0f; // close enough for mouth animation -const int AVATAR_IDENTITY_PACKET_SEND_INTERVAL_MSECS = 1000; - // See also static AvatarData::defaultFullAvatarModelUrl(). const QString DEFAULT_FULL_AVATAR_MODEL_NAME = QString("Default"); diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp index ba81922979..6dd13c7332 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp +++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp @@ -296,7 +296,8 @@ void EntityTreeRenderer::addPendingEntities(const render::ScenePointer& scene, r } } -void EntityTreeRenderer::updateChangedEntities(const render::ScenePointer& scene, const ViewFrustum& view, render::Transaction& transaction) { +void EntityTreeRenderer::updateChangedEntities(const render::ScenePointer& scene, const ViewFrustums& views, + render::Transaction& transaction) { PROFILE_RANGE_EX(simulation_physics, "ChangeInScene", 0xffff00ff, (uint64_t)_changedEntities.size()); PerformanceTimer pt("change"); std::unordered_set changedEntities; @@ -357,7 +358,7 @@ void EntityTreeRenderer::updateChangedEntities(const render::ScenePointer& scene // prioritize and sort the renderables uint64_t sortStart = usecTimestampNow(); - PrioritySortUtil::PriorityQueue sortedRenderables(view); + PrioritySortUtil::PriorityQueue sortedRenderables(views); { PROFILE_RANGE_EX(simulation_physics, "SortRenderables", 0xffff00ff, (uint64_t)_renderablesToUpdate.size()); std::unordered_map::iterator itr = _renderablesToUpdate.begin(); @@ -415,9 +416,20 @@ void EntityTreeRenderer::update(bool simulate) { if (scene) { render::Transaction transaction; addPendingEntities(scene, transaction); + + ViewFrustums views; + ViewFrustum view; - _viewState->copyCurrentViewFrustum(view); - updateChangedEntities(scene, view, transaction); + _viewState->copyViewFrustum(view); + views.push_back(view); + + if (_viewState->hasSecondaryViewFrustum()) { + _viewState->copySecondaryViewFrustum(view); + views.push_back(view); + } + + + updateChangedEntities(scene, views, transaction); scene->enqueueTransaction(transaction); } } diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.h b/libraries/entities-renderer/src/EntityTreeRenderer.h index f5cedfdd01..9ed4f9d21d 100644 --- a/libraries/entities-renderer/src/EntityTreeRenderer.h +++ b/libraries/entities-renderer/src/EntityTreeRenderer.h @@ -148,7 +148,8 @@ protected: private: void addPendingEntities(const render::ScenePointer& scene, render::Transaction& transaction); - void updateChangedEntities(const render::ScenePointer& scene, const ViewFrustum& view, render::Transaction& transaction); + void updateChangedEntities(const render::ScenePointer& scene, const ViewFrustums& views, + render::Transaction& transaction); EntityRendererPointer renderableForEntity(const EntityItemPointer& entity) const { return renderableForEntityId(entity->getID()); } render::ItemID renderableIdForEntity(const EntityItemPointer& entity) const { return renderableIdForEntityId(entity->getID()); } diff --git a/libraries/entities/src/DiffTraversal.h b/libraries/entities/src/DiffTraversal.h index 50fe74a75b..0fd014ac23 100644 --- a/libraries/entities/src/DiffTraversal.h +++ b/libraries/entities/src/DiffTraversal.h @@ -36,7 +36,7 @@ public: bool isVerySimilar(const View& view) const; ViewFrustum::intersection calculateIntersection(const AACube& cube) const; - std::vector viewFrustums; + ViewFrustums viewFrustums; uint64_t startTime { 0 }; float lodScaleFactor { 1.0f }; }; diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index b16f9c903e..17b0d90cfe 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -90,6 +90,8 @@ PacketVersion versionForPacketType(PacketType packetType) { return 18; // replace min_avatar_scale and max_avatar_scale with min_avatar_height and max_avatar_height case PacketType::Ping: return static_cast(PingVersion::IncludeConnectionID); + case PacketType::ViewFrustum: + return static_cast(ViewFrustumVersion::SendMultipleFrustums); default: return 20; } diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 9b48e3a132..c72bbb0129 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -328,4 +328,8 @@ enum class PingVersion : PacketVersion { IncludeConnectionID = 18 }; +enum class ViewFrustumVersion : PacketVersion { + SendMultipleFrustums = 21 +}; + #endif // hifi_PacketHeaders_h diff --git a/libraries/render-utils/src/AbstractViewStateInterface.h b/libraries/render-utils/src/AbstractViewStateInterface.h index 54fdc903ca..9d781b7d18 100644 --- a/libraries/render-utils/src/AbstractViewStateInterface.h +++ b/libraries/render-utils/src/AbstractViewStateInterface.h @@ -31,6 +31,10 @@ public: /// copies the current view frustum for rendering the view state virtual void copyCurrentViewFrustum(ViewFrustum& viewOut) const = 0; + virtual void copyViewFrustum(ViewFrustum& viewOut) const = 0; + virtual void copySecondaryViewFrustum(ViewFrustum& viewOut) const = 0; + virtual bool hasSecondaryViewFrustum() const = 0; + virtual QThread* getMainThread() = 0; virtual PickRay computePickRay(float x, float y) const = 0; diff --git a/libraries/shared/src/PrioritySortUtil.h b/libraries/shared/src/PrioritySortUtil.h index 279fa42ea4..ba15222611 100644 --- a/libraries/shared/src/PrioritySortUtil.h +++ b/libraries/shared/src/PrioritySortUtil.h @@ -84,14 +84,12 @@ namespace PrioritySortUtil { class PriorityQueue { public: PriorityQueue() = delete; - - PriorityQueue(const ViewFrustum& view) : _view(view) { } - - PriorityQueue(const ViewFrustum& view, float angularWeight, float centerWeight, float ageWeight) - : _view(view), _angularWeight(angularWeight), _centerWeight(centerWeight), _ageWeight(ageWeight) + PriorityQueue(const ViewFrustums& views) : _views(views) { } + PriorityQueue(const ViewFrustums& views, float angularWeight, float centerWeight, float ageWeight) + : _views(views), _angularWeight(angularWeight), _centerWeight(centerWeight), _ageWeight(ageWeight) { } - void setView(const ViewFrustum& view) { _view = view; } + void setViews(const ViewFrustums& views) { _views = views; } void setWeights(float angularWeight, float centerWeight, float ageWeight) { _angularWeight = angularWeight; @@ -109,7 +107,18 @@ namespace PrioritySortUtil { bool empty() const { return _queue.empty(); } private: + float computePriority(const T& thing) const { + float priority = std::numeric_limits::min(); + + for (const auto& view : _views) { + priority = std::max(priority, computePriority(view, thing)); + } + + return priority; + } + + float computePriority(const ViewFrustum& view, const T& thing) const { // priority = weighted linear combination of multiple values: // (a) angular size // (b) proximity to center of view @@ -117,11 +126,11 @@ namespace PrioritySortUtil { // where the relative "weights" are tuned to scale the contributing values into units of "priority". glm::vec3 position = thing.getPosition(); - glm::vec3 offset = position - _view.getPosition(); + glm::vec3 offset = position - view.getPosition(); float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero const float MIN_RADIUS = 0.1f; // WORKAROUND for zero size objects (we still want them to sort by distance) float radius = glm::min(thing.getRadius(), MIN_RADIUS); - float cosineAngle = (glm::dot(offset, _view.getDirection()) / distance); + float cosineAngle = (glm::dot(offset, view.getDirection()) / distance); float age = (float)(usecTimestampNow() - thing.getTimestamp()); // we modulatate "age" drift rate by the cosineAngle term to make periphrial objects sort forward @@ -134,8 +143,8 @@ namespace PrioritySortUtil { + _ageWeight * cosineAngleFactor * age; // decrement priority of things outside keyhole - if (distance - radius > _view.getCenterRadius()) { - if (!_view.sphereIntersectsFrustum(position, radius)) { + if (distance - radius > view.getCenterRadius()) { + if (!view.sphereIntersectsFrustum(position, radius)) { constexpr float OUT_OF_VIEW_PENALTY = -10.0f; priority += OUT_OF_VIEW_PENALTY; } @@ -143,7 +152,7 @@ namespace PrioritySortUtil { return priority; } - ViewFrustum _view; + ViewFrustums _views; std::priority_queue _queue; float _angularWeight { DEFAULT_ANGULAR_COEF }; float _centerWeight { DEFAULT_CENTER_COEF }; diff --git a/libraries/shared/src/ViewFrustum.h b/libraries/shared/src/ViewFrustum.h index ba8957bba3..128a717df8 100644 --- a/libraries/shared/src/ViewFrustum.h +++ b/libraries/shared/src/ViewFrustum.h @@ -189,5 +189,6 @@ private: }; using ViewFrustumPointer = std::shared_ptr; +using ViewFrustums = std::vector; #endif // hifi_ViewFrustum_h diff --git a/tests/render-perf/src/main.cpp b/tests/render-perf/src/main.cpp index 9249b3d957..8c17d7c5c2 100644 --- a/tests/render-perf/src/main.cpp +++ b/tests/render-perf/src/main.cpp @@ -441,6 +441,16 @@ protected: viewOut = _viewFrustum; } + void copyViewFrustum(ViewFrustum& viewOut) const override { + viewOut = _viewFrustum; + } + + void copySecondaryViewFrustum(ViewFrustum& viewOut) const override {} + + bool hasSecondaryViewFrustum() const override { + return false; + } + QThread* getMainThread() override { return QThread::currentThread(); }