diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index a4bf8fa253..288652715a 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -25,6 +25,23 @@ AvatarMixerClientData::AvatarMixerClientData(const QUuid& nodeID) : _avatar->setID(nodeID); } +uint64_t AvatarMixerClientData::getLastOtherAvatarEncodeTime(QUuid otherAvatar) const { + std::unordered_map::const_iterator itr = _lastOtherAvatarEncodeTime.find(otherAvatar); + if (itr != _lastOtherAvatarEncodeTime.end()) { + return itr->second; + } + return 0; +} + +void AvatarMixerClientData::setLastOtherAvatarEncodeTime(const QUuid& otherAvatar, const uint64_t& time) { + std::unordered_map::iterator itr = _lastOtherAvatarEncodeTime.find(otherAvatar); + if (itr != _lastOtherAvatarEncodeTime.end()) { + itr->second = time; + } else { + _lastOtherAvatarEncodeTime.emplace(std::pair(otherAvatar, time)); + } +} + void AvatarMixerClientData::queuePacket(QSharedPointer message, SharedNodePointer node) { if (!_packetQueue.node) { _packetQueue.node = node; diff --git a/assignment-client/src/avatars/AvatarMixerClientData.h b/assignment-client/src/avatars/AvatarMixerClientData.h index d5c7784da7..acd9be0702 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.h +++ b/assignment-client/src/avatars/AvatarMixerClientData.h @@ -110,16 +110,10 @@ public: bool getRequestsDomainListData() { return _requestsDomainListData; } void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; } - ViewFrustum getViewFrustom() const { return _currentViewFrustum; } + ViewFrustum getViewFrustum() const { return _currentViewFrustum; } - quint64 getLastOtherAvatarEncodeTime(QUuid otherAvatar) { - quint64 result = 0; - if (_lastOtherAvatarEncodeTime.find(otherAvatar) != _lastOtherAvatarEncodeTime.end()) { - result = _lastOtherAvatarEncodeTime[otherAvatar]; - } - _lastOtherAvatarEncodeTime[otherAvatar] = usecTimestampNow(); - return result; - } + uint64_t getLastOtherAvatarEncodeTime(QUuid otherAvatar) const; + void setLastOtherAvatarEncodeTime(const QUuid& otherAvatar, const uint64_t& time); QVector& getLastOtherAvatarSentJoints(QUuid otherAvatar) { _lastOtherAvatarSentJoints[otherAvatar].resize(_avatar->getJointCount()); @@ -143,7 +137,7 @@ private: // this is a map of the last time we encoded an "other" avatar for // sending to "this" node - std::unordered_map _lastOtherAvatarEncodeTime; + std::unordered_map _lastOtherAvatarEncodeTime; std::unordered_map> _lastOtherAvatarSentJoints; uint64_t _identityChangeTimestamp; diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index dd045c24ea..9ea1ed3637 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -32,6 +33,10 @@ #include "AvatarMixerClientData.h" #include "AvatarMixerSlave.h" +namespace PrioritySortUtil { + // we declare this callback here but override it later + std::function getAvatarAgeCallback = [] (const AvatarSharedPointer& avatar) { return 0; }; +} void AvatarMixerSlave::configure(ConstIter begin, ConstIter end) { _begin = begin; @@ -184,10 +189,8 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes - // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes - QList avatarList; + std::vector avatarsToSort; std::unordered_map avatarDataToNodes; - std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { // make sure this is an agent that we have avatar data for before considering it for inclusion if (otherNode->getType() == NodeType::Agent @@ -195,36 +198,61 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); - avatarList << otherAvatar; + avatarsToSort.push_back(otherAvatar); avatarDataToNodes[otherAvatar] = otherNode; } }); - AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); - ViewFrustum cameraView = nodeData->getViewFrustom(); - std::priority_queue sortedAvatars; - AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, - [&](AvatarSharedPointer avatar)->uint64_t { - auto avatarNode = avatarDataToNodes[avatar]; - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - return nodeData->getLastBroadcastTime(avatarNode->getUUID()); - }, [&](AvatarSharedPointer avatar)->float{ - glm::vec3 nodeBoxHalfScale = (avatar->getWorldPosition() - avatar->getGlobalBoundingBoxCorner() * avatar->getSensorToWorldScale()); - return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); - }, [&](AvatarSharedPointer avatar)->bool { + // now that we've assembled the avatarDataToNodes map we can replace PrioritySortUtil::getAvatarAgeCallback + // with the true implementation + PrioritySortUtil::getAvatarAgeCallback = [&] (const AvatarSharedPointer& avatar) { + auto avatarNode = avatarDataToNodes[avatar]; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + return nodeData->getLastOtherAvatarEncodeTime(avatarNode->getUUID()); + }; + + class SortableAvatar: public PrioritySortUtil::Sortable { + public: + SortableAvatar() = delete; + SortableAvatar(const AvatarSharedPointer& avatar) : _avatar(avatar) {} + glm::vec3 getPosition() const override { return _avatar->getWorldPosition(); } + float getRadius() const override { + glm::vec3 nodeBoxHalfScale = (_avatar->getWorldPosition() - _avatar->getGlobalBoundingBoxCorner() * _avatar->getSensorToWorldScale()); + return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); + } + uint64_t getTimestamp() const override { + // use the callback implemented above + return PrioritySortUtil::getAvatarAgeCallback(_avatar); + } + const AvatarSharedPointer& getAvatar() const { return _avatar; } + + private: + AvatarSharedPointer _avatar; + }; + + // prepare to sort + ViewFrustum cameraView = nodeData->getViewFrustum(); + PrioritySortUtil::PriorityQueue sortedAvatars(cameraView, + AvatarData::_avatarSortCoefficientSize, + AvatarData::_avatarSortCoefficientCenter, + AvatarData::_avatarSortCoefficientAge); + + // ignore or sort + const AvatarSharedPointer& thisAvatar = nodeData->getAvatarSharedPointer(); + for (const auto& avatar : avatarsToSort) { if (avatar == thisAvatar) { - return true; // ignore ourselves... + // don't echo updates to self + continue; } bool shouldIgnore = false; - - // We will also ignore other nodes for a couple of different reasons: + // We ignore other nodes for a couple of reasons: // 1) ignore bubbles and ignore specific node // 2) the node hasn't really updated it's frame data recently, this can // happen if for example the avatar is connected on a desktop and sending // updates at ~30hz. So every 3 frames we skip a frame. - auto avatarNode = avatarDataToNodes[avatar]; + auto avatarNode = avatarDataToNodes[avatar]; assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); @@ -240,7 +268,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { shouldIgnore = true; } else { - // Check to see if the space bubble is enabled // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { @@ -267,8 +294,6 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); } } - quint64 endIgnoreCalculation = usecTimestampNow(); - _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); if (!shouldIgnore) { AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); @@ -292,20 +317,21 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) ++numAvatarsWithSkippedFrames; } } - return shouldIgnore; - }); + quint64 endIgnoreCalculation = usecTimestampNow(); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + + if (!shouldIgnore) { + // sort this one for later + sortedAvatars.push(SortableAvatar(avatar)); + } + } // loop through our sorted avatars and allocate our bandwidth to them accordingly - int avatarRank = 0; - // this is overly conservative, because it includes some avatars we might not consider int remainingAvatars = (int)sortedAvatars.size(); - while (!sortedAvatars.empty()) { - AvatarPriority sortData = sortedAvatars.top(); + const auto& avatarData = sortedAvatars.top().getAvatar(); sortedAvatars.pop(); - const auto& avatarData = sortData.avatar; - avatarRank++; remainingAvatars--; auto otherNode = avatarDataToNodes[avatarData]; @@ -332,10 +358,8 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) nodeData->setLastBroadcastTime(otherNode->getUUID(), usecTimestampNow()); } + // determine if avatar is in view which determines how much data to send glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); - - - // determine if avatar is in view, to determine how much data to include... glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f * otherAvatar->getSensorToWorldScale(); AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); bool isInView = nodeData->otherAvatarInView(otherNodeBox); @@ -405,14 +429,18 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) // set the last sent sequence number for this sender on the receiver nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), otherNodeData->getLastReceivedSequenceNumber()); + nodeData->setLastOtherAvatarEncodeTime(otherNode->getUUID(), usecTimestampNow()); } + } else { + // TODO? this avatar is not included now, and will probably not be included next frame. + // It would be nice if we could tweak its future sort priority to put it at the back of the list. } avatarPacketList->endSegment(); quint64 endAvatarDataPacking = usecTimestampNow(); _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - }; + } quint64 startPacketSending = usecTimestampNow(); diff --git a/interface/src/avatar/AvatarManager.cpp b/interface/src/avatar/AvatarManager.cpp index 9ffe74d470..8a294182bd 100644 --- a/interface/src/avatar/AvatarManager.cpp +++ b/interface/src/avatar/AvatarManager.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -142,32 +143,39 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { PerformanceTimer perfTimer("otherAvatars"); - auto avatarMap = getHashCopy(); - QList avatarList = avatarMap.values(); + class SortableAvatar: public PrioritySortUtil::Sortable { + public: + SortableAvatar() = delete; + SortableAvatar(const AvatarSharedPointer& avatar) : _avatar(avatar) {} + glm::vec3 getPosition() const override { return _avatar->getWorldPosition(); } + float getRadius() const override { return std::static_pointer_cast(_avatar)->getBoundingRadius(); } + uint64_t getTimestamp() const override { return std::static_pointer_cast(_avatar)->getLastRenderUpdateTime(); } + const AvatarSharedPointer& getAvatar() const { return _avatar; } + private: + AvatarSharedPointer _avatar; + }; + ViewFrustum cameraView; qApp->copyDisplayViewFrustum(cameraView); + PrioritySortUtil::PriorityQueue sortedAvatars(cameraView, + AvatarData::_avatarSortCoefficientSize, + AvatarData::_avatarSortCoefficientCenter, + AvatarData::_avatarSortCoefficientAge); - std::priority_queue sortedAvatars; - AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, - - [](AvatarSharedPointer avatar)->uint64_t{ - return std::static_pointer_cast(avatar)->getLastRenderUpdateTime(); - }, - - [](AvatarSharedPointer avatar)->float{ - return std::static_pointer_cast(avatar)->getBoundingRadius(); - }, - - [this](AvatarSharedPointer avatar)->bool{ - const auto& castedAvatar = std::static_pointer_cast(avatar); - if (castedAvatar == _myAvatar || !castedAvatar->isInitialized()) { - // DO NOT update _myAvatar! Its update has already been done earlier in the main loop. - // DO NOT update or fade out uninitialized Avatars - return true; // ignore it - } - return false; - }); + // sort + auto avatarMap = getHashCopy(); + AvatarHash::iterator itr = avatarMap.begin(); + while (itr != avatarMap.end()) { + const auto& avatar = std::static_pointer_cast(*itr); + // DO NOT update _myAvatar! Its update has already been done earlier in the main loop. + // DO NOT update or fade out uninitialized Avatars + if (avatar != _myAvatar && avatar->isInitialized()) { + sortedAvatars.push(SortableAvatar(avatar)); + } + ++itr; + } + // process in sorted order uint64_t startTime = usecTimestampNow(); const uint64_t UPDATE_BUDGET = 2000; // usec uint64_t updateExpiry = startTime + UPDATE_BUDGET; @@ -176,8 +184,8 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { render::Transaction transaction; while (!sortedAvatars.empty()) { - const AvatarPriority& sortData = sortedAvatars.top(); - const auto& avatar = std::static_pointer_cast(sortData.avatar); + const SortableAvatar& sortData = sortedAvatars.top(); + const auto& avatar = std::static_pointer_cast(sortData.getAvatar()); bool ignoring = DependencyManager::get()->isPersonalMutingNode(avatar->getID()); if (ignoring) { @@ -207,7 +215,7 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { uint64_t now = usecTimestampNow(); if (now < updateExpiry) { // we're within budget - bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD; + bool inView = sortData.getPriority() > OUT_OF_VIEW_THRESHOLD; if (inView && avatar->hasNewJointData()) { numAvatarsUpdated++; } @@ -221,7 +229,7 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { // --> some avatar velocity measurements may be a little off // no time simulate, but we take the time to count how many were tragically missed - bool inView = sortData.priority > OUT_OF_VIEW_THRESHOLD; + bool inView = sortData.getPriority() > OUT_OF_VIEW_THRESHOLD; if (!inView) { break; } @@ -230,9 +238,9 @@ void AvatarManager::updateOtherAvatars(float deltaTime) { } sortedAvatars.pop(); while (inView && !sortedAvatars.empty()) { - const AvatarPriority& newSortData = sortedAvatars.top(); - const auto& newAvatar = std::static_pointer_cast(newSortData.avatar); - inView = newSortData.priority > OUT_OF_VIEW_THRESHOLD; + const SortableAvatar& newSortData = sortedAvatars.top(); + const auto& newAvatar = std::static_pointer_cast(newSortData.getAvatar()); + inView = newSortData.getPriority() > OUT_OF_VIEW_THRESHOLD; if (inView && newAvatar->hasNewJointData()) { numAVatarsNotUpdated++; } diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index cb43aeee5d..f68dce8d26 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -2387,63 +2387,10 @@ void RayToAvatarIntersectionResultFromScriptValue(const QScriptValue& object, Ra const float AvatarData::OUT_OF_VIEW_PENALTY = -10.0f; -float AvatarData::_avatarSortCoefficientSize { 0.5f }; +float AvatarData::_avatarSortCoefficientSize { 1.0f }; float AvatarData::_avatarSortCoefficientCenter { 0.25 }; float AvatarData::_avatarSortCoefficientAge { 1.0f }; -void AvatarData::sortAvatars( - QList avatarList, - const ViewFrustum& cameraView, - std::priority_queue& sortedAvatarsOut, - std::function getLastUpdated, - std::function getBoundingRadius, - std::function shouldIgnore) { - - PROFILE_RANGE(simulation, "sort"); - uint64_t now = usecTimestampNow(); - - glm::vec3 frustumCenter = cameraView.getPosition(); - const glm::vec3& forward = cameraView.getDirection(); - for (int32_t i = 0; i < avatarList.size(); ++i) { - const auto& avatar = avatarList.at(i); - - if (shouldIgnore(avatar)) { - continue; - } - - // priority = weighted linear combination of: - // (a) apparentSize - // (b) proximity to center of view - // (c) time since last update - glm::vec3 avatarPosition = avatar->getWorldPosition(); - glm::vec3 offset = avatarPosition - frustumCenter; - float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero - - // FIXME - AvatarData has something equivolent to this - float radius = getBoundingRadius(avatar); - - float apparentSize = 2.0f * radius / distance; - float cosineAngle = glm::dot(offset, forward) / distance; - float age = (float)(now - getLastUpdated(avatar)) / (float)(USECS_PER_SECOND); - - // NOTE: we are adding values of different units to get a single measure of "priority". - // Thus we multiply each component by a conversion "weight" that scales its units relative to the others. - // These weights are pure magic tuning and should be hard coded in the relation below, - // but are currently exposed for anyone who would like to explore fine tuning: - float priority = _avatarSortCoefficientSize * apparentSize - + _avatarSortCoefficientCenter * cosineAngle - + _avatarSortCoefficientAge * age; - - // decrement priority of avatars outside keyhole - if (distance > cameraView.getCenterRadius()) { - if (!cameraView.sphereIntersectsFrustum(avatarPosition, radius)) { - priority += OUT_OF_VIEW_PENALTY; - } - } - sortedAvatarsOut.push(AvatarPriority(avatar, priority)); - } -} - QScriptValue AvatarEntityMapToScriptValue(QScriptEngine* engine, const AvatarEntityMap& value) { QScriptValue obj = engine->newObject(); for (auto entityID : value.keys()) { diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 2f3154ad08..1df0e94496 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -629,14 +629,6 @@ public: static const float OUT_OF_VIEW_PENALTY; - static void sortAvatars( - QList avatarList, - const ViewFrustum& cameraView, - std::priority_queue& sortedAvatarsOut, - std::function getLastUpdated, - std::function getBoundingRadius, - std::function shouldIgnore); - // TODO: remove this HACK once we settle on optimal sort coefficients // These coefficients exposed for fine tuning the sort priority for transfering new _jointData to the render pipeline. static float _avatarSortCoefficientSize; diff --git a/libraries/shared/src/PrioritySortUtil.h b/libraries/shared/src/PrioritySortUtil.h index 1d11a04265..dc6a877bb9 100644 --- a/libraries/shared/src/PrioritySortUtil.h +++ b/libraries/shared/src/PrioritySortUtil.h @@ -12,6 +12,9 @@ #define hifi_PrioritySortUtil_h #include +#include + +#include "NumericalConstants.h" #include "ViewFrustum.h" /* PrioritySortUtil is a helper for sorting 3D things relative to a ViewFrustum. To use: @@ -32,11 +35,10 @@ (2) Make a PrioritySortUtil::PriorityQueue and add them to the queue: - PrioritySortUtil::Prioritizer prioritizer(viewFrustum); + PrioritySortUtil::PriorityQueue sortedThings(viewFrustum); std::priority_queue< PrioritySortUtil::Sortable > sortedThings; for (thing in things) { - float priority = prioritizer.computePriority(PrioritySortUtil::PrioritizableThing(thing)); - sortedThings.push(PrioritySortUtil::Sortable entry(thing, priority)); + sortedThings.push(SortableWrapper(thing)); } (3) Loop over your priority queue and do timeboxed work: @@ -65,6 +67,7 @@ namespace PrioritySortUtil { virtual uint64_t getTimestamp() const = 0; void setPriority(float priority) { _priority = priority; } + float getPriority() const { return _priority; } bool operator<(const Sortable& other) const { return _priority < other._priority; } private: float _priority { 0.0f }; @@ -109,11 +112,19 @@ namespace PrioritySortUtil { glm::vec3 position = thing.getPosition(); glm::vec3 offset = position - _view.getPosition(); float distance = glm::length(offset) + 0.001f; // add 1mm to avoid divide by zero - float radius = thing.getRadius(); + const float MIN_RADIUS = 0.1f; // WORKAROUND for zero size objects (we still want them to sort by distance) + float radius = glm::min(thing.getRadius(), MIN_RADIUS); + float cosineAngle = (glm::dot(offset, _view.getDirection()) / distance); + float age = (float)(usecTimestampNow() - thing.getTimestamp()); - float priority = _angularWeight * (radius / distance) - + _centerWeight * (glm::dot(offset, _view.getDirection()) / distance) - + _ageWeight * (float)(usecTimestampNow() - thing.getTimestamp()); + // we modulatate "age" drift rate by the cosineAngle term to make periphrial objects sort forward + // at a reduced rate but we don't want the "age" term to go zero or negative so we clamp it + const float MIN_COSINE_ANGLE_FACTOR = 0.1f; + float cosineAngleFactor = glm::max(cosineAngle, MIN_COSINE_ANGLE_FACTOR); + + float priority = _angularWeight * glm::max(radius, MIN_RADIUS) / distance + + _centerWeight * cosineAngle + + _ageWeight * cosineAngleFactor * age; // decrement priority of things outside keyhole if (distance - radius > _view.getCenterRadius()) {