From 7b10b8a6c3dd4a5b6ea58cfb232e7bbc257562a6 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 6 Jun 2017 19:04:12 -0700 Subject: [PATCH 001/115] Fix SharedNodePointer leak --- libraries/networking/src/LimitedNodeList.cpp | 18 +++++++++++------- libraries/networking/src/Node.h | 2 +- .../networking/src/ReceivedPacketProcessor.h | 4 +++- libraries/shared/src/GenericThread.cpp | 2 ++ 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index cba1e664ab..7981c7d598 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -586,7 +586,7 @@ SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t return matchingNode; } else { // we didn't have this node, so add them - Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, connectionSecret, this); + Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, connectionSecret); if (nodeType == NodeType::AudioMixer) { LimitedNodeList::flagTimeForConnectionStep(LimitedNodeList::AddedAudioMixer); @@ -619,24 +619,28 @@ SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t } // insert the new node and release our read lock - _nodeHash.insert(UUIDNodePair(newNode->getUUID(), newNodePointer)); + _nodeHash.emplace(newNode->getUUID(), newNodePointer); readLocker.unlock(); qCDebug(networking) << "Added" << *newNode; + auto weakPtr = newNodePointer.toWeakRef(); // We don't want the lambdas to hold a strong ref + emit nodeAdded(newNodePointer); if (newNodePointer->getActiveSocket()) { emit nodeActivated(newNodePointer); } else { - connect(newNodePointer.data(), &NetworkPeer::socketActivated, this, [=] { - emit nodeActivated(newNodePointer); - disconnect(newNodePointer.data(), &NetworkPeer::socketActivated, this, 0); + connect(newNodePointer.data(), &NetworkPeer::socketActivated, this, [this, weakPtr] { + auto sharedPtr = weakPtr.lock(); + if (sharedPtr) { + emit nodeActivated(sharedPtr); + disconnect(sharedPtr.data(), &NetworkPeer::socketActivated, this, 0); + } }); } // Signal when a socket changes, so we can start the hole punch over. - auto weakPtr = newNodePointer.toWeakRef(); // We don't want the lambda to hold a strong ref - connect(newNodePointer.data(), &NetworkPeer::socketUpdated, this, [=] { + connect(newNodePointer.data(), &NetworkPeer::socketUpdated, this, [this, weakPtr] { emit nodeSocketUpdated(weakPtr); }); diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index d1bbffd817..55a2bfffed 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -40,7 +40,7 @@ public: Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions, const QUuid& connectionSecret = QUuid(), - QObject* parent = 0); + QObject* parent = nullptr); bool operator==(const Node& otherNode) const { return _uuid == otherNode._uuid; } bool operator!=(const Node& otherNode) const { return !(*this == otherNode); } diff --git a/libraries/networking/src/ReceivedPacketProcessor.h b/libraries/networking/src/ReceivedPacketProcessor.h index dd790a9b3d..4e4a3d1d11 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.h +++ b/libraries/networking/src/ReceivedPacketProcessor.h @@ -20,6 +20,8 @@ class ReceivedPacketProcessor : public GenericThread { Q_OBJECT public: + static const unsigned long MAX_WAIT_TIME { 100 }; + ReceivedPacketProcessor(); /// Add packet from network receive thread to the processing queue. @@ -64,7 +66,7 @@ protected: virtual bool process() override; /// Determines the timeout of the wait when there are no packets to process. Default value means no timeout - virtual unsigned long getMaxWait() const { return ULONG_MAX; } + virtual unsigned long getMaxWait() const { return MAX_WAIT_TIME; } /// Override to do work before the packets processing loop. Default does nothing. virtual void preProcess() { } diff --git a/libraries/shared/src/GenericThread.cpp b/libraries/shared/src/GenericThread.cpp index 00a80a2864..2e126f12c9 100644 --- a/libraries/shared/src/GenericThread.cpp +++ b/libraries/shared/src/GenericThread.cpp @@ -10,6 +10,7 @@ // #include +#include #include "GenericThread.h" @@ -73,6 +74,7 @@ void GenericThread::threadRoutine() { } while (!_stopThread) { + QCoreApplication::processEvents(); // override this function to do whatever your class actually does, return false to exit thread early if (!process()) { From 45bb3237231a92962faa90cd40cb0a4cdd6a8e9a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 00:19:04 -0700 Subject: [PATCH 002/115] experimental support for mirrored audio packets to other mixers --- assignment-client/src/audio/AudioMixer.cpp | 19 +++++++++++ assignment-client/src/audio/AudioMixer.h | 1 + .../src/audio/AudioMixerClientData.cpp | 32 +++++++++++++++++++ .../src/audio/AudioMixerClientData.h | 2 ++ libraries/networking/src/LimitedNodeList.h | 10 ++++++ libraries/networking/src/NLPacket.h | 1 + libraries/networking/src/Node.h | 5 +++ .../networking/src/udt/PacketHeaders.cpp | 4 ++- libraries/networking/src/udt/PacketHeaders.h | 7 +++- 9 files changed, 79 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index bb03a6ec93..87f51ea73c 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -92,6 +92,14 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : packetReceiver.registerListener(PacketType::NodeMuteRequest, this, "handleNodeMuteRequestPacket"); packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); + packetReceiver.registerListenerForTypes({ + PacketType::MirroredMicrophoneAudioNoEcho, + PacketType::MirroredMicrophoneAudioWithEcho, + PacketType::MirroredInjectAudio, + PacketType::MirroredSilentAudioFrame }, + this, "queueMirroredAudioPacket" + ); + connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); } @@ -103,6 +111,17 @@ void AudioMixer::queueAudioPacket(QSharedPointer message, Share getOrCreateClientData(node.data())->queuePacket(message, node); } +void AudioMixer::queueMirroredAudioPacket(QSharedPointer message) { + // make sure we have a mirrored node for the original sender of the packet + auto nodeList = DependencyManager::get(); + + auto node = nodeList->addOrUpdateNode(message->getSourceID(), NodeType::Agent, + message->getSenderSockAddr(), message->getSenderSockAddr()); + node->setIsMirror(true); + + getOrCreateClientData(node.data())->queuePacket(message, node); +} + void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer message, SharedNodePointer sendingNode) { auto nodeList = DependencyManager::get(); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index 18e754016e..b7eebe6aa1 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -63,6 +63,7 @@ private slots: void handleKillAvatarPacket(QSharedPointer packet, SharedNodePointer sendingNode); void queueAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); + void queueMirroredAudioPacket(QSharedPointer packet); void removeHRTFsForFinishedInjector(const QUuid& streamID); void start(); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index d5e06504a6..135047f3a5 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -71,6 +71,9 @@ void AudioMixerClientData::processPackets() { case PacketType::SilentAudioFrame: { QMutexLocker lock(&getMutex()); parseData(*packet); + + potentiallyMirrorPacket(*packet); + break; } case PacketType::NegotiateAudioFormat: @@ -97,6 +100,35 @@ void AudioMixerClientData::processPackets() { assert(_packetQueue.empty()); } +void AudioMixerClientData::potentiallyMirrorPacket(ReceivedMessage& message) { + auto nodeList = DependencyManager::get(); + if (!nodeList->getMirrorSocket().isNull()) { + PacketType mirroredType; + + if (message.getType() == PacketType::MicrophoneAudioNoEcho) { + mirroredType = PacketType::MirroredMicrophoneAudioNoEcho; + } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { + mirroredType = PacketType::MirroredMicrophoneAudioNoEcho; + } else if (message.getType() == PacketType::InjectAudio) { + mirroredType = PacketType::MirroredInjectAudio; + } else if (message.getType() == PacketType::SilentAudioFrame) { + mirroredType = PacketType::MirroredSilentAudioFrame; + } else { + return; + } + + // construct an NLPacket to send to the mirror that has the contents of the received packet + std::unique_ptr messageData { new char[message.getSize()] }; + memcpy(messageData.get(), message.getMessage().data(), message.getSize()); + auto packet = NLPacket::fromReceivedPacket(std::move(messageData), message.getSize(), + message.getSenderSockAddr()); + + packet->setType(mirroredType); + + nodeList->sendPacket(std::move(packet), nodeList->getMirrorSocket()); + } +} + void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) { quint8 numberOfCodecs; message.readPrimitive(&numberOfCodecs); diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 8d76cda2f1..4207362ed3 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -124,6 +124,8 @@ private: QReadWriteLock _streamsLock; AudioStreamMap _audioStreams; // microphone stream from avatar is stored under key of null UUID + void potentiallyMirrorPacket(ReceivedMessage& packet); + using IgnoreZone = AABox; class IgnoreZoneMemo { public: diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 554386f786..16d7f20f99 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -278,6 +278,12 @@ public: void sendFakedHandshakeRequestToNode(SharedNodePointer node); #endif + void setMirrorSocket(const HifiSockAddr& mirrorSocket) { _mirrorSocket = mirrorSocket; } + const HifiSockAddr& getMirrorSocket() { return _mirrorSocket; } + + void setMasterSocket(const HifiSockAddr& masterSocket) { _masterSocket = masterSocket; } + const HifiSockAddr& getMasterSocket() { return _masterSocket; } + public slots: void reset(); void eraseAllNodes(); @@ -386,6 +392,10 @@ protected: } } + + HifiSockAddr _mirrorSocket; + HifiSockAddr _masterSocket; + private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); void possiblyTimeoutSTUNAddressLookup(); diff --git a/libraries/networking/src/NLPacket.h b/libraries/networking/src/NLPacket.h index 33de262dfb..f9056bbfaa 100644 --- a/libraries/networking/src/NLPacket.h +++ b/libraries/networking/src/NLPacket.h @@ -42,6 +42,7 @@ public: static std::unique_ptr fromReceivedPacket(std::unique_ptr data, qint64 size, const HifiSockAddr& senderSockAddr); + static std::unique_ptr fromBase(std::unique_ptr packet); // Provided for convenience, try to limit use diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 1092fcc7fa..dd05961678 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -80,6 +80,9 @@ public: bool isIgnoreRadiusEnabled() const { return _ignoreRadiusEnabled; } + bool isMirror() const { return _isMirror; } + void setIsMirror(bool isMirror) { _isMirror = isMirror; } + private: // privatize copy and assignment operator to disallow Node copying Node(const Node &otherNode); @@ -98,6 +101,8 @@ private: mutable QReadWriteLock _ignoredNodeIDSetLock; std::atomic_bool _ignoreRadiusEnabled; + + bool _isMirror { false }; }; Q_DECLARE_METATYPE(Node*) diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 9d970fa318..e29bfd0d78 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -39,7 +39,9 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ICEServerPeerInformation << PacketType::ICEServerQuery << PacketType::ICEServerHeartbeat << PacketType::ICEServerHeartbeatACK << PacketType::ICEPing << PacketType::ICEPingReply << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode - << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement; + << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement + << PacketType::MirroredMicrophoneAudioNoEcho << PacketType::MirroredMicrophoneAudioWithEcho + << PacketType::MirroredInjectAudio << PacketType::MirroredSilentAudioFrame; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 2cc3a2c42e..f2a13dc449 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -115,7 +115,12 @@ public: AdjustAvatarSorting, OctreeFileReplacement, CollisionEventChanges, - LAST_PACKET_TYPE = CollisionEventChanges + MirroredMicrophoneAudioNoEcho, + MirroredMicrophoneAudioWithEcho, + MirroredInjectAudio, + MirroredSilentAudioFrame, + LAST_PACKET_TYPE = MirroredSilentAudioFrame, + }; }; From f726c5bc3a466971b2f2fba20488bd53e500dd48 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 01:16:51 -0700 Subject: [PATCH 003/115] handle mirrored packets in AudioMixerClientData --- assignment-client/src/audio/AudioMixer.cpp | 8 ++++++-- .../src/audio/AudioMixerClientData.cpp | 17 +++++++++-------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 87f51ea73c..9b095d3aa7 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -55,7 +55,8 @@ QVector AudioMixer::_zoneSettings; QVector AudioMixer::_zoneReverbSettings; AudioMixer::AudioMixer(ReceivedMessage& message) : - ThreadedAssignment(message) { + ThreadedAssignment(message) +{ // Always clear settings first // This prevents previous assignment settings from sticking around @@ -101,6 +102,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : ); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); + } void AudioMixer::queueAudioPacket(QSharedPointer message, SharedNodePointer node) { @@ -115,7 +117,9 @@ void AudioMixer::queueMirroredAudioPacket(QSharedPointer messag // make sure we have a mirrored node for the original sender of the packet auto nodeList = DependencyManager::get(); - auto node = nodeList->addOrUpdateNode(message->getSourceID(), NodeType::Agent, + QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + auto node = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, message->getSenderSockAddr(), message->getSenderSockAddr()); node->setIsMirror(true); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 135047f3a5..7c747a3417 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -118,12 +118,9 @@ void AudioMixerClientData::potentiallyMirrorPacket(ReceivedMessage& message) { } // construct an NLPacket to send to the mirror that has the contents of the received packet - std::unique_ptr messageData { new char[message.getSize()] }; - memcpy(messageData.get(), message.getMessage().data(), message.getSize()); - auto packet = NLPacket::fromReceivedPacket(std::move(messageData), message.getSize(), - message.getSenderSockAddr()); - - packet->setType(mirroredType); + auto packet = NLPacket::create(mirroredType, message.getSize() + NUM_BYTES_RFC4122_UUID); + packet->write(message.getSourceID().toRfc4122()); + packet->write(message.getMessage()); nodeList->sendPacket(std::move(packet), nodeList->getMirrorSocket()); } @@ -220,8 +217,11 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { bool isMicStream = false; if (packetType == PacketType::MicrophoneAudioWithEcho + || packetType == PacketType::MirroredMicrophoneAudioWithEcho || packetType == PacketType::MicrophoneAudioNoEcho - || packetType == PacketType::SilentAudioFrame) { + || packetType == PacketType::MirroredMicrophoneAudioNoEcho + || packetType == PacketType::SilentAudioFrame + || packetType == PacketType::MirroredSilentAudioFrame) { QWriteLocker writeLocker { &_streamsLock }; @@ -256,7 +256,8 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { writeLocker.unlock(); isMicStream = true; - } else if (packetType == PacketType::InjectAudio) { + } else if (packetType == PacketType::InjectAudio + || packetType == PacketType::MirroredInjectAudio) { // this is injected audio // grab the stream identifier for this injected audio message.seek(sizeof(quint16)); From 281793c4c0e8c08e4916dbad4d53a3875be1b629 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 01:46:12 -0700 Subject: [PATCH 004/115] update last heard timestamp for mirrored nodes --- assignment-client/src/audio/AudioMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 9b095d3aa7..55335c1017 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -102,7 +102,6 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : ); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); - } void AudioMixer::queueAudioPacket(QSharedPointer message, SharedNodePointer node) { @@ -122,6 +121,7 @@ void AudioMixer::queueMirroredAudioPacket(QSharedPointer messag auto node = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, message->getSenderSockAddr(), message->getSenderSockAddr()); node->setIsMirror(true); + node->setLastHeardMicrostamp(usecTimestampNow()); getOrCreateClientData(node.data())->queuePacket(message, node); } From 0ee0082052a8f87d1a124b03f5a61740fe639356 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 02:05:40 -0700 Subject: [PATCH 005/115] handle new packet types in AudioMixerClientData::processPackets --- assignment-client/src/audio/AudioMixerClientData.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 7c747a3417..616d6252d8 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -68,7 +68,11 @@ void AudioMixerClientData::processPackets() { case PacketType::MicrophoneAudioWithEcho: case PacketType::InjectAudio: case PacketType::AudioStreamStats: - case PacketType::SilentAudioFrame: { + case PacketType::SilentAudioFrame: + case PacketType::MirroredMicrophoneAudioNoEcho: + case PacketType::MirroredMicrophoneAudioWithEcho: + case PacketType::MirroredInjectAudio: + case PacketType::MirroredSilentAudioFrame: { QMutexLocker lock(&getMutex()); parseData(*packet); From e629881665ff614ca4503e8bbfd9c849e74b0a0a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 02:18:27 -0700 Subject: [PATCH 006/115] mirror audio format negotiation --- assignment-client/src/audio/AudioMixerClientData.cpp | 4 ++++ libraries/networking/src/udt/PacketHeaders.h | 1 + 2 files changed, 5 insertions(+) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 616d6252d8..ec235600e2 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -81,7 +81,9 @@ void AudioMixerClientData::processPackets() { break; } case PacketType::NegotiateAudioFormat: + case PacketType::MirroredNegotiateAudioFormat: negotiateAudioFormat(*packet, node); + potentiallyMirrorPacket(*packet); break; case PacketType::RequestsDomainListData: parseRequestsDomainListData(*packet); @@ -117,6 +119,8 @@ void AudioMixerClientData::potentiallyMirrorPacket(ReceivedMessage& message) { mirroredType = PacketType::MirroredInjectAudio; } else if (message.getType() == PacketType::SilentAudioFrame) { mirroredType = PacketType::MirroredSilentAudioFrame; + } else if (message.getType() == PacketType::NegotiateAudioFormat) { + mirroredType = PacketType::MirroredNegotiateAudioFormat; } else { return; } diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index f2a13dc449..52b0662c2b 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -115,6 +115,7 @@ public: AdjustAvatarSorting, OctreeFileReplacement, CollisionEventChanges, + MirroredNegotiateAudioFormat, MirroredMicrophoneAudioNoEcho, MirroredMicrophoneAudioWithEcho, MirroredInjectAudio, From b5c26b596e05c5001d7697667d48debd1b9e6f56 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 02:39:54 -0700 Subject: [PATCH 007/115] seek past source UUID in mirrored packets --- assignment-client/src/audio/AudioMixerClientData.cpp | 7 +++++++ libraries/networking/src/udt/PacketHeaders.cpp | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index ec235600e2..b876db55b8 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -303,6 +303,13 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { // seek to the beginning of the packet so that the next reader is in the right spot message.seek(0); + if (packetType == PacketType::MirroredMicrophoneAudioWithEcho + || packetType == PacketType::MirroredMicrophoneAudioNoEcho + || packetType == PacketType::MirroredSilentAudioFrame + || packetType == PacketType::MirroredInjectAudio) { + message.seek(NUM_BYTES_RFC4122_UUID); + } + // check the overflow count before we parse data auto overflowBefore = matchingStream->getOverflowCount(); auto parseResult = matchingStream->parseData(message); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index e29bfd0d78..cdc78ee639 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -41,7 +41,8 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement << PacketType::MirroredMicrophoneAudioNoEcho << PacketType::MirroredMicrophoneAudioWithEcho - << PacketType::MirroredInjectAudio << PacketType::MirroredSilentAudioFrame; + << PacketType::MirroredInjectAudio << PacketType::MirroredSilentAudioFrame + << PacketType::MirroredNegotiateAudioFormat; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { From a085a55107e14c9dece091a7811e2bffe4e3a627 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 02:41:15 -0700 Subject: [PATCH 008/115] register a listener for audio format negotiation --- assignment-client/src/audio/AudioMixer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 55335c1017..0dd52584e0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -97,7 +97,9 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : PacketType::MirroredMicrophoneAudioNoEcho, PacketType::MirroredMicrophoneAudioWithEcho, PacketType::MirroredInjectAudio, - PacketType::MirroredSilentAudioFrame }, + PacketType::MirroredSilentAudioFrame, + PacketType::MirroredNegotiateAudioFormat + }, this, "queueMirroredAudioPacket" ); From 0b847488d7529c6b4e6994bcce88e55474f3e299 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 10:54:22 -0700 Subject: [PATCH 009/115] Fix comment --- libraries/networking/src/ReceivedPacketProcessor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/ReceivedPacketProcessor.h b/libraries/networking/src/ReceivedPacketProcessor.h index 4e4a3d1d11..5b54d4f309 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.h +++ b/libraries/networking/src/ReceivedPacketProcessor.h @@ -65,7 +65,7 @@ protected: /// Implements generic processing behavior for this thread. virtual bool process() override; - /// Determines the timeout of the wait when there are no packets to process. Default value means no timeout + /// Determines the timeout of the wait when there are no packets to process. Default value is 100ms to allow for regular event processing. virtual unsigned long getMaxWait() const { return MAX_WAIT_TIME; } /// Override to do work before the packets processing loop. Default does nothing. From 8ebf1db9a88e764ed77a8a9588fe794230076a18 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 10:58:34 -0700 Subject: [PATCH 010/115] switch to replicator/replicant and add new node types --- assignment-client/src/audio/AudioMixer.cpp | 14 +++--- assignment-client/src/audio/AudioMixer.h | 2 +- .../src/audio/AudioMixerClientData.cpp | 44 +++++++++---------- .../src/audio/AudioMixerClientData.h | 2 +- libraries/networking/src/LimitedNodeList.h | 4 -- libraries/networking/src/NodeType.h | 2 + .../networking/src/udt/PacketHeaders.cpp | 6 +-- libraries/networking/src/udt/PacketHeaders.h | 12 ++--- 8 files changed, 42 insertions(+), 44 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 0dd52584e0..8f8a6ff2c0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -94,11 +94,11 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : packetReceiver.registerListener(PacketType::KillAvatar, this, "handleKillAvatarPacket"); packetReceiver.registerListenerForTypes({ - PacketType::MirroredMicrophoneAudioNoEcho, - PacketType::MirroredMicrophoneAudioWithEcho, - PacketType::MirroredInjectAudio, - PacketType::MirroredSilentAudioFrame, - PacketType::MirroredNegotiateAudioFormat + PacketType::ReplicatedMicrophoneAudioNoEcho, + PacketType::ReplicatedMicrophoneAudioWithEcho, + PacketType::ReplicatedInjectAudio, + PacketType::ReplicatedSilentAudioFrame, + PacketType::ReplicatedNegotiateAudioFormat }, this, "queueMirroredAudioPacket" ); @@ -114,8 +114,8 @@ void AudioMixer::queueAudioPacket(QSharedPointer message, Share getOrCreateClientData(node.data())->queuePacket(message, node); } -void AudioMixer::queueMirroredAudioPacket(QSharedPointer message) { - // make sure we have a mirrored node for the original sender of the packet +void AudioMixer::queueReplicatedAudioPacket(QSharedPointer message) { + // make sure we have a replicated node for the original sender of the packet auto nodeList = DependencyManager::get(); QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); diff --git a/assignment-client/src/audio/AudioMixer.h b/assignment-client/src/audio/AudioMixer.h index b7eebe6aa1..e542d82a6a 100644 --- a/assignment-client/src/audio/AudioMixer.h +++ b/assignment-client/src/audio/AudioMixer.h @@ -63,7 +63,7 @@ private slots: void handleKillAvatarPacket(QSharedPointer packet, SharedNodePointer sendingNode); void queueAudioPacket(QSharedPointer packet, SharedNodePointer sendingNode); - void queueMirroredAudioPacket(QSharedPointer packet); + void queueReplicatedAudioPacket(QSharedPointer packet); void removeHRTFsForFinishedInjector(const QUuid& streamID); void start(); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b876db55b8..70c9b615d8 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -69,21 +69,21 @@ void AudioMixerClientData::processPackets() { case PacketType::InjectAudio: case PacketType::AudioStreamStats: case PacketType::SilentAudioFrame: - case PacketType::MirroredMicrophoneAudioNoEcho: - case PacketType::MirroredMicrophoneAudioWithEcho: - case PacketType::MirroredInjectAudio: - case PacketType::MirroredSilentAudioFrame: { + case PacketType::ReplicatedMicrophoneAudioNoEcho: + case PacketType::ReplicatedMicrophoneAudioWithEcho: + case PacketType::ReplicatedInjectAudio: + case PacketType::ReplicatedSilentAudioFrame: { QMutexLocker lock(&getMutex()); parseData(*packet); - potentiallyMirrorPacket(*packet); + replicatePacket(*packet); break; } case PacketType::NegotiateAudioFormat: - case PacketType::MirroredNegotiateAudioFormat: + case PacketType::ReplicatedNegotiateAudioFormat: negotiateAudioFormat(*packet, node); - potentiallyMirrorPacket(*packet); + replicatePacket(*packet); break; case PacketType::RequestsDomainListData: parseRequestsDomainListData(*packet); @@ -106,26 +106,26 @@ void AudioMixerClientData::processPackets() { assert(_packetQueue.empty()); } -void AudioMixerClientData::potentiallyMirrorPacket(ReceivedMessage& message) { +void AudioMixerClientData::replicatePacket(ReceivedMessage& message) { auto nodeList = DependencyManager::get(); if (!nodeList->getMirrorSocket().isNull()) { PacketType mirroredType; if (message.getType() == PacketType::MicrophoneAudioNoEcho) { - mirroredType = PacketType::MirroredMicrophoneAudioNoEcho; + mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { - mirroredType = PacketType::MirroredMicrophoneAudioNoEcho; + mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; } else if (message.getType() == PacketType::InjectAudio) { - mirroredType = PacketType::MirroredInjectAudio; + mirroredType = PacketType::ReplicatedInjectAudio; } else if (message.getType() == PacketType::SilentAudioFrame) { - mirroredType = PacketType::MirroredSilentAudioFrame; + mirroredType = PacketType::ReplicatedSilentAudioFrame; } else if (message.getType() == PacketType::NegotiateAudioFormat) { - mirroredType = PacketType::MirroredNegotiateAudioFormat; + mirroredType = PacketType::ReplicatedNegotiateAudioFormat; } else { return; } - // construct an NLPacket to send to the mirror that has the contents of the received packet + // construct an NLPacket to send to the replicant that has the contents of the received packet auto packet = NLPacket::create(mirroredType, message.getSize() + NUM_BYTES_RFC4122_UUID); packet->write(message.getSourceID().toRfc4122()); packet->write(message.getMessage()); @@ -225,11 +225,11 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { bool isMicStream = false; if (packetType == PacketType::MicrophoneAudioWithEcho - || packetType == PacketType::MirroredMicrophoneAudioWithEcho + || packetType == PacketType::ReplicatedMicrophoneAudioWithEcho || packetType == PacketType::MicrophoneAudioNoEcho - || packetType == PacketType::MirroredMicrophoneAudioNoEcho + || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho || packetType == PacketType::SilentAudioFrame - || packetType == PacketType::MirroredSilentAudioFrame) { + || packetType == PacketType::ReplicatedSilentAudioFrame) { QWriteLocker writeLocker { &_streamsLock }; @@ -265,7 +265,7 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { isMicStream = true; } else if (packetType == PacketType::InjectAudio - || packetType == PacketType::MirroredInjectAudio) { + || packetType == PacketType::ReplicatedInjectAudio) { // this is injected audio // grab the stream identifier for this injected audio message.seek(sizeof(quint16)); @@ -303,10 +303,10 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { // seek to the beginning of the packet so that the next reader is in the right spot message.seek(0); - if (packetType == PacketType::MirroredMicrophoneAudioWithEcho - || packetType == PacketType::MirroredMicrophoneAudioNoEcho - || packetType == PacketType::MirroredSilentAudioFrame - || packetType == PacketType::MirroredInjectAudio) { + if (packetType == PacketType::ReplicatedMicrophoneAudioWithEcho + || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho + || packetType == PacketType::ReplicatedSilentAudioFrame + || packetType == PacketType::ReplicatedInjectAudio) { message.seek(NUM_BYTES_RFC4122_UUID); } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 4207362ed3..5ab55dd48e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -124,7 +124,7 @@ private: QReadWriteLock _streamsLock; AudioStreamMap _audioStreams; // microphone stream from avatar is stored under key of null UUID - void potentiallyMirrorPacket(ReceivedMessage& packet); + void replicatePacket(ReceivedMessage& packet); using IgnoreZone = AABox; class IgnoreZoneMemo { diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 16d7f20f99..72fc652733 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -281,9 +281,6 @@ public: void setMirrorSocket(const HifiSockAddr& mirrorSocket) { _mirrorSocket = mirrorSocket; } const HifiSockAddr& getMirrorSocket() { return _mirrorSocket; } - void setMasterSocket(const HifiSockAddr& masterSocket) { _masterSocket = masterSocket; } - const HifiSockAddr& getMasterSocket() { return _masterSocket; } - public slots: void reset(); void eraseAllNodes(); @@ -394,7 +391,6 @@ protected: HifiSockAddr _mirrorSocket; - HifiSockAddr _masterSocket; private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 5ae7a835b6..7324dcaf14 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -25,6 +25,8 @@ namespace NodeType { const NodeType_t AssetServer = 'A'; const NodeType_t MessagesMixer = 'm'; const NodeType_t EntityScriptServer = 'S'; + const NodeType_t ReplicantAudioMixer = 'a'; + const NodeType_t ReplicantAvatarMixer = 'w'; const NodeType_t Unassigned = 1; void init(); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index cdc78ee639..ebc8e80e45 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -40,9 +40,9 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ICEServerHeartbeatACK << PacketType::ICEPing << PacketType::ICEPingReply << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement - << PacketType::MirroredMicrophoneAudioNoEcho << PacketType::MirroredMicrophoneAudioWithEcho - << PacketType::MirroredInjectAudio << PacketType::MirroredSilentAudioFrame - << PacketType::MirroredNegotiateAudioFormat; + << PacketType::ReplicatedMicrophoneAudioNoEcho << PacketType::ReplicatedMicrophoneAudioWithEcho + << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame + << PacketType::ReplicatedNegotiateAudioFormat; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 52b0662c2b..3ace60d502 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -115,12 +115,12 @@ public: AdjustAvatarSorting, OctreeFileReplacement, CollisionEventChanges, - MirroredNegotiateAudioFormat, - MirroredMicrophoneAudioNoEcho, - MirroredMicrophoneAudioWithEcho, - MirroredInjectAudio, - MirroredSilentAudioFrame, - LAST_PACKET_TYPE = MirroredSilentAudioFrame, + ReplicatedNegotiateAudioFormat, + ReplicatedMicrophoneAudioNoEcho, + ReplicatedMicrophoneAudioWithEcho, + ReplicatedInjectAudio, + ReplicatedSilentAudioFrame, + LAST_PACKET_TYPE = ReplicatedSilentAudioFrame, }; }; From 2ce924377aedbd4e466c9c19b03643f977c200ca Mon Sep 17 00:00:00 2001 From: seefo Date: Mon, 12 Jun 2017 11:56:38 -0700 Subject: [PATCH 011/115] Added initial settings for replication --- .../resources/describe-settings.json | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index a8c6dd84e7..b3ab6f935d 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1318,6 +1318,43 @@ "advanced": true } ] + }, + { + "name": "replication", + "label": "Replication", + "settings": [ + { + "name": "users", + "label": "Replicated Users", + "type": "table", + "can_add_new_rows": true, + "help": "Users that are replicated", + "numbered": false, + "columns": [ + { + "name": "user", + "label": "User", + "can_set": true + } + ] + }, + { + "name": "replicants", + "label": "Replicants", + "type": "table", + "can_add_new_rows": true, + "help": "Nodes or mirrors that are to replicate the domain", + "numbered": false, + "columns": [ + { + "address": "Address", + "port": "Port", + "mixer_type": "Mixer Type", + "can_set": true + } + ] + } + ] } ] } From ff2800e8d587fabc566cde169226ad0d1bc7ca54 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 11:49:48 -0700 Subject: [PATCH 012/115] Add isReplicant to Node --- domain-server/src/DomainServer.cpp | 5 +++++ libraries/networking/src/Node.cpp | 2 ++ libraries/networking/src/Node.h | 4 ++++ 3 files changed, 11 insertions(+) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index d637a20454..c9005c42e4 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2211,6 +2211,11 @@ void DomainServer::refreshStaticAssignmentAndAddToQueue(SharedAssignmentPointer& } void DomainServer::nodeAdded(SharedNodePointer node) { + // TODO Check to see if node is in list of replicant nodes + if (node->getType() == NodeType::Agent) { + node->setIsReplicant(true); + } + // we don't use updateNodeWithData, so add the DomainServerNodeData to the node here node->setLinkedData(std::unique_ptr { new DomainServerNodeData() }); } diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 60227eeaa1..baad621cd6 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -135,6 +135,7 @@ QDataStream& operator<<(QDataStream& out, const Node& node) { out << node._publicSocket; out << node._localSocket; out << node._permissions; + out << node._isReplicant; return out; } @@ -144,6 +145,7 @@ QDataStream& operator>>(QDataStream& in, Node& node) { in >> node._publicSocket; in >> node._localSocket; in >> node._permissions; + in >> node._isReplicant; return in; } diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index f7afbdd864..2c8270071c 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -48,6 +48,9 @@ public: char getType() const { return _type; } void setType(char type); + bool isReplicant() const { return _isReplicant; } + void setIsReplicant(bool isReplicant) { _isReplicant = isReplicant; } + const QUuid& getConnectionSecret() const { return _connectionSecret; } void setConnectionSecret(const QUuid& connectionSecret) { _connectionSecret = connectionSecret; } @@ -89,6 +92,7 @@ private: Node& operator=(Node otherNode); NodeType_t _type; + bool _isReplicant { false }; QUuid _connectionSecret; std::unique_ptr _linkedData; From 2ff065f75164fdbe1d833103622ef93051dda6c7 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 12:11:16 -0700 Subject: [PATCH 013/115] Add isReplicant to node list packet processing --- libraries/networking/src/LimitedNodeList.cpp | 4 +++- libraries/networking/src/LimitedNodeList.h | 2 +- libraries/networking/src/Node.cpp | 5 +++-- libraries/networking/src/Node.h | 4 ++-- libraries/networking/src/NodeList.cpp | 5 +++-- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index f9baff0daf..43b1602483 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -569,6 +569,7 @@ void LimitedNodeList::handleNodeKill(const SharedNodePointer& node) { SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions, + bool isReplicant, const QUuid& connectionSecret) { QReadLocker readLocker(&_nodeMutex); NodeHash::const_iterator it = _nodeHash.find(uuid); @@ -580,11 +581,12 @@ SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t matchingNode->setLocalSocket(localSocket); matchingNode->setPermissions(permissions); matchingNode->setConnectionSecret(connectionSecret); + matchingNode->setIsReplicant(isReplicant); return matchingNode; } else { // we didn't have this node, so add them - Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, connectionSecret); + Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, isReplicant, connectionSecret, this); if (nodeType == NodeType::AudioMixer) { LimitedNodeList::flagTimeForConnectionStep(LimitedNodeList::AddedAudioMixer); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 72fc652733..4bc9134a73 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -146,7 +146,7 @@ public: SharedNodePointer addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions = DEFAULT_AGENT_PERMISSIONS, - const QUuid& connectionSecret = QUuid()); + bool isReplicant = false, const QUuid& connectionSecret = QUuid()); static bool parseSTUNResponse(udt::BasePacket* packet, QHostAddress& newPublicAddress, uint16_t& newPublicPort); bool hasCompletedInitialSTUN() const { return _hasCompletedInitialSTUN; } diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index baad621cd6..8972f68d2a 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -51,11 +51,12 @@ const QString& NodeType::getNodeTypeName(NodeType_t nodeType) { } Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, - const HifiSockAddr& localSocket, const NodePermissions& permissions, const QUuid& connectionSecret, - QObject* parent) : + const HifiSockAddr& localSocket, const NodePermissions& permissions, bool isReplicant, + const QUuid& connectionSecret, QObject* parent) : NetworkPeer(uuid, publicSocket, localSocket, parent), _type(type), _connectionSecret(connectionSecret), + _isReplicant(isReplicant), _pingMs(-1), // "Uninitialized" _clockSkewUsec(0), _mutex(), diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 2c8270071c..8a2129f6f1 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -39,8 +39,8 @@ class Node : public NetworkPeer { public: Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, - const NodePermissions& permissions, const QUuid& connectionSecret = QUuid(), - QObject* parent = nullptr); + const NodePermissions& permissions, bool isReplicant, const QUuid& connectionSecret = QUuid(), + QObject* parent = 0); bool operator==(const Node& otherNode) const { return _uuid == otherNode._uuid; } bool operator!=(const Node& otherNode) const { return !(*this == otherNode); } diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 2aa30b84aa..6710f3ffa3 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -654,8 +654,9 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { QUuid nodeUUID, connectionUUID; HifiSockAddr nodePublicSocket, nodeLocalSocket; NodePermissions permissions; + bool isReplicant; - packetStream >> nodeType >> nodeUUID >> nodePublicSocket >> nodeLocalSocket >> permissions; + packetStream >> nodeType >> nodeUUID >> nodePublicSocket >> nodeLocalSocket >> permissions >> isReplicant; // if the public socket address is 0 then it's reachable at the same IP // as the domain server @@ -666,7 +667,7 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { packetStream >> connectionUUID; SharedNodePointer node = addOrUpdateNode(nodeUUID, nodeType, nodePublicSocket, - nodeLocalSocket, permissions, connectionUUID); + nodeLocalSocket, permissions, isReplicant, connectionUUID); } void NodeList::sendAssignment(Assignment& assignment) { From 84e1a6f89388d797492fea97b8f966156a48a394 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 13:02:54 -0700 Subject: [PATCH 014/115] Rename isReplicant to isReplicated --- domain-server/src/DomainServer.cpp | 4 ++-- libraries/networking/src/LimitedNodeList.cpp | 6 +++--- libraries/networking/src/LimitedNodeList.h | 2 +- libraries/networking/src/Node.cpp | 8 ++++---- libraries/networking/src/Node.h | 8 ++++---- libraries/networking/src/NodeList.cpp | 8 +++++--- 6 files changed, 19 insertions(+), 17 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index c9005c42e4..de53898057 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2211,9 +2211,9 @@ void DomainServer::refreshStaticAssignmentAndAddToQueue(SharedAssignmentPointer& } void DomainServer::nodeAdded(SharedNodePointer node) { - // TODO Check to see if node is in list of replicant nodes + // TODO Check to see if node is in list of replicated nodes if (node->getType() == NodeType::Agent) { - node->setIsReplicant(true); + node->setIsReplicated(true); } // we don't use updateNodeWithData, so add the DomainServerNodeData to the node here diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 43b1602483..7aa17ebf04 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -569,7 +569,7 @@ void LimitedNodeList::handleNodeKill(const SharedNodePointer& node) { SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions, - bool isReplicant, + bool isReplicated, const QUuid& connectionSecret) { QReadLocker readLocker(&_nodeMutex); NodeHash::const_iterator it = _nodeHash.find(uuid); @@ -581,12 +581,12 @@ SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t matchingNode->setLocalSocket(localSocket); matchingNode->setPermissions(permissions); matchingNode->setConnectionSecret(connectionSecret); - matchingNode->setIsReplicant(isReplicant); + matchingNode->setIsReplicated(isReplicated); return matchingNode; } else { // we didn't have this node, so add them - Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, isReplicant, connectionSecret, this); + Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, isReplicated, connectionSecret, this); if (nodeType == NodeType::AudioMixer) { LimitedNodeList::flagTimeForConnectionStep(LimitedNodeList::AddedAudioMixer); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 4bc9134a73..1ace6b2e23 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -146,7 +146,7 @@ public: SharedNodePointer addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions = DEFAULT_AGENT_PERMISSIONS, - bool isReplicant = false, const QUuid& connectionSecret = QUuid()); + bool isReplicated = false, const QUuid& connectionSecret = QUuid()); static bool parseSTUNResponse(udt::BasePacket* packet, QHostAddress& newPublicAddress, uint16_t& newPublicPort); bool hasCompletedInitialSTUN() const { return _hasCompletedInitialSTUN; } diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 8972f68d2a..49f032b823 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -51,12 +51,12 @@ const QString& NodeType::getNodeTypeName(NodeType_t nodeType) { } Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, - const HifiSockAddr& localSocket, const NodePermissions& permissions, bool isReplicant, + const HifiSockAddr& localSocket, const NodePermissions& permissions, bool isReplicated, const QUuid& connectionSecret, QObject* parent) : NetworkPeer(uuid, publicSocket, localSocket, parent), _type(type), _connectionSecret(connectionSecret), - _isReplicant(isReplicant), + _isReplicated(isReplicated), _pingMs(-1), // "Uninitialized" _clockSkewUsec(0), _mutex(), @@ -136,7 +136,7 @@ QDataStream& operator<<(QDataStream& out, const Node& node) { out << node._publicSocket; out << node._localSocket; out << node._permissions; - out << node._isReplicant; + out << node._isReplicated; return out; } @@ -146,7 +146,7 @@ QDataStream& operator>>(QDataStream& in, Node& node) { in >> node._publicSocket; in >> node._localSocket; in >> node._permissions; - in >> node._isReplicant; + in >> node._isReplicated; return in; } diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 8a2129f6f1..73c8538304 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -39,7 +39,7 @@ class Node : public NetworkPeer { public: Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, - const NodePermissions& permissions, bool isReplicant, const QUuid& connectionSecret = QUuid(), + const NodePermissions& permissions, bool isReplicated, const QUuid& connectionSecret = QUuid(), QObject* parent = 0); bool operator==(const Node& otherNode) const { return _uuid == otherNode._uuid; } @@ -48,8 +48,8 @@ public: char getType() const { return _type; } void setType(char type); - bool isReplicant() const { return _isReplicant; } - void setIsReplicant(bool isReplicant) { _isReplicant = isReplicant; } + bool isReplicated() const { return _isReplicated; } + void setIsReplicated(bool isReplicated) { _isReplicated = isReplicated; } const QUuid& getConnectionSecret() const { return _connectionSecret; } void setConnectionSecret(const QUuid& connectionSecret) { _connectionSecret = connectionSecret; } @@ -92,7 +92,7 @@ private: Node& operator=(Node otherNode); NodeType_t _type; - bool _isReplicant { false }; + bool _isReplicated { false }; QUuid _connectionSecret; std::unique_ptr _linkedData; diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 6710f3ffa3..c38a9470b1 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -654,9 +654,11 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { QUuid nodeUUID, connectionUUID; HifiSockAddr nodePublicSocket, nodeLocalSocket; NodePermissions permissions; - bool isReplicant; + bool isReplicated; - packetStream >> nodeType >> nodeUUID >> nodePublicSocket >> nodeLocalSocket >> permissions >> isReplicant; + packetStream >> nodeType >> nodeUUID >> nodePublicSocket >> nodeLocalSocket >> permissions >> isReplicated; + + qDebug() << "Node: " << nodeUUID << nodeType << isReplicated; // if the public socket address is 0 then it's reachable at the same IP // as the domain server @@ -667,7 +669,7 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { packetStream >> connectionUUID; SharedNodePointer node = addOrUpdateNode(nodeUUID, nodeType, nodePublicSocket, - nodeLocalSocket, permissions, isReplicant, connectionUUID); + nodeLocalSocket, permissions, isReplicated, connectionUUID); } void NodeList::sendAssignment(Assignment& assignment) { From 3142842d5d8a60a6863a35349f7ea410fba94e03 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 14:51:08 -0700 Subject: [PATCH 015/115] Fix Node initialization warnings --- libraries/networking/src/Node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 73c8538304..2ae0ab82e9 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -92,7 +92,6 @@ private: Node& operator=(Node otherNode); NodeType_t _type; - bool _isReplicated { false }; QUuid _connectionSecret; std::unique_ptr _linkedData; @@ -101,6 +100,7 @@ private: QMutex _mutex; MovingPercentile _clockSkewMovingPercentile; NodePermissions _permissions; + bool _isReplicated { false }; tbb::concurrent_unordered_set _ignoredNodeIDSet; mutable QReadWriteLock _ignoredNodeIDSetLock; From 4851d3b56409bef465365a552858ea8746992ade Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 14:52:02 -0700 Subject: [PATCH 016/115] Remove superfluous node log --- libraries/networking/src/NodeList.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index c38a9470b1..9b7a7c91dd 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -658,8 +658,6 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { packetStream >> nodeType >> nodeUUID >> nodePublicSocket >> nodeLocalSocket >> permissions >> isReplicated; - qDebug() << "Node: " << nodeUUID << nodeType << isReplicated; - // if the public socket address is 0 then it's reachable at the same IP // as the domain server if (nodePublicSocket.getAddress().isNull()) { From 3b3465ea05e76dc9accf09b02bff54937ff44cef Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 14:59:45 -0700 Subject: [PATCH 017/115] Add isUpstream to Node --- assignment-client/src/audio/AudioMixerSlave.cpp | 4 ++++ libraries/networking/src/Node.h | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/assignment-client/src/audio/AudioMixerSlave.cpp b/assignment-client/src/audio/AudioMixerSlave.cpp index d01d961e33..2d800c3561 100644 --- a/assignment-client/src/audio/AudioMixerSlave.cpp +++ b/assignment-client/src/audio/AudioMixerSlave.cpp @@ -74,6 +74,10 @@ void AudioMixerSlave::mix(const SharedNodePointer& node) { return; } + if (node->isUpstream()) { + return; + } + // check that the stream is valid auto avatarStream = data->getAvatarAudioStream(); if (avatarStream == nullptr) { diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 2ae0ab82e9..6d74cf4a26 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -51,6 +51,9 @@ public: bool isReplicated() const { return _isReplicated; } void setIsReplicated(bool isReplicated) { _isReplicated = isReplicated; } + bool isUpstream() const { return _isUpstream; } + void setIsUpstream(bool isUpstream) { _isUpstream = isUpstream; } + const QUuid& getConnectionSecret() const { return _connectionSecret; } void setConnectionSecret(const QUuid& connectionSecret) { _connectionSecret = connectionSecret; } @@ -101,6 +104,7 @@ private: MovingPercentile _clockSkewMovingPercentile; NodePermissions _permissions; bool _isReplicated { false }; + bool _isUpstream { false }; tbb::concurrent_unordered_set _ignoredNodeIDSet; mutable QReadWriteLock _ignoredNodeIDSetLock; From efdd3060b0cd16c998447770828970c48d42731c Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 15:00:03 -0700 Subject: [PATCH 018/115] Set isUpstream when receiving a replicated audio packet --- assignment-client/src/audio/AudioMixer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 8f8a6ff2c0..1246617540 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -122,6 +122,7 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess auto node = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, message->getSenderSockAddr(), message->getSenderSockAddr()); + node->setIsUpstream(true); node->setIsMirror(true); node->setLastHeardMicrostamp(usecTimestampNow()); From eb2a41744826a84718cb7769a07fa4d612f8e8bf Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 15:24:14 -0700 Subject: [PATCH 019/115] Add parent back to Node ctor --- libraries/networking/src/LimitedNodeList.cpp | 2 +- libraries/networking/src/Node.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 7aa17ebf04..7b0b2d5bc5 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -586,7 +586,7 @@ SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t return matchingNode; } else { // we didn't have this node, so add them - Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, isReplicated, connectionSecret, this); + Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, isReplicated, connectionSecret); if (nodeType == NodeType::AudioMixer) { LimitedNodeList::flagTimeForConnectionStep(LimitedNodeList::AddedAudioMixer); diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 6d74cf4a26..9082deb55b 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -40,7 +40,7 @@ public: Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions, bool isReplicated, const QUuid& connectionSecret = QUuid(), - QObject* parent = 0); + QObject* parent = nullptr); bool operator==(const Node& otherNode) const { return _uuid == otherNode._uuid; } bool operator!=(const Node& otherNode) const { return !(*this == otherNode); } From a1d3c0dc7b86b82bf25c2bfa23de40b3674c032a Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 15:27:24 -0700 Subject: [PATCH 020/115] Fix initialization ordering error in Node --- libraries/networking/src/Node.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 9082deb55b..11f88b0f52 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -98,12 +98,12 @@ private: QUuid _connectionSecret; std::unique_ptr _linkedData; + bool _isReplicated { false }; int _pingMs; qint64 _clockSkewUsec; QMutex _mutex; MovingPercentile _clockSkewMovingPercentile; NodePermissions _permissions; - bool _isReplicated { false }; bool _isUpstream { false }; tbb::concurrent_unordered_set _ignoredNodeIDSet; mutable QReadWriteLock _ignoredNodeIDSetLock; From 6635187da36253048be2c086352ee82a87bc602d Mon Sep 17 00:00:00 2001 From: seefo Date: Mon, 12 Jun 2017 13:41:54 -0700 Subject: [PATCH 021/115] Changed domain server settings --- .../resources/describe-settings.json | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index b3ab6f935d..0143414db8 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1328,7 +1328,7 @@ "label": "Replicated Users", "type": "table", "can_add_new_rows": true, - "help": "Users that are replicated", + "help": "Users that are replicated to downstream servers", "numbered": false, "columns": [ { @@ -1339,17 +1339,28 @@ ] }, { - "name": "replicants", - "label": "Replicants", + "name": "downstream_servers", + "label": "Downstream Servers", + "assignment-types": [0,1], "type": "table", "can_add_new_rows": true, - "help": "Nodes or mirrors that are to replicate the domain", + "help": "Downstream servers that are relayed data for replicated users", "numbered": false, "columns": [ { - "address": "Address", - "port": "Port", - "mixer_type": "Mixer Type", + "name": "address", + "label": "Address", + "can_set": true + }, + { + "name": "port", + "label": "Port", + "can_set": true + }, + { + "name": "server_type", + "label": "Server Type", + "placeholder": "Audio Mixer", "can_set": true } ] From c7b3b79a235c5ea7fd2f1f2c03da5c6abde2a4dd Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 11:13:43 -0700 Subject: [PATCH 022/115] use replicant nodes for audio packet replication --- .../src/audio/AudioMixerClientData.cpp | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 70c9b615d8..f28e8dfa0b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -108,30 +108,34 @@ void AudioMixerClientData::processPackets() { void AudioMixerClientData::replicatePacket(ReceivedMessage& message) { auto nodeList = DependencyManager::get(); - if (!nodeList->getMirrorSocket().isNull()) { - PacketType mirroredType; - if (message.getType() == PacketType::MicrophoneAudioNoEcho) { - mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; - } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { - mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; - } else if (message.getType() == PacketType::InjectAudio) { - mirroredType = PacketType::ReplicatedInjectAudio; - } else if (message.getType() == PacketType::SilentAudioFrame) { - mirroredType = PacketType::ReplicatedSilentAudioFrame; - } else if (message.getType() == PacketType::NegotiateAudioFormat) { - mirroredType = PacketType::ReplicatedNegotiateAudioFormat; - } else { - return; - } + PacketType mirroredType; - // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packet = NLPacket::create(mirroredType, message.getSize() + NUM_BYTES_RFC4122_UUID); - packet->write(message.getSourceID().toRfc4122()); - packet->write(message.getMessage()); - - nodeList->sendPacket(std::move(packet), nodeList->getMirrorSocket()); + if (message.getType() == PacketType::MicrophoneAudioNoEcho) { + mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; + } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { + mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; + } else if (message.getType() == PacketType::InjectAudio) { + mirroredType = PacketType::ReplicatedInjectAudio; + } else if (message.getType() == PacketType::SilentAudioFrame) { + mirroredType = PacketType::ReplicatedSilentAudioFrame; + } else if (message.getType() == PacketType::NegotiateAudioFormat) { + mirroredType = PacketType::ReplicatedNegotiateAudioFormat; + } else { + return; } + + // construct an NLPacket to send to the replicant that has the contents of the received packet + auto packet = NLPacket::create(mirroredType, message.getSize() + NUM_BYTES_RFC4122_UUID); + packet->write(message.getSourceID().toRfc4122()); + packet->write(message.getMessage()); + + // enumerate the replicant audio mixers and send them the replicated version of this packet + nodeList->eachMatchingNode([&](const SharedNodePointer& node)->bool { + return node->getType() == NodeType::ReplicantAudioMixer; + }, [&](const SharedNodePointer& node) { + nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); + }); } void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) { From d5b466e3ae36eec5ef4b2621e27d92b18ee5c3b6 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 11:31:35 -0700 Subject: [PATCH 023/115] fix replicant handling slot, add node strings for replicants --- assignment-client/src/audio/AudioMixer.cpp | 3 ++- libraries/networking/src/LimitedNodeList.cpp | 3 ++- libraries/networking/src/Node.cpp | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1246617540..0927e02655 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -100,10 +100,11 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : PacketType::ReplicatedSilentAudioFrame, PacketType::ReplicatedNegotiateAudioFormat }, - this, "queueMirroredAudioPacket" + this, "queueReplicatedAudioPacket" ); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); + } void AudioMixer::queueAudioPacket(QSharedPointer message, SharedNodePointer node) { diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 7b0b2d5bc5..afb0dce8af 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -744,7 +744,8 @@ void LimitedNodeList::removeSilentNodes() { SharedNodePointer node = it->second; node->getMutex().lock(); - if ((usecTimestampNow() - node->getLastHeardMicrostamp()) > (NODE_SILENCE_THRESHOLD_MSECS * USECS_PER_MSEC)) { + if (node->getType() != NodeType::ReplicantAudioMixer && node->getType() != NodeType::ReplicantAvatarMixer + && (usecTimestampNow() - node->getLastHeardMicrostamp()) > (NODE_SILENCE_THRESHOLD_MSECS * USECS_PER_MSEC)) { // call the NodeHash erase to get rid of this node it = _nodeHash.unsafe_erase(it); diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 49f032b823..a023e488c4 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -42,6 +42,8 @@ void NodeType::init() { TypeNameHash.insert(NodeType::MessagesMixer, "Messages Mixer"); TypeNameHash.insert(NodeType::AssetServer, "Asset Server"); TypeNameHash.insert(NodeType::EntityScriptServer, "Entity Script Server"); + TypeNameHash.insert(NodeType::ReplicantAudioMixer, "Replicant Audio Mixer"); + TypeNameHash.insert(NodeType::ReplicantAvatarMixer, "Replicant Avatar Mixer"); TypeNameHash.insert(NodeType::Unassigned, "Unassigned"); } From 2929573cb09f46018121d3815f1ea16f5f864b78 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 11:42:46 -0700 Subject: [PATCH 024/115] do not punch to replicant nodes --- libraries/networking/src/NodeList.cpp | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 9b7a7c91dd..5e06ad2b5c 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -712,14 +712,18 @@ void NodeList::pingPunchForInactiveNode(const SharedNodePointer& node) { } void NodeList::startNodeHolePunch(const SharedNodePointer& node) { - // connect to the correct signal on this node so we know when to ping it - connect(node.data(), &Node::pingTimerTimeout, this, &NodeList::handleNodePingTimeout); - // start the ping timer for this node - node->startPingTimer(); + // we don't hole punch to replicants, since it is assumed that we have a direct line to them + if (node->getType() != NodeType::ReplicantAudioMixer && node->getType() != NodeType::ReplicantAvatarMixer) { + // connect to the correct signal on this node so we know when to ping it + connect(node.data(), &Node::pingTimerTimeout, this, &NodeList::handleNodePingTimeout); - // ping this node immediately - pingPunchForInactiveNode(node); + // start the ping timer for this node + node->startPingTimer(); + + // ping this node immediately + pingPunchForInactiveNode(node); + } } void NodeList::handleNodePingTimeout() { @@ -1121,4 +1125,4 @@ void NodeList::setRequestsDomainListData(bool isRequesting) { void NodeList::startThread() { moveToNewNamedThread(this, "NodeList Thread", QThread::TimeCriticalPriority); -} \ No newline at end of file +} From fe668b1bb11d6ad49784abf4c90f7df00448c442 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 12:14:59 -0700 Subject: [PATCH 025/115] make codec negotiation stateless for replicated agents --- assignment-client/src/audio/AudioMixer.cpp | 9 +++-- .../src/audio/AudioMixerClientData.cpp | 33 ++++++++++++++++--- .../src/audio/AudioMixerClientData.h | 2 ++ libraries/networking/src/Node.cpp | 1 + libraries/networking/src/Node.h | 5 --- libraries/networking/src/NodeType.h | 1 + .../networking/src/udt/PacketHeaders.cpp | 3 +- libraries/networking/src/udt/PacketHeaders.h | 1 - 8 files changed, 38 insertions(+), 17 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 0927e02655..d997c3504b 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -97,8 +97,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : PacketType::ReplicatedMicrophoneAudioNoEcho, PacketType::ReplicatedMicrophoneAudioWithEcho, PacketType::ReplicatedInjectAudio, - PacketType::ReplicatedSilentAudioFrame, - PacketType::ReplicatedNegotiateAudioFormat + PacketType::ReplicatedSilentAudioFrame }, this, "queueReplicatedAudioPacket" ); @@ -121,10 +120,10 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - auto node = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, - message->getSenderSockAddr(), message->getSenderSockAddr()); + auto node = nodeList->addOrUpdateNode(nodeID, NodeType::ReplicatedAgent, + message->getSenderSockAddr(), message->getSenderSockAddr(), + DEFAULT_AGENT_PERMISSIONS, true); node->setIsUpstream(true); - node->setIsMirror(true); node->setLastHeardMicrostamp(usecTimestampNow()); getOrCreateClientData(node.data())->queuePacket(message, node); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index f28e8dfa0b..8ee7f9f383 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -73,6 +73,11 @@ void AudioMixerClientData::processPackets() { case PacketType::ReplicatedMicrophoneAudioWithEcho: case PacketType::ReplicatedInjectAudio: case PacketType::ReplicatedSilentAudioFrame: { + + if (node->getType() == NodeType::ReplicatedAgent && !_codec) { + setupCodecForReplicatedAgent(packet); + } + QMutexLocker lock(&getMutex()); parseData(*packet); @@ -81,9 +86,7 @@ void AudioMixerClientData::processPackets() { break; } case PacketType::NegotiateAudioFormat: - case PacketType::ReplicatedNegotiateAudioFormat: negotiateAudioFormat(*packet, node); - replicatePacket(*packet); break; case PacketType::RequestsDomainListData: parseRequestsDomainListData(*packet); @@ -119,15 +122,20 @@ void AudioMixerClientData::replicatePacket(ReceivedMessage& message) { mirroredType = PacketType::ReplicatedInjectAudio; } else if (message.getType() == PacketType::SilentAudioFrame) { mirroredType = PacketType::ReplicatedSilentAudioFrame; - } else if (message.getType() == PacketType::NegotiateAudioFormat) { - mirroredType = PacketType::ReplicatedNegotiateAudioFormat; } else { return; } // construct an NLPacket to send to the replicant that has the contents of the received packet auto packet = NLPacket::create(mirroredType, message.getSize() + NUM_BYTES_RFC4122_UUID); + + // since this packet will be non-sourced, we add the replicated node's ID here packet->write(message.getSourceID().toRfc4122()); + + // we won't negotiate an audio format with the replicant, because we aren't a listener + // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs + packet->writeString(_selectedCodecName); + packet->write(message.getMessage()); // enumerate the replicant audio mixers and send them the replicated version of this packet @@ -311,7 +319,12 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho || packetType == PacketType::ReplicatedSilentAudioFrame || packetType == PacketType::ReplicatedInjectAudio) { + + // skip past source ID for the replicated packet message.seek(NUM_BYTES_RFC4122_UUID); + + // skip past the codec string + message.readString(); } // check the overflow count before we parse data @@ -660,3 +673,15 @@ bool AudioMixerClientData::shouldIgnore(const SharedNodePointer self, const Shar return shouldIgnore; } + +void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer message) { + // first pull the codec string from the packet + + // read the string for the codec + auto codecString = message->readString(); + + qDebug() << "Manually setting codec for replicated agent" << codecString; + + const std::pair codec = AudioMixer::negotiateCodec({ codecString }); + setupCodec(codec.second, codec.first); +} diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 5ab55dd48e..72761e64f2 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -108,6 +108,8 @@ public: bool getRequestsDomainListData() { return _requestsDomainListData; } void setRequestsDomainListData(bool requesting) { _requestsDomainListData = requesting; } + void setupCodecForReplicatedAgent(QSharedPointer message); + signals: void injectorStreamFinished(const QUuid& streamIdentifier); diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index a023e488c4..a6b6bc18b2 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -44,6 +44,7 @@ void NodeType::init() { TypeNameHash.insert(NodeType::EntityScriptServer, "Entity Script Server"); TypeNameHash.insert(NodeType::ReplicantAudioMixer, "Replicant Audio Mixer"); TypeNameHash.insert(NodeType::ReplicantAvatarMixer, "Replicant Avatar Mixer"); + TypeNameHash.insert(NodeType::ReplicatedAgent, "Replicated Agent"); TypeNameHash.insert(NodeType::Unassigned, "Unassigned"); } diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 11f88b0f52..c905c9d551 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -86,9 +86,6 @@ public: bool isIgnoreRadiusEnabled() const { return _ignoreRadiusEnabled; } - bool isMirror() const { return _isMirror; } - void setIsMirror(bool isMirror) { _isMirror = isMirror; } - private: // privatize copy and assignment operator to disallow Node copying Node(const Node &otherNode); @@ -109,8 +106,6 @@ private: mutable QReadWriteLock _ignoredNodeIDSetLock; std::atomic_bool _ignoreRadiusEnabled; - - bool _isMirror { false }; }; Q_DECLARE_METATYPE(Node*) diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 7324dcaf14..7c327e08f7 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -27,6 +27,7 @@ namespace NodeType { const NodeType_t EntityScriptServer = 'S'; const NodeType_t ReplicantAudioMixer = 'a'; const NodeType_t ReplicantAvatarMixer = 'w'; + const NodeType_t ReplicatedAgent = 'z'; const NodeType_t Unassigned = 1; void init(); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ebc8e80e45..f2f0062861 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -41,8 +41,7 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement << PacketType::ReplicatedMicrophoneAudioNoEcho << PacketType::ReplicatedMicrophoneAudioWithEcho - << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame - << PacketType::ReplicatedNegotiateAudioFormat; + << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 3ace60d502..448ae812ff 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -115,7 +115,6 @@ public: AdjustAvatarSorting, OctreeFileReplacement, CollisionEventChanges, - ReplicatedNegotiateAudioFormat, ReplicatedMicrophoneAudioNoEcho, ReplicatedMicrophoneAudioWithEcho, ReplicatedInjectAudio, From a4aa9689a6098fcf8a445f03ed878189897f56a2 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 12:51:18 -0700 Subject: [PATCH 026/115] don't construct a packet of exact size for replicated audio packets --- assignment-client/src/audio/AudioMixer.cpp | 1 - assignment-client/src/audio/AudioMixerClientData.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index d997c3504b..2f2b424948 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -103,7 +103,6 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : ); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); - } void AudioMixer::queueAudioPacket(QSharedPointer message, SharedNodePointer node) { diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 8ee7f9f383..6ef795f57b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -127,7 +127,7 @@ void AudioMixerClientData::replicatePacket(ReceivedMessage& message) { } // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packet = NLPacket::create(mirroredType, message.getSize() + NUM_BYTES_RFC4122_UUID); + auto packet = NLPacket::create(mirroredType); // since this packet will be non-sourced, we add the replicated node's ID here packet->write(message.getSourceID().toRfc4122()); From 88af8b584252996a218014f82a946c8e61e34709 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 12:56:15 -0700 Subject: [PATCH 027/115] use new downstream/upstream nomeclature --- assignment-client/src/audio/AudioMixer.cpp | 2 +- assignment-client/src/audio/AudioMixerClientData.cpp | 4 ++-- libraries/networking/src/LimitedNodeList.cpp | 2 +- libraries/networking/src/Node.cpp | 5 ++--- libraries/networking/src/NodeList.cpp | 2 +- libraries/networking/src/NodeType.h | 5 ++--- 6 files changed, 9 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 2f2b424948..9fb0a96382 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -119,7 +119,7 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - auto node = nodeList->addOrUpdateNode(nodeID, NodeType::ReplicatedAgent, + auto node = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, message->getSenderSockAddr(), message->getSenderSockAddr(), DEFAULT_AGENT_PERMISSIONS, true); node->setIsUpstream(true); diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 6ef795f57b..b95ca5b616 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -74,7 +74,7 @@ void AudioMixerClientData::processPackets() { case PacketType::ReplicatedInjectAudio: case PacketType::ReplicatedSilentAudioFrame: { - if (node->getType() == NodeType::ReplicatedAgent && !_codec) { + if (node->isUpstream() && !_codec) { setupCodecForReplicatedAgent(packet); } @@ -140,7 +140,7 @@ void AudioMixerClientData::replicatePacket(ReceivedMessage& message) { // enumerate the replicant audio mixers and send them the replicated version of this packet nodeList->eachMatchingNode([&](const SharedNodePointer& node)->bool { - return node->getType() == NodeType::ReplicantAudioMixer; + return node->getType() == NodeType::DownstreamAudioMixer; }, [&](const SharedNodePointer& node) { nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); }); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index afb0dce8af..7ca991667e 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -744,7 +744,7 @@ void LimitedNodeList::removeSilentNodes() { SharedNodePointer node = it->second; node->getMutex().lock(); - if (node->getType() != NodeType::ReplicantAudioMixer && node->getType() != NodeType::ReplicantAvatarMixer + if (node->getType() != NodeType::DownstreamAudioMixer && node->getType() != NodeType::DownstreamAvatarMixer && (usecTimestampNow() - node->getLastHeardMicrostamp()) > (NODE_SILENCE_THRESHOLD_MSECS * USECS_PER_MSEC)) { // call the NodeHash erase to get rid of this node it = _nodeHash.unsafe_erase(it); diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index a6b6bc18b2..9331487db9 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -42,9 +42,8 @@ void NodeType::init() { TypeNameHash.insert(NodeType::MessagesMixer, "Messages Mixer"); TypeNameHash.insert(NodeType::AssetServer, "Asset Server"); TypeNameHash.insert(NodeType::EntityScriptServer, "Entity Script Server"); - TypeNameHash.insert(NodeType::ReplicantAudioMixer, "Replicant Audio Mixer"); - TypeNameHash.insert(NodeType::ReplicantAvatarMixer, "Replicant Avatar Mixer"); - TypeNameHash.insert(NodeType::ReplicatedAgent, "Replicated Agent"); + TypeNameHash.insert(NodeType::DownstreamAudioMixer, "Downstream Audio Mixer"); + TypeNameHash.insert(NodeType::DownstreamAvatarMixer, "Downstream Avatar Mixer"); TypeNameHash.insert(NodeType::Unassigned, "Unassigned"); } diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 5e06ad2b5c..ddd2137422 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -714,7 +714,7 @@ void NodeList::pingPunchForInactiveNode(const SharedNodePointer& node) { void NodeList::startNodeHolePunch(const SharedNodePointer& node) { // we don't hole punch to replicants, since it is assumed that we have a direct line to them - if (node->getType() != NodeType::ReplicantAudioMixer && node->getType() != NodeType::ReplicantAvatarMixer) { + if (node->getType() != NodeType::DownstreamAudioMixer && node->getType() != NodeType::DownstreamAvatarMixer) { // connect to the correct signal on this node so we know when to ping it connect(node.data(), &Node::pingTimerTimeout, this, &NodeList::handleNodePingTimeout); diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 7c327e08f7..7f0d5f0636 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -25,9 +25,8 @@ namespace NodeType { const NodeType_t AssetServer = 'A'; const NodeType_t MessagesMixer = 'm'; const NodeType_t EntityScriptServer = 'S'; - const NodeType_t ReplicantAudioMixer = 'a'; - const NodeType_t ReplicantAvatarMixer = 'w'; - const NodeType_t ReplicatedAgent = 'z'; + const NodeType_t DownstreamAudioMixer = 'a'; + const NodeType_t DownstreamAvatarMixer = 'w'; const NodeType_t Unassigned = 1; void init(); From 91c25d4270646862b186d34b464eff76d17cd00e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 13:15:02 -0700 Subject: [PATCH 028/115] don't continously set codec for upstream agent --- assignment-client/src/audio/AudioMixerClientData.cpp | 7 +++++-- assignment-client/src/audio/AudioMixerClientData.h | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index b95ca5b616..082ac364be 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -74,7 +74,7 @@ void AudioMixerClientData::processPackets() { case PacketType::ReplicatedInjectAudio: case PacketType::ReplicatedSilentAudioFrame: { - if (node->isUpstream() && !_codec) { + if (node->isUpstream() && !_hasSetupCodecForUpstreamNode) { setupCodecForReplicatedAgent(packet); } @@ -261,7 +261,8 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { avatarAudioStream->setupCodec(_codec, _selectedCodecName, AudioConstants::MONO); qDebug() << "creating new AvatarAudioStream... codec:" << _selectedCodecName; - connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec, this, &AudioMixerClientData::handleMismatchAudioFormat); + connect(avatarAudioStream, &InboundAudioStream::mismatchedAudioCodec, + this, &AudioMixerClientData::handleMismatchAudioFormat); auto emplaced = _audioStreams.emplace( QUuid(), @@ -684,4 +685,6 @@ void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer codec = AudioMixer::negotiateCodec({ codecString }); setupCodec(codec.second, codec.first); + + _hasSetupCodecForUpstreamNode = true; } diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 72761e64f2..0f9e4c4c18 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -185,6 +185,8 @@ private: bool _shouldMuteClient { false }; bool _requestsDomainListData { false }; + + bool _hasSetupCodecForUpstreamNode { false }; }; #endif // hifi_AudioMixerClientData_h From 7ed9483467083190f715e8b348b1017f3cef563e Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 13:57:55 -0700 Subject: [PATCH 029/115] use blocking queued for addOrUpdateNode because of parenting --- assignment-client/src/audio/AudioMixer.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 9fb0a96382..1e93d0515c 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -119,13 +119,13 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - auto node = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, - message->getSenderSockAddr(), message->getSenderSockAddr(), - DEFAULT_AGENT_PERMISSIONS, true); - node->setIsUpstream(true); - node->setLastHeardMicrostamp(usecTimestampNow()); + auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, + message->getSenderSockAddr(), message->getSenderSockAddr(), + DEFAULT_AGENT_PERMISSIONS, true); + replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); + replicatedNode->setIsUpstream(true); - getOrCreateClientData(node.data())->queuePacket(message, node); + getOrCreateClientData(replicatedNode.data())->queuePacket(message, replicatedNode); } void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer message, SharedNodePointer sendingNode) { From 1868971cfc4c66f40993676c7793ef5c69fc2667 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 14:46:53 -0700 Subject: [PATCH 030/115] fix debug for manual codec in upstream agents --- assignment-client/src/audio/AudioMixerClientData.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 082ac364be..44c51c4ae2 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -681,7 +681,8 @@ void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointerreadString(); - qDebug() << "Manually setting codec for replicated agent" << codecString; + qDebug() << "Manually setting codec for replicated agent" << uuidStringWithoutCurlyBraces(getNodeID()) + << "-" << codecString; const std::pair codec = AudioMixer::negotiateCodec({ codecString }); setupCodec(codec.second, codec.first); From 03a8d7b8c849547ed7f3be7dc5ecef4437da5d46 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 14:56:28 -0700 Subject: [PATCH 031/115] only replicate packets for agents being replicated --- .../src/audio/AudioMixerClientData.cpp | 77 +++++++++++-------- .../src/audio/AudioMixerClientData.h | 2 +- 2 files changed, 45 insertions(+), 34 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 44c51c4ae2..359c7e1b9e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -80,8 +80,8 @@ void AudioMixerClientData::processPackets() { QMutexLocker lock(&getMutex()); parseData(*packet); - - replicatePacket(*packet); + + optionallyReplicatePacket(*packet, *node); break; } @@ -109,41 +109,52 @@ void AudioMixerClientData::processPackets() { assert(_packetQueue.empty()); } -void AudioMixerClientData::replicatePacket(ReceivedMessage& message) { - auto nodeList = DependencyManager::get(); +void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) { - PacketType mirroredType; + // first, make sure that this is a packet from a node we are supposed to replicate + if (node.isReplicated()) { + auto nodeList = DependencyManager::get(); - if (message.getType() == PacketType::MicrophoneAudioNoEcho) { - mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; - } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { - mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; - } else if (message.getType() == PacketType::InjectAudio) { - mirroredType = PacketType::ReplicatedInjectAudio; - } else if (message.getType() == PacketType::SilentAudioFrame) { - mirroredType = PacketType::ReplicatedSilentAudioFrame; - } else { - return; + // now make sure it's a packet type that we want to replicate + PacketType mirroredType; + + if (message.getType() == PacketType::MicrophoneAudioNoEcho) { + mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; + } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { + mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; + } else if (message.getType() == PacketType::InjectAudio) { + mirroredType = PacketType::ReplicatedInjectAudio; + } else if (message.getType() == PacketType::SilentAudioFrame) { + mirroredType = PacketType::ReplicatedSilentAudioFrame; + } else { + return; + } + + std::unique_ptr packet; + + // enumerate the replicant audio mixers and send them the replicated version of this packet + nodeList->eachMatchingNode([&](const SharedNodePointer& node)->bool { + return node->getType() == NodeType::DownstreamAudioMixer; + }, [&](const SharedNodePointer& node) { + // construct the packet only once, if we have any downstream audio mixers to send to + if (!packet) { + // construct an NLPacket to send to the replicant that has the contents of the received packet + packet = NLPacket::create(mirroredType); + + // since this packet will be non-sourced, we add the replicated node's ID here + packet->write(message.getSourceID().toRfc4122()); + + // we won't negotiate an audio format with the replicant, because we aren't a listener + // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs + packet->writeString(_selectedCodecName); + + packet->write(message.getMessage()); + } + + nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); + }); } - // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packet = NLPacket::create(mirroredType); - - // since this packet will be non-sourced, we add the replicated node's ID here - packet->write(message.getSourceID().toRfc4122()); - - // we won't negotiate an audio format with the replicant, because we aren't a listener - // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs - packet->writeString(_selectedCodecName); - - packet->write(message.getMessage()); - - // enumerate the replicant audio mixers and send them the replicated version of this packet - nodeList->eachMatchingNode([&](const SharedNodePointer& node)->bool { - return node->getType() == NodeType::DownstreamAudioMixer; - }, [&](const SharedNodePointer& node) { - nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); - }); } void AudioMixerClientData::negotiateAudioFormat(ReceivedMessage& message, const SharedNodePointer& node) { diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index 0f9e4c4c18..b397ab0b8e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -126,7 +126,7 @@ private: QReadWriteLock _streamsLock; AudioStreamMap _audioStreams; // microphone stream from avatar is stored under key of null UUID - void replicatePacket(ReceivedMessage& packet); + void optionallyReplicatePacket(ReceivedMessage& packet, const Node& node); using IgnoreZone = AABox; class IgnoreZoneMemo { From 4042e0494644cd443ce8c2adca02df0ee3cf6c9a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 15:32:09 -0700 Subject: [PATCH 032/115] add isDownstream for cleaner conditionals --- libraries/networking/src/LimitedNodeList.cpp | 2 +- libraries/networking/src/Node.cpp | 4 ++++ libraries/networking/src/NodeList.cpp | 2 +- libraries/networking/src/NodeType.h | 1 + 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 7ca991667e..09300cd919 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -744,7 +744,7 @@ void LimitedNodeList::removeSilentNodes() { SharedNodePointer node = it->second; node->getMutex().lock(); - if (node->getType() != NodeType::DownstreamAudioMixer && node->getType() != NodeType::DownstreamAvatarMixer + if (!NodeType::isDownstream(node->getType()) && (usecTimestampNow() - node->getLastHeardMicrostamp()) > (NODE_SILENCE_THRESHOLD_MSECS * USECS_PER_MSEC)) { // call the NodeHash erase to get rid of this node it = _nodeHash.unsafe_erase(it); diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 9331487db9..26e75f36f5 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -52,6 +52,10 @@ const QString& NodeType::getNodeTypeName(NodeType_t nodeType) { return matchedTypeName != TypeNameHash.end() ? matchedTypeName.value() : UNKNOWN_NodeType_t_NAME; } +bool NodeType::isDownstream(NodeType_t nodeType) { + return nodeType == NodeType::DownstreamAudioMixer || nodeType == NodeType::DownstreamAvatarMixer; +} + Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions, bool isReplicated, const QUuid& connectionSecret, QObject* parent) : diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index ddd2137422..38d8c79251 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -714,7 +714,7 @@ void NodeList::pingPunchForInactiveNode(const SharedNodePointer& node) { void NodeList::startNodeHolePunch(const SharedNodePointer& node) { // we don't hole punch to replicants, since it is assumed that we have a direct line to them - if (node->getType() != NodeType::DownstreamAudioMixer && node->getType() != NodeType::DownstreamAvatarMixer) { + if (!NodeType::isDownstream(node->getType())) { // connect to the correct signal on this node so we know when to ping it connect(node.data(), &Node::pingTimerTimeout, this, &NodeList::handleNodePingTimeout); diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 7f0d5f0636..ef0aa78357 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -31,6 +31,7 @@ namespace NodeType { void init(); const QString& getNodeTypeName(NodeType_t nodeType); + bool isDownstream(NodeType_t nodeType); } typedef QSet NodeSet; From 9fa97d611a7b89c775fe3713cf6b0c3b8fac4098 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 15:36:46 -0700 Subject: [PATCH 033/115] remove invoked addOrUpdate and move node to node list thread --- assignment-client/src/audio/AudioMixer.cpp | 5 +++++ libraries/networking/src/LimitedNodeList.cpp | 3 +++ 2 files changed, 8 insertions(+) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 1e93d0515c..b4e06863db 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -120,8 +120,13 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, +<<<<<<< HEAD message->getSenderSockAddr(), message->getSenderSockAddr(), DEFAULT_AGENT_PERMISSIONS, true); +======= + message->getSenderSockAddr(), message->getSenderSockAddr()); + +>>>>>>> remove invoked addOrUpdate and move node to node list thread replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 09300cd919..0494efc7b4 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -588,6 +588,9 @@ SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t // we didn't have this node, so add them Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, isReplicated, connectionSecret); + // move the newly constructed node to the LNL thread + newNode->moveToThread(thread()); + if (nodeType == NodeType::AudioMixer) { LimitedNodeList::flagTimeForConnectionStep(LimitedNodeList::AddedAudioMixer); } From 4539d615d73df45c174c8dea5733ecc5ed751ca8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 15:41:34 -0700 Subject: [PATCH 034/115] add downstream server settings handling to ThreadedAssignment --- assignment-client/src/audio/AudioMixer.cpp | 9 ++--- libraries/networking/src/Node.cpp | 11 ++++++ libraries/networking/src/Node.h | 1 + libraries/networking/src/NodeType.h | 2 ++ .../networking/src/ThreadedAssignment.cpp | 35 +++++++++++++++++++ libraries/networking/src/ThreadedAssignment.h | 2 ++ 6 files changed, 54 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b4e06863db..f8c2cf86e8 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -120,13 +120,8 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess QUuid nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, -<<<<<<< HEAD message->getSenderSockAddr(), message->getSenderSockAddr(), DEFAULT_AGENT_PERMISSIONS, true); -======= - message->getSenderSockAddr(), message->getSenderSockAddr()); - ->>>>>>> remove invoked addOrUpdate and move node to node list thread replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); @@ -536,7 +531,7 @@ void AudioMixer::clearDomainSettings() { _zoneReverbSettings.clear(); } -void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { +void AudioMixer::parseSettingsObject(const QJsonObject& settingsObject) { qDebug() << "AVX2 Support:" << (cpuSupportsAVX2() ? "enabled" : "disabled"); if (settingsObject.contains(AUDIO_THREADING_GROUP_KEY)) { @@ -754,6 +749,8 @@ void AudioMixer::parseSettingsObject(const QJsonObject &settingsObject) { } } } + + parseDownstreamServers(settingsObject, NodeType::AudioMixer); } AudioMixer::Timer::Timing::Timing(uint64_t& sum) : _sum(sum) { diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index 26e75f36f5..3435843814 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -56,6 +56,17 @@ bool NodeType::isDownstream(NodeType_t nodeType) { return nodeType == NodeType::DownstreamAudioMixer || nodeType == NodeType::DownstreamAvatarMixer; } +NodeType_t NodeType::downstreamType(NodeType_t primaryType) { + switch (primaryType) { + case AudioMixer: + return DownstreamAudioMixer; + case AvatarMixer: + return DownstreamAvatarMixer; + default: + return Unassigned; + } +} + Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions, bool isReplicated, const QUuid& connectionSecret, QObject* parent) : diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index c905c9d551..fd2cf6b65b 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -37,6 +37,7 @@ class Node : public NetworkPeer { Q_OBJECT public: + Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, const NodePermissions& permissions, bool isReplicated, const QUuid& connectionSecret = QUuid(), diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index ef0aa78357..12e74603ef 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -30,8 +30,10 @@ namespace NodeType { const NodeType_t Unassigned = 1; void init(); + const QString& getNodeTypeName(NodeType_t nodeType); bool isDownstream(NodeType_t nodeType); + NodeType_t downstreamType(NodeType_t primaryType); } typedef QSet NodeSet; diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 9db540cae2..12dc59beed 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -10,6 +10,7 @@ // #include +#include #include #include #include @@ -132,3 +133,37 @@ void ThreadedAssignment::domainSettingsRequestFailed() { qCDebug(networking) << "Failed to retreive settings object from domain-server. Bailing on assignment."; setFinished(true); } + +void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType) { + static const QString REPLICATION_GROUP_KEY = "replication"; + static const QString DOWNSTREAM_SERVERS_SETTING_KEY = "downstream_servers"; + if (settingsObject.contains(REPLICATION_GROUP_KEY)) { + const QJsonObject replicationObject = settingsObject[REPLICATION_GROUP_KEY].toObject(); + + const QJsonArray downstreamServers = replicationObject[DOWNSTREAM_SERVERS_SETTING_KEY].toArray(); + + auto nodeList = DependencyManager::get(); + + foreach(const QJsonValue& downstreamServerValue, downstreamServers) { + const QJsonArray downstreamServer = downstreamServerValue.toArray(); + + // make sure we have the number of values we need + if (downstreamServer.size() >= 3) { + // make sure this is a downstream server that matches our type + if (downstreamServer[2].toString() == NodeType::getNodeTypeName(nodeType)) { + // read the address and port and construct a HifiSockAddr from them + HifiSockAddr downstreamServerAddr { + downstreamServer[0].toString(), + static_cast(downstreamServer[1].toInt()) + }; + + // manually add the downstream node to our node list + nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), + downstreamServerAddr, downstreamServerAddr); + + } + } + + } + } +} diff --git a/libraries/networking/src/ThreadedAssignment.h b/libraries/networking/src/ThreadedAssignment.h index e9832a2a91..f96755a776 100644 --- a/libraries/networking/src/ThreadedAssignment.h +++ b/libraries/networking/src/ThreadedAssignment.h @@ -40,6 +40,8 @@ signals: protected: void commonInit(const QString& targetName, NodeType_t nodeType); + void parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType); + bool _isFinished; QTimer _domainServerTimer; QTimer _statsTimer; From 4688fe4c394fb7c734a28cda3e68b65391010a3f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 15:54:09 -0700 Subject: [PATCH 035/115] fix comment for change from replicant to downstream --- assignment-client/src/audio/AudioMixerClientData.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 359c7e1b9e..de7bdbc780 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -132,7 +132,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c std::unique_ptr packet; - // enumerate the replicant audio mixers and send them the replicated version of this packet + // enumerate the downstream audio mixers and send them the replicated version of this packet nodeList->eachMatchingNode([&](const SharedNodePointer& node)->bool { return node->getType() == NodeType::DownstreamAudioMixer; }, [&](const SharedNodePointer& node) { From 880157695a708671b44cccd3e32df09f9cd0079a Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 16:18:15 -0700 Subject: [PATCH 036/115] do not attempt to hole punch an upstream node --- libraries/networking/src/NodeList.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 38d8c79251..06548499e8 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -713,8 +713,10 @@ void NodeList::pingPunchForInactiveNode(const SharedNodePointer& node) { void NodeList::startNodeHolePunch(const SharedNodePointer& node) { - // we don't hole punch to replicants, since it is assumed that we have a direct line to them - if (!NodeType::isDownstream(node->getType())) { + // we don't hole punch to downstream servers, since it is assumed that we have a direct line to them + // we also don't hole punch to relayed upstream nodes, since we do not communicate directly with them + + if (!NodeType::isDownstream(node->getType()) && !node->isUpstream()) { // connect to the correct signal on this node so we know when to ping it connect(node.data(), &Node::pingTimerTimeout, this, &NodeList::handleNodePingTimeout); From 63c8273a4164cb8c57fc408ee0027abcb0537655 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 16:23:57 -0700 Subject: [PATCH 037/115] do not send keep alive pings to upstream nodes --- libraries/networking/src/NodeList.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 06548499e8..73c405e3aa 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -768,8 +768,10 @@ void NodeList::stopKeepalivePingTimer() { } void NodeList::sendKeepAlivePings() { + // send keep-alive ping packets to nodes of types we care about that are not relayed to us from an upstream node + eachMatchingNode([this](const SharedNodePointer& node)->bool { - return _nodeTypesOfInterest.contains(node->getType()); + return !node->isUpstream() && _nodeTypesOfInterest.contains(node->getType()); }, [&](const SharedNodePointer& node) { sendPacket(constructPingPacket(), *node); }); From 682fa247450fe00bcd8130c3531433fee5916ad0 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 16:51:25 -0700 Subject: [PATCH 038/115] fix downstream server parsing from settings --- .../networking/src/ThreadedAssignment.cpp | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 12dc59beed..5aeb076c11 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -145,25 +145,25 @@ void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObjec auto nodeList = DependencyManager::get(); foreach(const QJsonValue& downstreamServerValue, downstreamServers) { - const QJsonArray downstreamServer = downstreamServerValue.toArray(); + const QJsonObject downstreamServer = downstreamServerValue.toObject(); - // make sure we have the number of values we need - if (downstreamServer.size() >= 3) { - // make sure this is a downstream server that matches our type - if (downstreamServer[2].toString() == NodeType::getNodeTypeName(nodeType)) { - // read the address and port and construct a HifiSockAddr from them - HifiSockAddr downstreamServerAddr { - downstreamServer[0].toString(), - static_cast(downstreamServer[1].toInt()) - }; + static const QString DOWNSTREAM_SERVER_ADDRESS = "address"; + static const QString DOWNSTREAM_SERVER_PORT = "port"; + static const QString DOWNSTREAM_SERVER_TYPE = "server_type"; - // manually add the downstream node to our node list - nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), - downstreamServerAddr, downstreamServerAddr); + // make sure we have the settings we need for this downstream server + if (downstreamServer.contains(DOWNSTREAM_SERVER_ADDRESS) && downstreamServer.contains(DOWNSTREAM_SERVER_PORT) + && downstreamServer[DOWNSTREAM_SERVER_TYPE].toString() == NodeType::getNodeTypeName(nodeType)) { + // read the address and port and construct a HifiSockAddr from them + HifiSockAddr downstreamServerAddr { + downstreamServer[DOWNSTREAM_SERVER_ADDRESS].toString(), + (quint16) downstreamServer[DOWNSTREAM_SERVER_PORT].toString().toInt() + }; - } + // manually add the downstream node to our node list + nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), + downstreamServerAddr, downstreamServerAddr); } - } } } From 30d2e9fd237f7833efd63c78f0b4a766c36fc158 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 17:51:54 -0700 Subject: [PATCH 039/115] add unsafeEachNode to iterate nodes when read lock held elsewhere --- .../src/audio/AudioMixerClientData.cpp | 31 ++++++++++--------- libraries/networking/src/LimitedNodeList.h | 10 ++++++ 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index de7bdbc780..c51f240b54 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -113,6 +113,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c // first, make sure that this is a packet from a node we are supposed to replicate if (node.isReplicated()) { + auto nodeList = DependencyManager::get(); // now make sure it's a packet type that we want to replicate @@ -133,25 +134,25 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c std::unique_ptr packet; // enumerate the downstream audio mixers and send them the replicated version of this packet - nodeList->eachMatchingNode([&](const SharedNodePointer& node)->bool { - return node->getType() == NodeType::DownstreamAudioMixer; - }, [&](const SharedNodePointer& node) { - // construct the packet only once, if we have any downstream audio mixers to send to - if (!packet) { - // construct an NLPacket to send to the replicant that has the contents of the received packet - packet = NLPacket::create(mirroredType); + nodeList->unsafeEachNode([&](const SharedNodePointer& node) { + if (node->getType() == NodeType::DownstreamAudioMixer) { + // construct the packet only once, if we have any downstream audio mixers to send to + if (!packet) { + // construct an NLPacket to send to the replicant that has the contents of the received packet + packet = NLPacket::create(mirroredType); - // since this packet will be non-sourced, we add the replicated node's ID here - packet->write(message.getSourceID().toRfc4122()); + // since this packet will be non-sourced, we add the replicated node's ID here + packet->write(message.getSourceID().toRfc4122()); - // we won't negotiate an audio format with the replicant, because we aren't a listener - // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs - packet->writeString(_selectedCodecName); + // we won't negotiate an audio format with the replicant, because we aren't a listener + // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs + packet->writeString(_selectedCodecName); + + packet->write(message.getMessage()); + } - packet->write(message.getMessage()); + nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); } - - nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); }); } diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 1ace6b2e23..5e2c88ed94 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -257,6 +257,16 @@ public: return SharedNodePointer(); } + // This is unsafe because it does not take a lock + // Must only be called when you know that a read lock on the node mutex is held + // and will be held for the duration of your iteration + template + void unsafeEachNode(NodeLambda functor) { + for (NodeHash::const_iterator it = _nodeHash.cbegin(); it != _nodeHash.cend(); ++it) { + functor(it->second); + } + } + void putLocalPortIntoSharedMemory(const QString key, QObject* parent, quint16 localPort); bool getLocalServerPortFromSharedMemory(const QString key, quint16& localPort); From ab3a0ddba2b7c9a8ee04a5163dcbd63255dd3a48 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 12 Jun 2017 19:49:02 -0700 Subject: [PATCH 040/115] don't emit codec mismatch upstream, handle replicated silent frames --- libraries/audio/src/InboundAudioStream.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 88ec7e7bc0..65cccf1fe0 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -147,7 +147,8 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { } case SequenceNumberStats::OnTime: { // Packet is on time; parse its data to the ringbuffer - if (message.getType() == PacketType::SilentAudioFrame) { + if (message.getType() == PacketType::SilentAudioFrame + || message.getType() == PacketType::ReplicatedSilentAudioFrame) { // If we recieved a SilentAudioFrame from our sender, we might want to drop // some of the samples in order to catch up to our desired jitter buffer size. writeDroppableSilentFrames(networkFrames); @@ -168,7 +169,10 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // inform others of the mismatch auto sendingNode = DependencyManager::get()->nodeWithUUID(message.getSourceID()); - emit mismatchedAudioCodec(sendingNode, _selectedCodecName, codecInPacket); + if (sendingNode) { + emit mismatchedAudioCodec(sendingNode, _selectedCodecName, codecInPacket); + } + } } break; From aa9574fc5a0316e0ca06e86d65cac0539ccdcd07 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 09:46:12 -0700 Subject: [PATCH 041/115] add re-replication support to audio mixer --- .../src/audio/AudioMixerClientData.cpp | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index c51f240b54..d0e749563a 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -119,6 +119,23 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c // now make sure it's a packet type that we want to replicate PacketType mirroredType; + switch (message.getType()) { + case PacketType::MicrophoneAudioNoEcho: + case PacketType::ReplicatedMicrophoneAudioNoEcho: + mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; + case PacketType::MicrophoneAudioWithEcho: + case PacketType::ReplicatedMicrophoneAudioWithEcho: + mirroredType = PacketType::ReplicatedMicrophoneAudioWithEcho; + case PacketType::InjectAudio: + case PacketType::ReplicatedInjectAudio: + mirroredType = PacketType::ReplicatedInjectAudio; + case PacketType::SilentAudioFrame: + case PacketType::ReplicatedSilentAudioFrame: + mirroredType = PacketType::ReplicatedSilentAudioFrame; + default: + return; + } + if (message.getType() == PacketType::MicrophoneAudioNoEcho) { mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { @@ -142,7 +159,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c packet = NLPacket::create(mirroredType); // since this packet will be non-sourced, we add the replicated node's ID here - packet->write(message.getSourceID().toRfc4122()); + packet->write(node->getUUID().toRfc4122()); // we won't negotiate an audio format with the replicant, because we aren't a listener // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs From eca35ce01300d0b55a010bf529a5e66211274bda Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 11:18:59 -0700 Subject: [PATCH 042/115] fix double check for packet types --- .../src/audio/AudioMixerClientData.cpp | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index d0e749563a..e7b45b2d4e 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -123,31 +123,23 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c case PacketType::MicrophoneAudioNoEcho: case PacketType::ReplicatedMicrophoneAudioNoEcho: mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; + break; case PacketType::MicrophoneAudioWithEcho: case PacketType::ReplicatedMicrophoneAudioWithEcho: mirroredType = PacketType::ReplicatedMicrophoneAudioWithEcho; + break; case PacketType::InjectAudio: case PacketType::ReplicatedInjectAudio: mirroredType = PacketType::ReplicatedInjectAudio; + break; case PacketType::SilentAudioFrame: case PacketType::ReplicatedSilentAudioFrame: mirroredType = PacketType::ReplicatedSilentAudioFrame; + break; default: return; } - if (message.getType() == PacketType::MicrophoneAudioNoEcho) { - mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; - } else if (message.getType() == PacketType::MicrophoneAudioWithEcho) { - mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; - } else if (message.getType() == PacketType::InjectAudio) { - mirroredType = PacketType::ReplicatedInjectAudio; - } else if (message.getType() == PacketType::SilentAudioFrame) { - mirroredType = PacketType::ReplicatedSilentAudioFrame; - } else { - return; - } - std::unique_ptr packet; // enumerate the downstream audio mixers and send them the replicated version of this packet From c1bbb2a084d4f4aa11764ee1f8b4471d85c8e3bc Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 11:24:05 -0700 Subject: [PATCH 043/115] fix incorrect UUID in replicated packets --- assignment-client/src/audio/AudioMixerClientData.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index e7b45b2d4e..3942c3177b 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -143,15 +143,15 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c std::unique_ptr packet; // enumerate the downstream audio mixers and send them the replicated version of this packet - nodeList->unsafeEachNode([&](const SharedNodePointer& node) { - if (node->getType() == NodeType::DownstreamAudioMixer) { + nodeList->unsafeEachNode([&](const SharedNodePointer& downstreamNode) { + if (downstreamNode->getType() == NodeType::DownstreamAudioMixer) { // construct the packet only once, if we have any downstream audio mixers to send to if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet packet = NLPacket::create(mirroredType); // since this packet will be non-sourced, we add the replicated node's ID here - packet->write(node->getUUID().toRfc4122()); + packet->write(node.getUUID().toRfc4122()); // we won't negotiate an audio format with the replicant, because we aren't a listener // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs @@ -160,7 +160,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c packet->write(message.getMessage()); } - nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); + nodeList->sendUnreliablePacket(*packet, downstreamNode->getPublicSocket()); } }); } From a877bf44fb5098e952de91f21975dddc7ad0cd1f Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Tue, 13 Jun 2017 11:26:40 -0700 Subject: [PATCH 044/115] CR --- assignment-client/src/octree/OctreeInboundPacketProcessor.cpp | 2 +- assignment-client/src/octree/OctreeInboundPacketProcessor.h | 2 +- libraries/networking/src/ReceivedPacketProcessor.h | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index d2fef4dfbd..04409b3b21 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -47,7 +47,7 @@ void OctreeInboundPacketProcessor::resetStats() { _singleSenderStats.clear(); } -unsigned long OctreeInboundPacketProcessor::getMaxWait() const { +uint32_t OctreeInboundPacketProcessor::getMaxWait() const { // calculate time until next sendNackPackets() quint64 nextNackTime = _lastNackTime + TOO_LONG_SINCE_LAST_NACK; quint64 now = usecTimestampNow(); diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 4611fcada0..a7fa297d24 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -80,7 +80,7 @@ protected: virtual void processPacket(QSharedPointer message, SharedNodePointer sendingNode) override; - virtual unsigned long getMaxWait() const override; + virtual uint32_t getMaxWait() const override; virtual void preProcess() override; virtual void midProcess() override; diff --git a/libraries/networking/src/ReceivedPacketProcessor.h b/libraries/networking/src/ReceivedPacketProcessor.h index 5b54d4f309..f71abce1f1 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.h +++ b/libraries/networking/src/ReceivedPacketProcessor.h @@ -20,7 +20,7 @@ class ReceivedPacketProcessor : public GenericThread { Q_OBJECT public: - static const unsigned long MAX_WAIT_TIME { 100 }; + static const uint64_t MAX_WAIT_TIME { 100 }; // Max wait time in ms ReceivedPacketProcessor(); @@ -66,7 +66,7 @@ protected: virtual bool process() override; /// Determines the timeout of the wait when there are no packets to process. Default value is 100ms to allow for regular event processing. - virtual unsigned long getMaxWait() const { return MAX_WAIT_TIME; } + virtual uint32_t getMaxWait() const { return MAX_WAIT_TIME; } /// Override to do work before the packets processing loop. Default does nothing. virtual void preProcess() { } From 3f3cc89b8d0a9408fc94fc2f0bb4e7ba978eae44 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 11:58:08 -0700 Subject: [PATCH 045/115] fix for header of re-replicated packets --- .../src/audio/AudioMixerClientData.cpp | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 3942c3177b..848dc969d8 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -109,6 +109,13 @@ void AudioMixerClientData::processPackets() { assert(_packetQueue.empty()); } +bool isReplicatedPacket(PacketType packetType) { + return packetType == PacketType::ReplicatedMicrophoneAudioNoEcho + || packetType == PacketType::ReplicatedMicrophoneAudioWithEcho + || packetType == PacketType::ReplicatedInjectAudio + || packetType == PacketType::ReplicatedSilentAudioFrame; +} + void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) { // first, make sure that this is a packet from a node we are supposed to replicate @@ -126,7 +133,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c break; case PacketType::MicrophoneAudioWithEcho: case PacketType::ReplicatedMicrophoneAudioWithEcho: - mirroredType = PacketType::ReplicatedMicrophoneAudioWithEcho; + mirroredType = PacketType::ReplicatedMicrophoneAudioWithEcho; break; case PacketType::InjectAudio: case PacketType::ReplicatedInjectAudio: @@ -150,12 +157,14 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c // construct an NLPacket to send to the replicant that has the contents of the received packet packet = NLPacket::create(mirroredType); - // since this packet will be non-sourced, we add the replicated node's ID here - packet->write(node.getUUID().toRfc4122()); + if (!isReplicatedPacket(message.getType())) { + // since this packet will be non-sourced, we add the replicated node's ID here + packet->write(node.getUUID().toRfc4122()); - // we won't negotiate an audio format with the replicant, because we aren't a listener - // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs - packet->writeString(_selectedCodecName); + // we won't negotiate an audio format with the replicant, because we aren't a listener + // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs + packet->writeString(_selectedCodecName); + } packet->write(message.getMessage()); } From ea56c568a76b60cea5b181ee3549cdfc60af9089 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Mon, 12 Jun 2017 16:55:37 -0700 Subject: [PATCH 046/115] Add support for replicating by username --- domain-server/src/DomainServer.cpp | 31 +++++++++++++++++-- domain-server/src/DomainServer.h | 6 ++++ .../src/DomainServerSettingsManager.cpp | 10 +++--- .../src/DomainServerSettingsManager.h | 1 + libraries/networking/src/Node.h | 1 + 5 files changed, 43 insertions(+), 6 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index de53898057..afe9b7ca50 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -117,6 +117,8 @@ DomainServer::DomainServer(int argc, char* argv[]) : // if permissions are updated, relay the changes to the Node datastructures connect(&_settingsManager, &DomainServerSettingsManager::updateNodePermissions, &_gatekeeper, &DomainGatekeeper::updateNodePermissions); + connect(&_settingsManager, &DomainServerSettingsManager::settingsUpdated, + this, &DomainServer::updateReplicatedNodes); setupGroupCacheRefresh(); @@ -2210,9 +2212,34 @@ void DomainServer::refreshStaticAssignmentAndAddToQueue(SharedAssignmentPointer& _unfulfilledAssignments.enqueue(assignment); } +void DomainServer::updateReplicatedNodes() { + static const QString REPLICATION_SETTINGS_KEY = "replication"; + _replicatedUsernames.clear(); + auto settings = _settingsManager.getSettingsMap(); + if (settings.contains(REPLICATION_SETTINGS_KEY)) { + auto replicationSettings = settings.value(REPLICATION_SETTINGS_KEY).toJsonObject(); + auto usersSettings = replicationSettings.value("users").toArray(); + for (auto& username : usersSettings) { + _replicatedUsernames.push_back(username.toString()); + } + } + + auto nodeList = DependencyManager::get(); + nodeList->eachNode([&](const SharedNodePointer& otherNode) { + if (shouldReplicateNode(*otherNode)) { + otherNode->setIsReplicated(true); + } + }); +} + +bool DomainServer::shouldReplicateNode(const Node& node) { + QString verifiedUsername = node.getPermissions().getVerifiedUserName(); + auto it = find(_replicatedUsernames.cbegin(), _replicatedUsernames.cend(), verifiedUsername); + return it != _replicatedUsernames.end() && node.getType() == NodeType::Agent; +}; + void DomainServer::nodeAdded(SharedNodePointer node) { - // TODO Check to see if node is in list of replicated nodes - if (node->getType() == NodeType::Agent) { + if (shouldReplicateNode(*node)) { node->setIsReplicated(true); } diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 63b82cb37d..3538f85f68 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -102,6 +102,8 @@ private slots: void handleOctreeFileReplacement(QByteArray octreeFile); + void updateReplicatedNodes(); + signals: void iceServerChanged(); void userConnected(); @@ -161,12 +163,16 @@ private: QJsonObject jsonForSocket(const HifiSockAddr& socket); QJsonObject jsonObjectForNode(const SharedNodePointer& node); + bool DomainServer::shouldReplicateNode(const Node& node); + void setupGroupCacheRefresh(); QString pathForRedirect(QString path = QString()) const; SubnetList _acSubnetWhitelist; + std::vector _replicatedUsernames; + DomainGatekeeper _gatekeeper; HTTPManager _httpManager; diff --git a/domain-server/src/DomainServerSettingsManager.cpp b/domain-server/src/DomainServerSettingsManager.cpp index d6b57b450a..3d256bd2b9 100644 --- a/domain-server/src/DomainServerSettingsManager.cpp +++ b/domain-server/src/DomainServerSettingsManager.cpp @@ -991,6 +991,7 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection unpackPermissions(); apiRefreshGroupInformation(); emit updateNodePermissions(); + emit settingsUpdated(); } return true; @@ -1196,13 +1197,14 @@ QJsonObject DomainServerSettingsManager::settingDescriptionFromGroup(const QJson bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject) { static const QString SECURITY_ROOT_KEY = "security"; static const QString AC_SUBNET_WHITELIST_KEY = "ac_subnet_whitelist"; + static const QString REPLICATION_KEY = "replication"; auto& settingsVariant = _configMap.getConfig(); bool needRestart = false; // Iterate on the setting groups foreach(const QString& rootKey, postedObject.keys()) { - QJsonValue rootValue = postedObject[rootKey]; + const QJsonValue& rootValue = postedObject[rootKey]; if (!settingsVariant.contains(rootKey)) { // we don't have a map below this key yet, so set it up now @@ -1247,7 +1249,7 @@ bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJ if (!matchingDescriptionObject.isEmpty()) { updateSetting(rootKey, rootValue, *thisMap, matchingDescriptionObject); - if (rootKey != SECURITY_ROOT_KEY) { + if (rootKey != SECURITY_ROOT_KEY && rootKey != REPLICATION_KEY) { needRestart = true; } } else { @@ -1261,9 +1263,9 @@ bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJ // if we matched the setting then update the value if (!matchingDescriptionObject.isEmpty()) { - QJsonValue settingValue = rootValue.toObject()[settingKey]; + const QJsonValue& settingValue = rootValue.toObject()[settingKey]; updateSetting(settingKey, settingValue, *thisMap, matchingDescriptionObject); - if (rootKey != SECURITY_ROOT_KEY || settingKey == AC_SUBNET_WHITELIST_KEY) { + if ((rootKey != SECURITY_ROOT_KEY && rootKey != REPLICATION_KEY) || settingKey == AC_SUBNET_WHITELIST_KEY) { needRestart = true; } } else { diff --git a/domain-server/src/DomainServerSettingsManager.h b/domain-server/src/DomainServerSettingsManager.h index d56a786d4b..4c7d8dfbc9 100644 --- a/domain-server/src/DomainServerSettingsManager.h +++ b/domain-server/src/DomainServerSettingsManager.h @@ -108,6 +108,7 @@ public: signals: void updateNodePermissions(); + void settingsUpdated(); public slots: void apiGetGroupIDJSONCallback(QNetworkReply& requestReply); diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index fd2cf6b65b..33c9e2c205 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -105,6 +105,7 @@ private: bool _isUpstream { false }; tbb::concurrent_unordered_set _ignoredNodeIDSet; mutable QReadWriteLock _ignoredNodeIDSetLock; + std::vector _replicatedUsernames { }; std::atomic_bool _ignoreRadiusEnabled; }; From 2e23230e82f53c76180e838b3a17bcaa9241f91f Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 08:36:33 -0700 Subject: [PATCH 047/115] Remove DomainServer qualification inside header --- domain-server/src/DomainServer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 3538f85f68..92df7a88e0 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -163,7 +163,7 @@ private: QJsonObject jsonForSocket(const HifiSockAddr& socket); QJsonObject jsonObjectForNode(const SharedNodePointer& node); - bool DomainServer::shouldReplicateNode(const Node& node); + bool shouldReplicateNode(const Node& node); void setupGroupCacheRefresh(); From 029e6de2a0720946e82986f3c58ee43330a88a6a Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 10:22:59 -0700 Subject: [PATCH 048/115] Fix replicated users not being checked after username verified --- domain-server/src/DomainServer.cpp | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index afe9b7ca50..65f85215cb 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -131,6 +131,8 @@ DomainServer::DomainServer(int argc, char* argv[]) : setupNodeListAndAssignments(); + updateReplicatedNodes(); + if (_type != NonMetaverse) { // if we have a metaverse domain, we'll use an access token for API calls resetAccountManagerAccessToken(); @@ -960,6 +962,11 @@ void DomainServer::handleConnectedNode(SharedNodePointer newNode) { emit userConnected(); } + if (shouldReplicateNode(*newNode)) { + qDebug() << "Setting node to replicated: " << newNode->getUUID(); + newNode->setIsReplicated(true); + } + // send out this node to our other connected nodes broadcastNewNode(newNode); } @@ -2217,16 +2224,19 @@ void DomainServer::updateReplicatedNodes() { _replicatedUsernames.clear(); auto settings = _settingsManager.getSettingsMap(); if (settings.contains(REPLICATION_SETTINGS_KEY)) { - auto replicationSettings = settings.value(REPLICATION_SETTINGS_KEY).toJsonObject(); - auto usersSettings = replicationSettings.value("users").toArray(); - for (auto& username : usersSettings) { - _replicatedUsernames.push_back(username.toString()); + auto replicationSettings = settings.value(REPLICATION_SETTINGS_KEY).toMap(); + if (replicationSettings.contains("users")) { + auto usersSettings = replicationSettings.value("users").toList(); + for (auto& username : usersSettings) { + _replicatedUsernames.push_back(username.toString()); + } } } auto nodeList = DependencyManager::get(); nodeList->eachNode([&](const SharedNodePointer& otherNode) { if (shouldReplicateNode(*otherNode)) { + qDebug() << "Setting node to replicated: " << otherNode->getUUID(); otherNode->setIsReplicated(true); } }); @@ -2234,15 +2244,12 @@ void DomainServer::updateReplicatedNodes() { bool DomainServer::shouldReplicateNode(const Node& node) { QString verifiedUsername = node.getPermissions().getVerifiedUserName(); + qDebug() << "Verified username: " << verifiedUsername; auto it = find(_replicatedUsernames.cbegin(), _replicatedUsernames.cend(), verifiedUsername); return it != _replicatedUsernames.end() && node.getType() == NodeType::Agent; }; void DomainServer::nodeAdded(SharedNodePointer node) { - if (shouldReplicateNode(*node)) { - node->setIsReplicated(true); - } - // we don't use updateNodeWithData, so add the DomainServerNodeData to the node here node->setLinkedData(std::unique_ptr { new DomainServerNodeData() }); } From d0e8612a65c7d2cf0ab8cc88c4859e21a56a611e Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 15:25:22 -0700 Subject: [PATCH 049/115] First pass at AvatarMixer replication --- assignment-client/src/avatars/AvatarMixer.cpp | 75 +++++++++++++++++++ assignment-client/src/avatars/AvatarMixer.h | 3 + .../src/avatars/AvatarMixerClientData.cpp | 4 + libraries/networking/src/LimitedNodeList.cpp | 4 + .../networking/src/udt/PacketHeaders.cpp | 5 +- libraries/networking/src/udt/PacketHeaders.h | 5 +- 6 files changed, 93 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 870149f1bc..f499a787cb 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -54,15 +54,79 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : packetReceiver.registerListener(PacketType::RadiusIgnoreRequest, this, "handleRadiusIgnoreRequestPacket"); packetReceiver.registerListener(PacketType::RequestsDomainListData, this, "handleRequestsDomainListDataPacket"); + packetReceiver.registerListenerForTypes({ + PacketType::ReplicatedAvatarIdentity, + PacketType::ReplicatedAvatarData, + PacketType::ReplicatedKillAvatar + }, this, "handleReplicatedPackets"); + auto nodeList = DependencyManager::get(); connect(nodeList.data(), &NodeList::packetVersionMismatch, this, &AvatarMixer::handlePacketVersionMismatch); } +void AvatarMixer::handleReplicatedPackets(QSharedPointer message) { + auto nodeList = DependencyManager::get(); + auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + auto replicatedNode = nodeList->nodeWithUUID(nodeID); + if (!replicatedNode) { + QMetaObject::invokeMethod(nodeList.data(), "addOrUpdateNode", Qt::BlockingQueuedConnection, + Q_RETURN_ARG(SharedNodePointer, replicatedNode), + Q_ARG(QUuid, nodeID), Q_ARG(NodeType_t, NodeType::Agent), + Q_ARG(HifiSockAddr, message->getSenderSockAddr()), + Q_ARG(HifiSockAddr, message->getSenderSockAddr())); + } + + replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); + replicatedNode->setIsUpstream(true); + + switch (message->getType()) { + case PacketType::AvatarData: + queueIncomingPacket(message, replicatedNode); + break; + case PacketType::AvatarIdentity: + handleAvatarIdentityPacket(message, replicatedNode); + break; + case PacketType::KillAvatar: + handleKillAvatarPacket(message); + break + } + +} + +void AvatarMixer::replicatePacket(ReceivedMessage& message) { + PacketType replicatedType; + if (message.getType() == PacketType::AvatarData) { + replicatedType = PacketType::ReplicatedAvatarData; + } else if (message.getType() == PacketType::AvatarIdentity) { + replicatedType = PacketType::ReplicatedAvatarIdentity; + } else if (message.getType() == PacketType::KillAvatar) { + replicatedType = PacketType::ReplicatedKillAvatar; + } else { + Q_UNREACHABLE(); + return; + } + + auto nodeList = DependencyManager::get(); + nodeList->eachMatchingNode([&](const SharedNodePointer& node) { + return node->getType() == NodeType::ReplicantAvatarMixer; + }, [&](const SharedNodePointer& node) { + // construct an NLPacket to send to the replicant that has the contents of the received packet + auto packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); + packet->write(message.getSourceID().toRfc4122()); + packet->write(message.getMessage()); + + nodeList->sendPacket(std::move(packet), *node); + }); +} + void AvatarMixer::queueIncomingPacket(QSharedPointer message, SharedNodePointer node) { auto start = usecTimestampNow(); getOrCreateClientData(node)->queuePacket(message, node); auto end = usecTimestampNow(); _queueIncomingPacketElapsedTime += (end - start); + + replicatePacket(*message); } @@ -397,6 +461,13 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes if (nodeData != nullptr) { AvatarData& avatar = nodeData->getAvatar(); + auto data = message->getMessage(); + + if (message->getType() == PacketType::ReplicatedAvatarData) { + data = data.mid(NUM_BYTES_RFC4122_UUID); + } + + // parse the identity packet and update the change timestamp if appropriate AvatarData::Identity identity; AvatarData::parseAvatarIdentityPacket(message->getMessage(), identity); @@ -414,6 +485,8 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes } auto end = usecTimestampNow(); _handleAvatarIdentityPacketElapsedTime += (end - start); + + replicatePacket(*message); } void AvatarMixer::handleKillAvatarPacket(QSharedPointer message) { @@ -421,6 +494,8 @@ void AvatarMixer::handleKillAvatarPacket(QSharedPointer message DependencyManager::get()->processKillNode(*message); auto end = usecTimestampNow(); _handleKillAvatarPacketElapsedTime += (end - start); + + replicatePacket(*message); } void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 1925ec1ebd..ecbed4dca7 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -46,6 +46,7 @@ private slots: void handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode); void handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); + void handleReplicatedPackets(QSharedPointer message); void domainSettingsRequestComplete(); void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID); void start(); @@ -61,6 +62,8 @@ private: void manageDisplayName(const SharedNodePointer& node); + void replicatePacket(ReceivedMessage& message); + p_high_resolution_clock::time_point _lastFrameTimestamp; // FIXME - new throttling - use these values somehow diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 4d80bc7d17..41717adeb5 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -62,6 +62,10 @@ int AvatarMixerClientData::parseData(ReceivedMessage& message) { // pull the sequence number from the data first uint16_t sequenceNumber; + if (message.getType() == PacketType::ReplicatedAvatarData) { + message.seek(NUM_BYTES_RFC4122_UUID); + } + message.readPrimitive(&sequenceNumber); if (sequenceNumber < _lastReceivedSequenceNumber && _lastReceivedSequenceNumber != UINT16_MAX) { diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 0494efc7b4..f28a972720 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -549,6 +549,10 @@ bool LimitedNodeList::killNodeWithUUID(const QUuid& nodeUUID) { } void LimitedNodeList::processKillNode(ReceivedMessage& message) { + if (message.getType() == PacketType::ReplicatedAvatarData) { + message.seek(NUM_BYTES_RFC4122_UUID); + } + // read the node id QUuid nodeUUID = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index f2f0062861..9f213d29ff 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -41,7 +41,8 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ICEServerHeartbeatDenied << PacketType::AssignmentClientStatus << PacketType::StopNode << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement << PacketType::ReplicatedMicrophoneAudioNoEcho << PacketType::ReplicatedMicrophoneAudioWithEcho - << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame; + << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame + << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedAvatarData << PacketType::ReplicatedKillAvatar; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { @@ -121,7 +122,7 @@ static void ensureProtocolVersionsSignature() { std::call_once(once, [&] { QByteArray buffer; QDataStream stream(&buffer, QIODevice::WriteOnly); - uint8_t numberOfProtocols = static_cast(PacketType::LAST_PACKET_TYPE) + 1; + uint8_t numberOfProtocols = static_cast(PacketType::NUM_PACKET_TYPE); stream << numberOfProtocols; for (uint8_t packetType = 0; packetType < numberOfProtocols; packetType++) { uint8_t packetTypeVersion = static_cast(versionForPacketType(static_cast(packetType))); diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 448ae812ff..137108ef75 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -119,8 +119,11 @@ public: ReplicatedMicrophoneAudioWithEcho, ReplicatedInjectAudio, ReplicatedSilentAudioFrame, - LAST_PACKET_TYPE = ReplicatedSilentAudioFrame, + ReplicatedAvatarIdentity, + ReplicatedAvatarData, + ReplicatedKillAvatar, + NUM_PACKET_TYPE }; }; From a0d107c72c961eb1c5d275cc391afdb08da55237 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 15:45:12 -0700 Subject: [PATCH 050/115] Only replicate replicated nodes --- assignment-client/src/avatars/AvatarMixer.cpp | 69 +++++++++++-------- assignment-client/src/avatars/AvatarMixer.h | 4 +- 2 files changed, 42 insertions(+), 31 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index f499a787cb..2b074f71e6 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -88,36 +88,47 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag handleAvatarIdentityPacket(message, replicatedNode); break; case PacketType::KillAvatar: - handleKillAvatarPacket(message); - break + handleKillAvatarPacket(message, replicatedNode); + break; + default: + // Do nothing + break; } } -void AvatarMixer::replicatePacket(ReceivedMessage& message) { - PacketType replicatedType; - if (message.getType() == PacketType::AvatarData) { - replicatedType = PacketType::ReplicatedAvatarData; - } else if (message.getType() == PacketType::AvatarIdentity) { - replicatedType = PacketType::ReplicatedAvatarIdentity; - } else if (message.getType() == PacketType::KillAvatar) { - replicatedType = PacketType::ReplicatedKillAvatar; - } else { - Q_UNREACHABLE(); - return; +void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) { + // first, make sure that this is a packet from a node we are supposed to replicate + if (node.isReplicated()) { + // now make sure it's a packet type that we want to replicate + PacketType replicatedType; + switch (message.getType()) { + case PacketType::AvatarData: + replicatedType = PacketType::ReplicatedAvatarData; + break; + case PacketType::AvatarIdentity: + replicatedType = PacketType::ReplicatedAvatarIdentity; + break; + case PacketType::KillAvatar: + replicatedType = PacketType::ReplicatedKillAvatar; + break; + default: + Q_UNREACHABLE(); + return; + } + + auto nodeList = DependencyManager::get(); + nodeList->eachMatchingNode([&](const SharedNodePointer& node) { + return node->getType() == NodeType::ReplicantAvatarMixer; + }, [&](const SharedNodePointer& node) { + // construct an NLPacket to send to the replicant that has the contents of the received packet + auto packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); + packet->write(message.getSourceID().toRfc4122()); + packet->write(message.getMessage()); + + nodeList->sendPacket(std::move(packet), *node); + }); } - - auto nodeList = DependencyManager::get(); - nodeList->eachMatchingNode([&](const SharedNodePointer& node) { - return node->getType() == NodeType::ReplicantAvatarMixer; - }, [&](const SharedNodePointer& node) { - // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); - packet->write(message.getSourceID().toRfc4122()); - packet->write(message.getMessage()); - - nodeList->sendPacket(std::move(packet), *node); - }); } void AvatarMixer::queueIncomingPacket(QSharedPointer message, SharedNodePointer node) { @@ -126,7 +137,7 @@ void AvatarMixer::queueIncomingPacket(QSharedPointer message, S auto end = usecTimestampNow(); _queueIncomingPacketElapsedTime += (end - start); - replicatePacket(*message); + optionallyReplicatePacket(*message, *node); } @@ -486,16 +497,16 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes auto end = usecTimestampNow(); _handleAvatarIdentityPacketElapsedTime += (end - start); - replicatePacket(*message); + optionallyReplicatePacket(*message, *senderNode); } -void AvatarMixer::handleKillAvatarPacket(QSharedPointer message) { +void AvatarMixer::handleKillAvatarPacket(QSharedPointer message, SharedNodePointer node) { auto start = usecTimestampNow(); DependencyManager::get()->processKillNode(*message); auto end = usecTimestampNow(); _handleKillAvatarPacketElapsedTime += (end - start); - replicatePacket(*message); + optionallyReplicatePacket(*message, *node); } void AvatarMixer::handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode) { diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index ecbed4dca7..17a6db3b08 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -42,7 +42,7 @@ private slots: void handleAdjustAvatarSorting(QSharedPointer message, SharedNodePointer senderNode); void handleViewFrustumPacket(QSharedPointer message, SharedNodePointer senderNode); void handleAvatarIdentityPacket(QSharedPointer message, SharedNodePointer senderNode); - void handleKillAvatarPacket(QSharedPointer message); + void handleKillAvatarPacket(QSharedPointer message, SharedNodePointer senderNode); void handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode); void handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); @@ -62,7 +62,7 @@ private: void manageDisplayName(const SharedNodePointer& node); - void replicatePacket(ReceivedMessage& message); + void optionallyReplicatePacket(ReceivedMessage& message, const Node& node); p_high_resolution_clock::time_point _lastFrameTimestamp; From b4ce9fb4fcb732980fcce9d1d806aed1b8ba33ca Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 16:12:13 -0700 Subject: [PATCH 051/115] Remove blocking call + reduce packet construction --- assignment-client/src/avatars/AvatarMixer.cpp | 24 +++++++++---------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 2b074f71e6..75b3d23563 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -68,14 +68,8 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag auto nodeList = DependencyManager::get(); auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); - auto replicatedNode = nodeList->nodeWithUUID(nodeID); - if (!replicatedNode) { - QMetaObject::invokeMethod(nodeList.data(), "addOrUpdateNode", Qt::BlockingQueuedConnection, - Q_RETURN_ARG(SharedNodePointer, replicatedNode), - Q_ARG(QUuid, nodeID), Q_ARG(NodeType_t, NodeType::Agent), - Q_ARG(HifiSockAddr, message->getSenderSockAddr()), - Q_ARG(HifiSockAddr, message->getSenderSockAddr())); - } + auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, + message->getSenderSockAddr(), message->getSenderSockAddr()) replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); @@ -117,16 +111,20 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node return; } + std::unique_ptr packet; + auto nodeList = DependencyManager::get(); nodeList->eachMatchingNode([&](const SharedNodePointer& node) { return node->getType() == NodeType::ReplicantAvatarMixer; }, [&](const SharedNodePointer& node) { - // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); - packet->write(message.getSourceID().toRfc4122()); - packet->write(message.getMessage()); + if (!packet) { + // construct an NLPacket to send to the replicant that has the contents of the received packet + packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); + packet->write(message.getSourceID().toRfc4122()); + packet->write(message.getMessage()); + } - nodeList->sendPacket(std::move(packet), *node); + nodeList->sendUnreliablePacket(*packet, *node); }); } } From 9aebf68664ca4d1598e2c0df815f4bb3357497a2 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 16:32:54 -0700 Subject: [PATCH 052/115] Parse replicant from domain settings --- assignment-client/src/avatars/AvatarMixer.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 75b3d23563..aae447be60 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -69,7 +69,8 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, - message->getSenderSockAddr(), message->getSenderSockAddr()) + message->getSenderSockAddr(), message->getSenderSockAddr(), + DEFAULT_AGENT_PERMISSIONS, true); replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); @@ -115,7 +116,7 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node auto nodeList = DependencyManager::get(); nodeList->eachMatchingNode([&](const SharedNodePointer& node) { - return node->getType() == NodeType::ReplicantAvatarMixer; + return node->getType() == NodeType::DownstreamAvatarMixer; }, [&](const SharedNodePointer& node) { if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet @@ -848,4 +849,6 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { qCDebug(avatars) << "This domain requires a minimum avatar scale of" << _domainMinimumScale << "and a maximum avatar scale of" << _domainMaximumScale; + + parseDownstreamServers(domainSettings, NodeType::AvatarMixer); } From 5e3479560354cbfb374bcbdf94516af8a5a7bb68 Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Mon, 12 Jun 2017 18:38:23 -0700 Subject: [PATCH 053/115] Fix packet types filtering --- assignment-client/src/avatars/AvatarMixer.cpp | 9 ++++----- assignment-client/src/avatars/AvatarMixerClientData.cpp | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index aae447be60..ab51e00e74 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -76,13 +76,13 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag replicatedNode->setIsUpstream(true); switch (message->getType()) { - case PacketType::AvatarData: + case PacketType::ReplicatedAvatarData: queueIncomingPacket(message, replicatedNode); break; - case PacketType::AvatarIdentity: + case PacketType::ReplicatedAvatarIdentity: handleAvatarIdentityPacket(message, replicatedNode); break; - case PacketType::KillAvatar: + case PacketType::ReplicatedKillAvatar: handleKillAvatarPacket(message, replicatedNode); break; default: @@ -108,7 +108,6 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node replicatedType = PacketType::ReplicatedKillAvatar; break; default: - Q_UNREACHABLE(); return; } @@ -125,7 +124,7 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node packet->write(message.getMessage()); } - nodeList->sendUnreliablePacket(*packet, *node); + nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); }); } } diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 41717adeb5..3c9edcd4a5 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -45,6 +45,7 @@ int AvatarMixerClientData::processPackets() { switch (packet->getType()) { case PacketType::AvatarData: + case PacketType::ReplicatedAvatarData: parseData(*packet); break; default: From 44a63ca27ebac8663b0009738e1f89b84e90f8a1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 10:44:28 -0700 Subject: [PATCH 054/115] handle node killed and remove double IDs in kill packets --- assignment-client/src/avatars/AvatarMixer.cpp | 48 ++++++++++++++++--- libraries/networking/src/LimitedNodeList.cpp | 2 +- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index ab51e00e74..17cdf8b7d6 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -119,8 +119,18 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node }, [&](const SharedNodePointer& node) { if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet - packet = NLPacket::create(replicatedType, message.getSize() + NUM_BYTES_RFC4122_UUID); - packet->write(message.getSourceID().toRfc4122()); + auto packetSize = message.getSize(); + + if (replicatedType != PacketType::ReplicatedKillAvatar) { + packetSize += NUM_BYTES_RFC4122_UUID; + } + + packet = NLPacket::create(replicatedType, packetSize); + + if (replicatedType != PacketType::ReplicatedKillAvatar) { + packet->write(message.getSourceID().toRfc4122()); + } + packet->write(message.getMessage()); } @@ -352,13 +362,38 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { } } + std::unique_ptr killPacket; + std::unique_ptr replicatedKillPacket; + // this was an avatar we were sending to other people // send a kill packet for it to our other nodes - auto killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID + sizeof(KillAvatarReason)); - killPacket->write(killedNode->getUUID().toRfc4122()); - killPacket->writePrimitive(KillAvatarReason::AvatarDisconnected); + nodeList->eachMatchingNode([&](const SharedNodePointer& node) { + // we relay avatar kill packets to agents that are not upstream + // and downstream avatar mixers, if the node that was just killed was being replicated + return (node->getType() == NodeType::Agent && !node->isUpstream()) + || (killedNode->isReplicated() && node->getType() == NodeType::DownstreamAvatarMixer); + }, [&](const SharedNodePointer& node) { + if (node->getType() == NodeType::Agent) { + if (!killPacket) { + killPacket = NLPacket::create(PacketType::KillAvatar, NUM_BYTES_RFC4122_UUID + sizeof(KillAvatarReason)); + killPacket->write(killedNode->getUUID().toRfc4122()); + killPacket->writePrimitive(KillAvatarReason::AvatarDisconnected); + } + + nodeList->sendUnreliablePacket(*killPacket, *node); + } else { + // send a replicated kill packet to the downstream avatar mixer + if (!replicatedKillPacket) { + replicatedKillPacket = NLPacket::create(PacketType::ReplicatedKillAvatar, + NUM_BYTES_RFC4122_UUID + sizeof(KillAvatarReason)); + replicatedKillPacket->write(killedNode->getUUID().toRfc4122()); + replicatedKillPacket->writePrimitive(KillAvatarReason::AvatarDisconnected); + } + + nodeList->sendUnreliablePacket(*replicatedKillPacket, node->getPublicSocket()); + } + }); - nodeList->broadcastToNodes(std::move(killPacket), NodeSet() << NodeType::Agent); // we also want to remove sequence number data for this avatar on our other avatars // so invoke the appropriate method on the AvatarMixerClientData for other avatars @@ -756,7 +791,6 @@ void AvatarMixer::run() { connect(&domainHandler, &DomainHandler::settingsReceiveFail, this, &AvatarMixer::domainSettingsRequestFailed); ThreadedAssignment::commonInit(AVATAR_MIXER_LOGGING_NAME, NodeType::AvatarMixer); - } AvatarMixerClientData* AvatarMixer::getOrCreateClientData(SharedNodePointer node) { diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index f28a972720..b39c86e6e3 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -550,7 +550,7 @@ bool LimitedNodeList::killNodeWithUUID(const QUuid& nodeUUID) { void LimitedNodeList::processKillNode(ReceivedMessage& message) { if (message.getType() == PacketType::ReplicatedAvatarData) { - message.seek(NUM_BYTES_RFC4122_UUID); + message.seek(0); } // read the node id From 8ce6590f0b807f24bac9f920879499866f47348c Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 10:56:47 -0700 Subject: [PATCH 055/115] remove ReplicatedAvatarData and immediate replication of identity --- assignment-client/src/avatars/AvatarMixer.cpp | 48 +++---------------- libraries/networking/src/LimitedNodeList.cpp | 4 -- .../networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 1 - 4 files changed, 8 insertions(+), 47 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 17cdf8b7d6..34fcc3c0ed 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -56,7 +56,6 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : packetReceiver.registerListenerForTypes({ PacketType::ReplicatedAvatarIdentity, - PacketType::ReplicatedAvatarData, PacketType::ReplicatedKillAvatar }, this, "handleReplicatedPackets"); @@ -76,13 +75,13 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag replicatedNode->setIsUpstream(true); switch (message->getType()) { - case PacketType::ReplicatedAvatarData: - queueIncomingPacket(message, replicatedNode); - break; case PacketType::ReplicatedAvatarIdentity: handleAvatarIdentityPacket(message, replicatedNode); break; case PacketType::ReplicatedKillAvatar: + // seek back in the message so the kill packet handler can start by reading the source ID + message->seek(0); + handleKillAvatarPacket(message, replicatedNode); break; default: @@ -94,22 +93,9 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) { // first, make sure that this is a packet from a node we are supposed to replicate - if (node.isReplicated()) { - // now make sure it's a packet type that we want to replicate - PacketType replicatedType; - switch (message.getType()) { - case PacketType::AvatarData: - replicatedType = PacketType::ReplicatedAvatarData; - break; - case PacketType::AvatarIdentity: - replicatedType = PacketType::ReplicatedAvatarIdentity; - break; - case PacketType::KillAvatar: - replicatedType = PacketType::ReplicatedKillAvatar; - break; - default: - return; - } + if (node.isReplicated() + && (message.getType() == PacketType::KillAvatar || message.getType() == PacketType::ReplicatedKillAvatar)) { + PacketType replicatedType = PacketType::ReplicatedKillAvatar; std::unique_ptr packet; @@ -119,18 +105,7 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node }, [&](const SharedNodePointer& node) { if (!packet) { // construct an NLPacket to send to the replicant that has the contents of the received packet - auto packetSize = message.getSize(); - - if (replicatedType != PacketType::ReplicatedKillAvatar) { - packetSize += NUM_BYTES_RFC4122_UUID; - } - - packet = NLPacket::create(replicatedType, packetSize); - - if (replicatedType != PacketType::ReplicatedKillAvatar) { - packet->write(message.getSourceID().toRfc4122()); - } - + packet = NLPacket::create(replicatedType, message.getSize()); packet->write(message.getMessage()); } @@ -144,8 +119,6 @@ void AvatarMixer::queueIncomingPacket(QSharedPointer message, S getOrCreateClientData(node)->queuePacket(message, node); auto end = usecTimestampNow(); _queueIncomingPacketElapsedTime += (end - start); - - optionallyReplicatePacket(*message, *node); } @@ -505,13 +478,6 @@ void AvatarMixer::handleAvatarIdentityPacket(QSharedPointer mes if (nodeData != nullptr) { AvatarData& avatar = nodeData->getAvatar(); - auto data = message->getMessage(); - - if (message->getType() == PacketType::ReplicatedAvatarData) { - data = data.mid(NUM_BYTES_RFC4122_UUID); - } - - // parse the identity packet and update the change timestamp if appropriate AvatarData::Identity identity; AvatarData::parseAvatarIdentityPacket(message->getMessage(), identity); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index b39c86e6e3..0494efc7b4 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -549,10 +549,6 @@ bool LimitedNodeList::killNodeWithUUID(const QUuid& nodeUUID) { } void LimitedNodeList::processKillNode(ReceivedMessage& message) { - if (message.getType() == PacketType::ReplicatedAvatarData) { - message.seek(0); - } - // read the node id QUuid nodeUUID = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 9f213d29ff..ad30a184ca 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -42,7 +42,7 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement << PacketType::ReplicatedMicrophoneAudioNoEcho << PacketType::ReplicatedMicrophoneAudioWithEcho << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame - << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedAvatarData << PacketType::ReplicatedKillAvatar; + << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedKillAvatar; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 137108ef75..b7d55c3266 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -120,7 +120,6 @@ public: ReplicatedInjectAudio, ReplicatedSilentAudioFrame, ReplicatedAvatarIdentity, - ReplicatedAvatarData, ReplicatedKillAvatar, NUM_PACKET_TYPE From a8ea8724d39e42e143c4944aba71c1af43138c1f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 12:39:47 -0700 Subject: [PATCH 056/115] split broadcast of avatar data for agent and downstream mixer --- .../src/avatars/AvatarMixerSlave.cpp | 631 +++++++++--------- .../src/avatars/AvatarMixerSlave.h | 3 + 2 files changed, 325 insertions(+), 309 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 2ad8bb58ed..7732058f2d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -81,6 +81,20 @@ static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 start = usecTimestampNow(); + if (node->getLinkedData()) { + if (node->getType() == NodeType::Agent && node->getActiveSocket() && !node->isUpstream()) { + broadcastAvatarDataToAgent(node); + } else if (node->getType() == NodeType::DownstreamAvatarMixer) { + broadcastAvatarDataToDownstreamMixer(node); + } + } + + quint64 end = usecTimestampNow(); + _stats.jobElapsedTime += (end - start); +} + +void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) { + auto nodeList = DependencyManager::get(); // setup for distributed random floating point values @@ -88,331 +102,330 @@ void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { std::mt19937 generator(randomDevice()); std::uniform_real_distribution distribution; - if (node->getLinkedData() && (node->getType() == NodeType::Agent) && node->getActiveSocket()) { - _stats.nodesBroadcastedTo++; + _stats.nodesBroadcastedTo++; - AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); - nodeData->resetInViewStats(); + nodeData->resetInViewStats(); - const AvatarData& avatar = nodeData->getAvatar(); - glm::vec3 myPosition = avatar.getClientGlobalPosition(); + const AvatarData& avatar = nodeData->getAvatar(); + glm::vec3 myPosition = avatar.getClientGlobalPosition(); - // reset the internal state for correct random number distribution - distribution.reset(); + // reset the internal state for correct random number distribution + distribution.reset(); - // reset the number of sent avatars - nodeData->resetNumAvatarsSentLastFrame(); + // reset the number of sent avatars + nodeData->resetNumAvatarsSentLastFrame(); - // keep a counter of the number of considered avatars - int numOtherAvatars = 0; + // keep a counter of the number of considered avatars + int numOtherAvatars = 0; - // keep track of outbound data rate specifically for avatar data - int numAvatarDataBytes = 0; - int identityBytesSent = 0; + // keep track of outbound data rate specifically for avatar data + int numAvatarDataBytes = 0; + int identityBytesSent = 0; - // max number of avatarBytes per frame - auto maxAvatarBytesPerFrame = (_maxKbpsPerNode * BYTES_PER_KILOBIT) / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND; + // max number of avatarBytes per frame + auto maxAvatarBytesPerFrame = (_maxKbpsPerNode * BYTES_PER_KILOBIT) / AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND; - // FIXME - find a way to not send the sessionID for every avatar - int minimumBytesPerAvatar = AvatarDataPacket::AVATAR_HAS_FLAGS_SIZE + NUM_BYTES_RFC4122_UUID; + // FIXME - find a way to not send the sessionID for every avatar + int minimumBytesPerAvatar = AvatarDataPacket::AVATAR_HAS_FLAGS_SIZE + NUM_BYTES_RFC4122_UUID; - int overBudgetAvatars = 0; + int overBudgetAvatars = 0; - // keep track of the number of other avatars held back in this frame - int numAvatarsHeldBack = 0; + // keep track of the number of other avatars held back in this frame + int numAvatarsHeldBack = 0; - // keep track of the number of other avatar frames skipped - int numAvatarsWithSkippedFrames = 0; + // keep track of the number of other avatar frames skipped + int numAvatarsWithSkippedFrames = 0; - // When this is true, the AvatarMixer will send Avatar data to a client - // about avatars they've ignored or that are out of view - bool PALIsOpen = nodeData->getRequestsDomainListData(); + // When this is true, the AvatarMixer will send Avatar data to a client + // about avatars they've ignored or that are out of view + bool PALIsOpen = nodeData->getRequestsDomainListData(); - // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them - bool getsAnyIgnored = PALIsOpen && node->getCanKick(); + // When this is true, the AvatarMixer will send Avatar data to a client about avatars that have ignored them + bool getsAnyIgnored = PALIsOpen && node->getCanKick(); - if (PALIsOpen) { - // Increase minimumBytesPerAvatar if the PAL is open - minimumBytesPerAvatar += sizeof(AvatarDataPacket::AvatarGlobalPosition) + - sizeof(AvatarDataPacket::AudioLoudness); - } - - // setup a PacketList for the avatarPackets - auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); - - // Define the minimum bubble size - static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); - // Define the scale of the box for the current node - glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Set up the bounding box for the current node - AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { - nodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - nodeBox.embiggen(4.0f); - - - // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes - // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes - QList avatarList; - std::unordered_map avatarDataToNodes; - - std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - - // theoretically it's possible for a Node to be in the NodeList (and therefore end up here), - // but not have yet sent data that's linked to the node. Check for that case and don't - // consider those nodes. - if (otherNodeData) { - AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); - avatarList << otherAvatar; - avatarDataToNodes[otherAvatar] = otherNode; - } - }); - - AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); - ViewFrustum cameraView = nodeData->getViewFrustom(); - std::priority_queue sortedAvatars; - AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, - - [&](AvatarSharedPointer avatar)->uint64_t{ - auto avatarNode = avatarDataToNodes[avatar]; - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - return nodeData->getLastBroadcastTime(avatarNode->getUUID()); - }, - - [&](AvatarSharedPointer avatar)->float{ - glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); - return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); - }, - - [&](AvatarSharedPointer avatar)->bool{ - if (avatar == thisAvatar) { - return true; // ignore ourselves... - } - - bool shouldIgnore = false; - - // We will also ignore other nodes for a couple of different reasons: - // 1) ignore bubbles and ignore specific node - // 2) the node hasn't really updated it's frame data recently, this can - // happen if for example the avatar is connected on a desktop and sending - // updates at ~30hz. So every 3 frames we skip a frame. - auto avatarNode = avatarDataToNodes[avatar]; - - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - - const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); - assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data - quint64 startIgnoreCalculation = usecTimestampNow(); - - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!avatarNode->getLinkedData() - || avatarNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) - || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - shouldIgnore = true; - } else { - - // Check to see if the space bubble is enabled - // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored - if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { - - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Set up the bounding box for the current other node - AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - otherNodeBox.embiggen(4.0f); - - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, avatarNode); - shouldIgnore = !getsAnyIgnored; - } - } - // Not close enough to ignore - if (!shouldIgnore) { - nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); - } - } - quint64 endIgnoreCalculation = usecTimestampNow(); - _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); - - if (!shouldIgnore) { - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); - - // FIXME - This code does appear to be working. But it seems brittle. - // It supports determining if the frame of data for this "other" - // avatar has already been sent to the reciever. This has been - // verified to work on a desktop display that renders at 60hz and - // therefore sends to mixer at 30hz. Each second you'd expect to - // have 15 (45hz-30hz) duplicate frames. In this case, the stat - // avg_other_av_skips_per_second does report 15. - // - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - shouldIgnore = true; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } - } - return shouldIgnore; - }); - - // loop through our sorted avatars and allocate our bandwidth to them accordingly - int avatarRank = 0; - - // this is overly conservative, because it includes some avatars we might not consider - int remainingAvatars = (int)sortedAvatars.size(); - - while (!sortedAvatars.empty()) { - AvatarPriority sortData = sortedAvatars.top(); - sortedAvatars.pop(); - const auto& avatarData = sortData.avatar; - avatarRank++; - remainingAvatars--; - - auto otherNode = avatarDataToNodes[avatarData]; - assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map - - // NOTE: Here's where we determine if we are over budget and drop to bare minimum data - int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; - bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; - - quint64 startAvatarDataPacking = usecTimestampNow(); - - ++numOtherAvatars; - - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - - // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO - // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. - if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { - identityBytesSent += sendIdentityPacket(otherNodeData, node); - } - - const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); - glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); - - // determine if avatar is in view, to determine how much data to include... - glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool isInView = nodeData->otherAvatarInView(otherNodeBox); - - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); - - AvatarData::AvatarDataDetail detail; - - if (overBudget) { - overBudgetAvatars++; - _stats.overBudgetAvatars++; - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; - } else if (!isInView) { - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; - nodeData->incrementAvatarOutOfView(); - } else { - detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO - ? AvatarData::SendAllData : AvatarData::CullSmallData; - nodeData->incrementAvatarInView(); - } - - bool includeThisAvatar = true; - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray - bool dropFaceTracking = false; - - quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - quint64 end = usecTimestampNow(); - _stats.toByteArrayElapsedTime += (end - start); - - static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; - - dropFaceTracking = true; // first try dropping the facial data - bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; - bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - } - - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; - includeThisAvatar = false; - } - } - - if (includeThisAvatar) { - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->write(bytes); - - if (detail != AvatarData::NoData) { - _stats.numOthersIncluded++; - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // remember the last time we sent details about this other node to the receiver - nodeData->setLastBroadcastTime(otherNode->getUUID(), start); - } - } - - avatarPacketList->endSegment(); - - quint64 endAvatarDataPacking = usecTimestampNow(); - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); - }; - - quint64 startPacketSending = usecTimestampNow(); - - // close the current packet so that we're always sending something - avatarPacketList->closeCurrentPacket(true); - - _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); - _stats.numBytesSent += numAvatarDataBytes; - - // send the avatar data PacketList - nodeList->sendPacketList(std::move(avatarPacketList), *node); - - // record the bytes sent for other avatar data in the AvatarMixerClientData - nodeData->recordSentAvatarData(numAvatarDataBytes); - - // record the number of avatars held back this frame - nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); - nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); - - quint64 endPacketSending = usecTimestampNow(); - _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); + if (PALIsOpen) { + // Increase minimumBytesPerAvatar if the PAL is open + minimumBytesPerAvatar += sizeof(AvatarDataPacket::AvatarGlobalPosition) + + sizeof(AvatarDataPacket::AudioLoudness); } - quint64 end = usecTimestampNow(); - _stats.jobElapsedTime += (end - start); + // setup a PacketList for the avatarPackets + auto avatarPacketList = NLPacketList::create(PacketType::BulkAvatarData); + + // Define the minimum bubble size + static const glm::vec3 minBubbleSize = glm::vec3(0.3f, 1.3f, 0.3f); + // Define the scale of the box for the current node + glm::vec3 nodeBoxScale = (nodeData->getPosition() - nodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current node + AABox nodeBox(nodeData->getGlobalBoundingBoxCorner(), nodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(nodeBoxScale, minBubbleSize))) { + nodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + nodeBox.embiggen(4.0f); + + + // setup list of AvatarData as well as maps to map betweeen the AvatarData and the original nodes + // for calling the AvatarData::sortAvatars() function and getting our sorted list of client nodes + QList avatarList; + std::unordered_map avatarDataToNodes; + + std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + + // theoretically it's possible for a Node to be in the NodeList (and therefore end up here), + // but not have yet sent data that's linked to the node. Check for that case and don't + // consider those nodes. + if (otherNodeData) { + AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); + avatarList << otherAvatar; + avatarDataToNodes[otherAvatar] = otherNode; + } + }); + + AvatarSharedPointer thisAvatar = nodeData->getAvatarSharedPointer(); + ViewFrustum cameraView = nodeData->getViewFrustom(); + std::priority_queue sortedAvatars; + AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, + + [&](AvatarSharedPointer avatar)->uint64_t{ + auto avatarNode = avatarDataToNodes[avatar]; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + return nodeData->getLastBroadcastTime(avatarNode->getUUID()); + }, + + [&](AvatarSharedPointer avatar)->float{ + glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); + return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); + }, + + [&](AvatarSharedPointer avatar)->bool{ + if (avatar == thisAvatar) { + return true; // ignore ourselves... + } + + bool shouldIgnore = false; + + // We will also ignore other nodes for a couple of different reasons: + // 1) ignore bubbles and ignore specific node + // 2) the node hasn't really updated it's frame data recently, this can + // happen if for example the avatar is connected on a desktop and sending + // updates at ~30hz. So every 3 frames we skip a frame. + auto avatarNode = avatarDataToNodes[avatar]; + + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + + const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); + assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data + quint64 startIgnoreCalculation = usecTimestampNow(); + + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!avatarNode->getLinkedData() + || avatarNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) + || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + shouldIgnore = true; + } else { + + // Check to see if the space bubble is enabled + // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored + if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { + + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current other node + AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + otherNodeBox.embiggen(4.0f); + + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, avatarNode); + shouldIgnore = !getsAnyIgnored; + } + } + // Not close enough to ignore + if (!shouldIgnore) { + nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); + } + } + quint64 endIgnoreCalculation = usecTimestampNow(); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + + if (!shouldIgnore) { + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); + + // FIXME - This code does appear to be working. But it seems brittle. + // It supports determining if the frame of data for this "other" + // avatar has already been sent to the reciever. This has been + // verified to work on a desktop display that renders at 60hz and + // therefore sends to mixer at 30hz. Each second you'd expect to + // have 15 (45hz-30hz) duplicate frames. In this case, the stat + // avg_other_av_skips_per_second does report 15. + // + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + shouldIgnore = true; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } + } + return shouldIgnore; + }); + + // loop through our sorted avatars and allocate our bandwidth to them accordingly + int avatarRank = 0; + + // this is overly conservative, because it includes some avatars we might not consider + int remainingAvatars = (int)sortedAvatars.size(); + + while (!sortedAvatars.empty()) { + AvatarPriority sortData = sortedAvatars.top(); + sortedAvatars.pop(); + const auto& avatarData = sortData.avatar; + avatarRank++; + remainingAvatars--; + + auto otherNode = avatarDataToNodes[avatarData]; + assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map + + // NOTE: Here's where we determine if we are over budget and drop to bare minimum data + int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; + bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; + + quint64 startAvatarDataPacking = usecTimestampNow(); + + ++numOtherAvatars; + + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + + // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO + // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. + if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { + identityBytesSent += sendIdentityPacket(otherNodeData, node); + } + + const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); + glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); + + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + bool isInView = nodeData->otherAvatarInView(otherNodeBox); + + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); + + AvatarData::AvatarDataDetail detail; + + if (overBudget) { + overBudgetAvatars++; + _stats.overBudgetAvatars++; + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; + } else if (!isInView) { + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; + nodeData->incrementAvatarOutOfView(); + } else { + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::CullSmallData; + nodeData->incrementAvatarInView(); + } + + bool includeThisAvatar = true; + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray + bool dropFaceTracking = false; + + quint64 start = usecTimestampNow(); + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); + + static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; + + dropFaceTracking = true; // first try dropping the facial data + bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; + bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + } + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; + includeThisAvatar = false; + } + } + + if (includeThisAvatar) { + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->write(bytes); + + if (detail != AvatarData::NoData) { + _stats.numOthersIncluded++; + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // remember the last time we sent details about this other node to the receiver + nodeData->setLastBroadcastTime(otherNode->getUUID(), start); + } + } + + avatarPacketList->endSegment(); + + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + }; + + quint64 startPacketSending = usecTimestampNow(); + + // close the current packet so that we're always sending something + avatarPacketList->closeCurrentPacket(true); + + _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); + _stats.numBytesSent += numAvatarDataBytes; + + // send the avatar data PacketList + nodeList->sendPacketList(std::move(avatarPacketList), *node); + + // record the bytes sent for other avatar data in the AvatarMixerClientData + nodeData->recordSentAvatarData(numAvatarDataBytes); + + // record the number of avatars held back this frame + nodeData->recordNumOtherAvatarStarves(numAvatarsHeldBack); + nodeData->recordNumOtherAvatarSkips(numAvatarsWithSkippedFrames); + + quint64 endPacketSending = usecTimestampNow(); + _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); +} + +void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node) { + } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index 04141d9d72..c11ac3291c 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -93,6 +93,9 @@ public: private: int sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); + void broadcastAvatarDataToAgent(const SharedNodePointer& node); + void broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node); + // frame state ConstIter _begin; ConstIter _end; From 98abb237832821810eb0d7759d4f3ecdb679a207 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 12:41:39 -0700 Subject: [PATCH 057/115] remove ReplicatedAvatarData handling --- assignment-client/src/avatars/AvatarMixerClientData.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerClientData.cpp b/assignment-client/src/avatars/AvatarMixerClientData.cpp index 3c9edcd4a5..4d80bc7d17 100644 --- a/assignment-client/src/avatars/AvatarMixerClientData.cpp +++ b/assignment-client/src/avatars/AvatarMixerClientData.cpp @@ -45,7 +45,6 @@ int AvatarMixerClientData::processPackets() { switch (packet->getType()) { case PacketType::AvatarData: - case PacketType::ReplicatedAvatarData: parseData(*packet); break; default: @@ -63,10 +62,6 @@ int AvatarMixerClientData::parseData(ReceivedMessage& message) { // pull the sequence number from the data first uint16_t sequenceNumber; - if (message.getType() == PacketType::ReplicatedAvatarData) { - message.seek(NUM_BYTES_RFC4122_UUID); - } - message.readPrimitive(&sequenceNumber); if (sequenceNumber < _lastReceivedSequenceNumber && _lastReceivedSequenceNumber != UINT16_MAX) { From dc94f83591cc135b9281708dc08c03ce04938e71 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 13:42:19 -0700 Subject: [PATCH 058/115] build out bulk avatar data replication --- .../src/avatars/AvatarMixerSlave.cpp | 454 +++++++++++------- libraries/networking/src/NLPacketList.h | 2 + .../networking/src/udt/PacketHeaders.cpp | 2 +- libraries/networking/src/udt/PacketHeaders.h | 2 +- libraries/networking/src/udt/PacketList.cpp | 4 +- libraries/networking/src/udt/PacketList.h | 5 +- 6 files changed, 278 insertions(+), 191 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 7732058f2d..802ea442c3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -81,12 +81,10 @@ static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; void AvatarMixerSlave::broadcastAvatarData(const SharedNodePointer& node) { quint64 start = usecTimestampNow(); - if (node->getLinkedData()) { - if (node->getType() == NodeType::Agent && node->getActiveSocket() && !node->isUpstream()) { - broadcastAvatarDataToAgent(node); - } else if (node->getType() == NodeType::DownstreamAvatarMixer) { - broadcastAvatarDataToDownstreamMixer(node); - } + if (node->getType() == NodeType::Agent && node->getLinkedData() && node->getActiveSocket() && !node->isUpstream()) { + broadcastAvatarDataToAgent(node); + } else if (node->getType() == NodeType::DownstreamAvatarMixer) { + broadcastAvatarDataToDownstreamMixer(node); } quint64 end = usecTimestampNow(); @@ -174,12 +172,11 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) std::unordered_map avatarDataToNodes; std::for_each(_begin, _end, [&](const SharedNodePointer& otherNode) { - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + // make sure this is an agent that we have avatar data for before considering it for inclusion + if (otherNode->getType() == NodeType::Agent + && otherNode->getLinkedData()) { + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - // theoretically it's possible for a Node to be in the NodeList (and therefore end up here), - // but not have yet sent data that's linked to the node. Check for that case and don't - // consider those nodes. - if (otherNodeData) { AvatarSharedPointer otherAvatar = otherNodeData->getAvatarSharedPointer(); avatarList << otherAvatar; avatarDataToNodes[otherAvatar] = otherNode; @@ -190,217 +187,216 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) ViewFrustum cameraView = nodeData->getViewFrustom(); std::priority_queue sortedAvatars; AvatarData::sortAvatars(avatarList, cameraView, sortedAvatars, + [&](AvatarSharedPointer avatar)->uint64_t{ + auto avatarNode = avatarDataToNodes[avatar]; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + return nodeData->getLastBroadcastTime(avatarNode->getUUID()); + }, - [&](AvatarSharedPointer avatar)->uint64_t{ - auto avatarNode = avatarDataToNodes[avatar]; - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - return nodeData->getLastBroadcastTime(avatarNode->getUUID()); - }, + [&](AvatarSharedPointer avatar)->float{ + glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); + return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); + }, - [&](AvatarSharedPointer avatar)->float{ - glm::vec3 nodeBoxHalfScale = (avatar->getPosition() - avatar->getGlobalBoundingBoxCorner()); - return glm::max(nodeBoxHalfScale.x, glm::max(nodeBoxHalfScale.y, nodeBoxHalfScale.z)); - }, + [&](AvatarSharedPointer avatar)->bool{ + if (avatar == thisAvatar) { + return true; // ignore ourselves... + } - [&](AvatarSharedPointer avatar)->bool{ - if (avatar == thisAvatar) { - return true; // ignore ourselves... - } + bool shouldIgnore = false; - bool shouldIgnore = false; + // We will also ignore other nodes for a couple of different reasons: + // 1) ignore bubbles and ignore specific node + // 2) the node hasn't really updated it's frame data recently, this can + // happen if for example the avatar is connected on a desktop and sending + // updates at ~30hz. So every 3 frames we skip a frame. + auto avatarNode = avatarDataToNodes[avatar]; - // We will also ignore other nodes for a couple of different reasons: - // 1) ignore bubbles and ignore specific node - // 2) the node hasn't really updated it's frame data recently, this can - // happen if for example the avatar is connected on a desktop and sending - // updates at ~30hz. So every 3 frames we skip a frame. - auto avatarNode = avatarDataToNodes[avatar]; + assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map - assert(avatarNode); // we can't have gotten here without the avatarData being a valid key in the map + const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); + assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data + quint64 startIgnoreCalculation = usecTimestampNow(); - const AvatarMixerClientData* avatarNodeData = reinterpret_cast(avatarNode->getLinkedData()); - assert(avatarNodeData); // we can't have gotten here without avatarNode having valid data - quint64 startIgnoreCalculation = usecTimestampNow(); + // make sure we have data for this avatar, that it isn't the same node, + // and isn't an avatar that the viewing node has ignored + // or that has ignored the viewing node + if (!avatarNode->getLinkedData() + || avatarNode->getUUID() == node->getUUID() + || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) + || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { + shouldIgnore = true; + } else { - // make sure we have data for this avatar, that it isn't the same node, - // and isn't an avatar that the viewing node has ignored - // or that has ignored the viewing node - if (!avatarNode->getLinkedData() - || avatarNode->getUUID() == node->getUUID() - || (node->isIgnoringNodeWithID(avatarNode->getUUID()) && !PALIsOpen) - || (avatarNode->isIgnoringNodeWithID(node->getUUID()) && !getsAnyIgnored)) { - shouldIgnore = true; - } else { + // Check to see if the space bubble is enabled + // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored + if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { - // Check to see if the space bubble is enabled - // Don't bother with these checks if the other avatar has their bubble enabled and we're gettingAnyIgnored - if (node->isIgnoreRadiusEnabled() || (avatarNode->isIgnoreRadiusEnabled() && !getsAnyIgnored)) { + // Define the scale of the box for the current other node + glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + // Set up the bounding box for the current other node + AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + // Clamp the size of the bounding box to a minimum scale + if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { + otherNodeBox.setScaleStayCentered(minBubbleSize); + } + // Quadruple the scale of both bounding boxes + otherNodeBox.embiggen(4.0f); - // Define the scale of the box for the current other node - glm::vec3 otherNodeBoxScale = (avatarNodeData->getPosition() - avatarNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - // Set up the bounding box for the current other node - AABox otherNodeBox(avatarNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - // Clamp the size of the bounding box to a minimum scale - if (glm::any(glm::lessThan(otherNodeBoxScale, minBubbleSize))) { - otherNodeBox.setScaleStayCentered(minBubbleSize); - } - // Quadruple the scale of both bounding boxes - otherNodeBox.embiggen(4.0f); + // Perform the collision check between the two bounding boxes + if (nodeBox.touches(otherNodeBox)) { + nodeData->ignoreOther(node, avatarNode); + shouldIgnore = !getsAnyIgnored; + } + } + // Not close enough to ignore + if (!shouldIgnore) { + nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); + } + } + quint64 endIgnoreCalculation = usecTimestampNow(); + _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); - // Perform the collision check between the two bounding boxes - if (nodeBox.touches(otherNodeBox)) { - nodeData->ignoreOther(node, avatarNode); - shouldIgnore = !getsAnyIgnored; - } - } - // Not close enough to ignore - if (!shouldIgnore) { - nodeData->removeFromRadiusIgnoringSet(node, avatarNode->getUUID()); - } - } - quint64 endIgnoreCalculation = usecTimestampNow(); - _stats.ignoreCalculationElapsedTime += (endIgnoreCalculation - startIgnoreCalculation); + if (!shouldIgnore) { + AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); + AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); - if (!shouldIgnore) { - AvatarDataSequenceNumber lastSeqToReceiver = nodeData->getLastBroadcastSequenceNumber(avatarNode->getUUID()); - AvatarDataSequenceNumber lastSeqFromSender = avatarNodeData->getLastReceivedSequenceNumber(); + // FIXME - This code does appear to be working. But it seems brittle. + // It supports determining if the frame of data for this "other" + // avatar has already been sent to the reciever. This has been + // verified to work on a desktop display that renders at 60hz and + // therefore sends to mixer at 30hz. Each second you'd expect to + // have 15 (45hz-30hz) duplicate frames. In this case, the stat + // avg_other_av_skips_per_second does report 15. + // + // make sure we haven't already sent this data from this sender to this receiver + // or that somehow we haven't sent + if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { + ++numAvatarsHeldBack; + shouldIgnore = true; + } else if (lastSeqFromSender - lastSeqToReceiver > 1) { + // this is a skip - we still send the packet but capture the presence of the skip so we see it happening + ++numAvatarsWithSkippedFrames; + } + } + return shouldIgnore; + }); - // FIXME - This code does appear to be working. But it seems brittle. - // It supports determining if the frame of data for this "other" - // avatar has already been sent to the reciever. This has been - // verified to work on a desktop display that renders at 60hz and - // therefore sends to mixer at 30hz. Each second you'd expect to - // have 15 (45hz-30hz) duplicate frames. In this case, the stat - // avg_other_av_skips_per_second does report 15. - // - // make sure we haven't already sent this data from this sender to this receiver - // or that somehow we haven't sent - if (lastSeqToReceiver == lastSeqFromSender && lastSeqToReceiver != 0) { - ++numAvatarsHeldBack; - shouldIgnore = true; - } else if (lastSeqFromSender - lastSeqToReceiver > 1) { - // this is a skip - we still send the packet but capture the presence of the skip so we see it happening - ++numAvatarsWithSkippedFrames; - } - } - return shouldIgnore; - }); + // loop through our sorted avatars and allocate our bandwidth to them accordingly + int avatarRank = 0; - // loop through our sorted avatars and allocate our bandwidth to them accordingly - int avatarRank = 0; + // this is overly conservative, because it includes some avatars we might not consider + int remainingAvatars = (int)sortedAvatars.size(); - // this is overly conservative, because it includes some avatars we might not consider - int remainingAvatars = (int)sortedAvatars.size(); + while (!sortedAvatars.empty()) { + AvatarPriority sortData = sortedAvatars.top(); + sortedAvatars.pop(); + const auto& avatarData = sortData.avatar; + avatarRank++; + remainingAvatars--; - while (!sortedAvatars.empty()) { - AvatarPriority sortData = sortedAvatars.top(); - sortedAvatars.pop(); - const auto& avatarData = sortData.avatar; - avatarRank++; - remainingAvatars--; + auto otherNode = avatarDataToNodes[avatarData]; + assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map - auto otherNode = avatarDataToNodes[avatarData]; - assert(otherNode); // we can't have gotten here without the avatarData being a valid key in the map + // NOTE: Here's where we determine if we are over budget and drop to bare minimum data + int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; + bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; - // NOTE: Here's where we determine if we are over budget and drop to bare minimum data - int minimRemainingAvatarBytes = minimumBytesPerAvatar * remainingAvatars; - bool overBudget = (identityBytesSent + numAvatarDataBytes + minimRemainingAvatarBytes) > maxAvatarBytesPerFrame; + quint64 startAvatarDataPacking = usecTimestampNow(); - quint64 startAvatarDataPacking = usecTimestampNow(); + ++numOtherAvatars; - ++numOtherAvatars; + const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); - const AvatarMixerClientData* otherNodeData = reinterpret_cast(otherNode->getLinkedData()); + // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO + // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. + if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { + identityBytesSent += sendIdentityPacket(otherNodeData, node); + } - // If the time that the mixer sent AVATAR DATA about Avatar B to Avatar A is BEFORE OR EQUAL TO - // the time that Avatar B flagged an IDENTITY DATA change, send IDENTITY DATA about Avatar B to Avatar A. - if (nodeData->getLastBroadcastTime(otherNode->getUUID()) <= otherNodeData->getIdentityChangeTimestamp()) { - identityBytesSent += sendIdentityPacket(otherNodeData, node); - } + const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); + glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); - const AvatarData* otherAvatar = otherNodeData->getConstAvatarData(); - glm::vec3 otherPosition = otherAvatar->getClientGlobalPosition(); + // determine if avatar is in view, to determine how much data to include... + glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; + AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); + bool isInView = nodeData->otherAvatarInView(otherNodeBox); - // determine if avatar is in view, to determine how much data to include... - glm::vec3 otherNodeBoxScale = (otherPosition - otherNodeData->getGlobalBoundingBoxCorner()) * 2.0f; - AABox otherNodeBox(otherNodeData->getGlobalBoundingBoxCorner(), otherNodeBoxScale); - bool isInView = nodeData->otherAvatarInView(otherNodeBox); + // start a new segment in the PacketList for this avatar + avatarPacketList->startSegment(); - // start a new segment in the PacketList for this avatar - avatarPacketList->startSegment(); + AvatarData::AvatarDataDetail detail; - AvatarData::AvatarDataDetail detail; + if (overBudget) { + overBudgetAvatars++; + _stats.overBudgetAvatars++; + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; + } else if (!isInView) { + detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; + nodeData->incrementAvatarOutOfView(); + } else { + detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO + ? AvatarData::SendAllData : AvatarData::CullSmallData; + nodeData->incrementAvatarInView(); + } - if (overBudget) { - overBudgetAvatars++; - _stats.overBudgetAvatars++; - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::NoData; - } else if (!isInView) { - detail = PALIsOpen ? AvatarData::PALMinimum : AvatarData::MinimumData; - nodeData->incrementAvatarOutOfView(); - } else { - detail = distribution(generator) < AVATAR_SEND_FULL_UPDATE_RATIO - ? AvatarData::SendAllData : AvatarData::CullSmallData; - nodeData->incrementAvatarInView(); - } + bool includeThisAvatar = true; + auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); + QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); + bool distanceAdjust = true; + glm::vec3 viewerPosition = myPosition; + AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray + bool dropFaceTracking = false; - bool includeThisAvatar = true; - auto lastEncodeForOther = nodeData->getLastOtherAvatarEncodeTime(otherNode->getUUID()); - QVector& lastSentJointsForOther = nodeData->getLastOtherAvatarSentJoints(otherNode->getUUID()); - bool distanceAdjust = true; - glm::vec3 viewerPosition = myPosition; - AvatarDataPacket::HasFlags hasFlagsOut; // the result of the toByteArray - bool dropFaceTracking = false; - - quint64 start = usecTimestampNow(); - QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); - quint64 end = usecTimestampNow(); - _stats.toByteArrayElapsedTime += (end - start); - - static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; - - dropFaceTracking = true; // first try dropping the facial data - bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, - hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + quint64 start = usecTimestampNow(); + QByteArray bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); + static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID); if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; - bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data"; + + dropFaceTracking = true; // first try dropping the facial data + bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther, hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData"; + bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther, + hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther); + } + + if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { + qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; + includeThisAvatar = false; + } } - if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) { - qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!"; - includeThisAvatar = false; + if (includeThisAvatar) { + numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->write(bytes); + + if (detail != AvatarData::NoData) { + _stats.numOthersIncluded++; + + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), + otherNodeData->getLastReceivedSequenceNumber()); + + // remember the last time we sent details about this other node to the receiver + nodeData->setLastBroadcastTime(otherNode->getUUID(), start); + } } - } - if (includeThisAvatar) { - numAvatarDataBytes += avatarPacketList->write(otherNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->write(bytes); + avatarPacketList->endSegment(); - if (detail != AvatarData::NoData) { - _stats.numOthersIncluded++; - - // increment the number of avatars sent to this reciever - nodeData->incrementNumAvatarsSentLastFrame(); - - // set the last sent sequence number for this sender on the receiver - nodeData->setLastBroadcastSequenceNumber(otherNode->getUUID(), - otherNodeData->getLastReceivedSequenceNumber()); - - // remember the last time we sent details about this other node to the receiver - nodeData->setLastBroadcastTime(otherNode->getUUID(), start); - } - } - - avatarPacketList->endSegment(); - - quint64 endAvatarDataPacking = usecTimestampNow(); - _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); }; quint64 startPacketSending = usecTimestampNow(); @@ -426,6 +422,92 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) } void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node) { + _stats.nodesBroadcastedTo++; + // setup a PacketList for the replicated bulk avatar data + auto avatarPacketList = NLPacketList::create(PacketType::ReplicatedBulkAvatarData); + + int numAvatarDataBytes = 0; + + std::for_each(_begin, _end, [&](const SharedNodePointer& agentNode) { + // collect agents that we have avatar data for + if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData()) { + const AvatarMixerClientData* agentNodeData = reinterpret_cast(agentNode->getLinkedData()); + + AvatarSharedPointer otherAvatar = agentNodeData->getAvatarSharedPointer(); + + quint64 startAvatarDataPacking = usecTimestampNow(); + + // we cannot send a downstream avatar mixer any updates that expect them to have previous state for this avatar + // since we have no idea if they're online and receiving our packets + + // so we always send a full update for this avatar + quint64 start = usecTimestampNow(); + AvatarDataPacket::HasFlags flagsOut; + QByteArray avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, {}, + flagsOut, false, false, + glm::vec3(0), nullptr); + quint64 end = usecTimestampNow(); + _stats.toByteArrayElapsedTime += (end - start); + + // figure out how large our avatar byte array can be to fit in the packet list + // given that we need it and the avatar UUID and the size of the byte array (16 bit) + // to fit in a segment of the packet list + auto maxAvatarByteArraySize = avatarPacketList->getMaxSegmentSize(); + maxAvatarByteArraySize -= NUM_BYTES_RFC4122_UUID; + maxAvatarByteArraySize -= sizeof(quint16); + + if (avatarByteArray.size() > maxAvatarByteArraySize) { + qCWarning(avatars) << "Replicated avatar data too large for" << otherAvatar->getSessionUUID() + << "-" << avatarByteArray.size() << "bytes"; + + avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, {}, + flagsOut, true, false, + glm::vec3(0), nullptr); + + if (avatarByteArray.size() > maxAvatarByteArraySize) { + qCWarning(avatars) << "Replicated avatar data without facial data still too large for" + << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; + + avatarByteArray = otherAvatar->toByteArray(AvatarData::MinimumData, 0, {}, + flagsOut, true, false, + glm::vec3(0), nullptr); + } + } + + if (avatarByteArray.size() <= maxAvatarByteArraySize) { + // start a new segment in the packet list for this avatar + avatarPacketList->startSegment(); + + numAvatarDataBytes += avatarPacketList->write(agentNode->getUUID().toRfc4122()); + numAvatarDataBytes += avatarPacketList->writePrimitive((quint16) avatarByteArray.size()); + numAvatarDataBytes += avatarPacketList->write(avatarByteArray); + + avatarPacketList->endSegment(); + + } else { + qCWarning(avatars) << "Could not fit minimum data avatar for" << otherAvatar->getSessionUUID() + << "to packet list -" << avatarByteArray.size() << "bytes"; + } + + quint64 endAvatarDataPacking = usecTimestampNow(); + _stats.avatarDataPackingElapsedTime += (endAvatarDataPacking - startAvatarDataPacking); + } + }); + + quint64 startPacketSending = usecTimestampNow(); + + // close the current packet so that we're always sending something + avatarPacketList->closeCurrentPacket(true); + + _stats.numPacketsSent += (int)avatarPacketList->getNumPackets(); + _stats.numBytesSent += numAvatarDataBytes; + + // send the replicated bulk avatar data + auto nodeList = DependencyManager::get(); + nodeList->sendPacketList(std::move(avatarPacketList), node->getPublicSocket()); + + quint64 endPacketSending = usecTimestampNow(); + _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); } diff --git a/libraries/networking/src/NLPacketList.h b/libraries/networking/src/NLPacketList.h index 48ce5ef81a..910d39f71b 100644 --- a/libraries/networking/src/NLPacketList.h +++ b/libraries/networking/src/NLPacketList.h @@ -23,6 +23,8 @@ public: PacketVersion getVersion() const { return _packetVersion; } const QUuid& getSourceID() const { return _sourceID; } + + qint64 getMaxSegmentSize() const override { return NLPacket::maxPayloadSize(_packetType, _isOrdered); } private: NLPacketList(PacketType packetType, QByteArray extendedHeader = QByteArray(), bool isReliable = false, diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index ad30a184ca..fc6251566e 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -42,7 +42,7 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::DomainServerRemovedNode << PacketType::UsernameFromIDReply << PacketType::OctreeFileReplacement << PacketType::ReplicatedMicrophoneAudioNoEcho << PacketType::ReplicatedMicrophoneAudioWithEcho << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame - << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedKillAvatar; + << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedKillAvatar << PacketType::ReplicatedBulkAvatarData; PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index b7d55c3266..9b37a7d76d 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -121,7 +121,7 @@ public: ReplicatedSilentAudioFrame, ReplicatedAvatarIdentity, ReplicatedKillAvatar, - + ReplicatedBulkAvatarData, NUM_PACKET_TYPE }; }; diff --git a/libraries/networking/src/udt/PacketList.cpp b/libraries/networking/src/udt/PacketList.cpp index 8651f9eed4..d69ff39197 100644 --- a/libraries/networking/src/udt/PacketList.cpp +++ b/libraries/networking/src/udt/PacketList.cpp @@ -36,8 +36,8 @@ std::unique_ptr PacketList::fromReceivedPackets(std::list> _packets; + + bool _isOrdered = false; private: friend class ::LimitedNodeList; @@ -93,7 +97,6 @@ private: Packet::MessageNumber _messageNumber; bool _isReliable = false; - bool _isOrdered = false; std::unique_ptr _currentPacket; From 01263d9435424dc5a93d6a3f21458ea0b0f978ce Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:29:12 -0700 Subject: [PATCH 059/115] only send identity packets directly to agents not upstream --- assignment-client/src/avatars/AvatarMixer.cpp | 14 ++++++++------ .../src/avatars/AvatarMixerSlave.cpp | 18 +++++++++++------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 34fcc3c0ed..4b032496b1 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -126,12 +126,14 @@ AvatarMixer::~AvatarMixer() { } void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - QByteArray individualData = nodeData->getAvatar().identityByteArray(); - individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); - auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); - identityPackets->write(individualData); - DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); - ++_sumIdentityPackets; + if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + QByteArray individualData = nodeData->getAvatar().identityByteArray(); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); + auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); + identityPackets->write(individualData); + DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + ++_sumIdentityPackets; + } } std::chrono::microseconds AvatarMixer::timeFrame(p_high_resolution_clock::time_point& timestamp) { diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 802ea442c3..581052cab3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -67,13 +67,17 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); - individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); - identityPackets->write(individualData); - DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); - _stats.numIdentityPackets++; - return individualData.size(); + if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious + auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); + identityPackets->write(individualData); + DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + _stats.numIdentityPackets++; + return individualData.size(); + } else { + return -1; + } } static const int AVATAR_MIXER_BROADCAST_FRAMES_PER_SECOND = 45; From 6c5947d319a809cad6c20b8185316b3693a103ba Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 14:43:59 -0700 Subject: [PATCH 060/115] Add periodic sending of avatar identity data to downstream nodes --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 581052cab3..17fadd8284 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -425,9 +425,13 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node) _stats.packetSendingElapsedTime += (endPacketSending - startPacketSending); } +uint64_t REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US = 5 * 1000 * 1000; + void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node) { _stats.nodesBroadcastedTo++; + AvatarMixerClientData* nodeData = reinterpret_cast(node->getLinkedData()); + // setup a PacketList for the replicated bulk avatar data auto avatarPacketList = NLPacketList::create(PacketType::ReplicatedBulkAvatarData); @@ -438,6 +442,13 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData()) { const AvatarMixerClientData* agentNodeData = reinterpret_cast(agentNode->getLinkedData()); + auto now = usecTimestampNow(); + auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getUUID()); + if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() + || (now - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { + sendIdentityPacket(agentNodeData, node); + } + AvatarSharedPointer otherAvatar = agentNodeData->getAvatarSharedPointer(); quint64 startAvatarDataPacking = usecTimestampNow(); From 52150ad97152bb28851cd60bd3c3cc5bfb4ebffb Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 15:29:25 -0700 Subject: [PATCH 061/115] Add node data to downstream avatar mixers --- assignment-client/src/avatars/AvatarMixer.cpp | 6 +++++- libraries/networking/src/ThreadedAssignment.cpp | 7 ++++--- libraries/networking/src/ThreadedAssignment.h | 5 ++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 4b032496b1..582c39b124 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -851,5 +851,9 @@ void AvatarMixer::parseDomainServerSettings(const QJsonObject& domainSettings) { << "and a maximum avatar scale of" << _domainMaximumScale; - parseDownstreamServers(domainSettings, NodeType::AvatarMixer); + parseDownstreamServers(domainSettings, NodeType::AvatarMixer, [](Node& node) { + if (!node.getLinkedData()) { + node.setLinkedData(std::unique_ptr { new AvatarMixerClientData(node.getUUID()) }); + } + }); } diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 5aeb076c11..24573e0ffc 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -134,7 +134,7 @@ void ThreadedAssignment::domainSettingsRequestFailed() { setFinished(true); } -void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType) { +void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType, DownstreamNodeFoundCallback callback) { static const QString REPLICATION_GROUP_KEY = "replication"; static const QString DOWNSTREAM_SERVERS_SETTING_KEY = "downstream_servers"; if (settingsObject.contains(REPLICATION_GROUP_KEY)) { @@ -161,8 +161,9 @@ void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObjec }; // manually add the downstream node to our node list - nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), - downstreamServerAddr, downstreamServerAddr); + auto node = nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), + downstreamServerAddr, downstreamServerAddr); + callback(*node); } } } diff --git a/libraries/networking/src/ThreadedAssignment.h b/libraries/networking/src/ThreadedAssignment.h index f96755a776..0cc7b2f40c 100644 --- a/libraries/networking/src/ThreadedAssignment.h +++ b/libraries/networking/src/ThreadedAssignment.h @@ -18,6 +18,8 @@ #include "Assignment.h" +using DownstreamNodeFoundCallback = std::function; + class ThreadedAssignment : public Assignment { Q_OBJECT public: @@ -40,7 +42,8 @@ signals: protected: void commonInit(const QString& targetName, NodeType_t nodeType); - void parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType); + void parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType, + DownstreamNodeFoundCallback callback = [](Node& downstreamNode) {}); bool _isFinished; QTimer _domainServerTimer; From da3cd59a968fb8816655d91aa9da08a26fb3d433 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 15:29:55 -0700 Subject: [PATCH 062/115] Fix avatar mixer not setting last broadcast time for downstream nodes --- .../src/avatars/AvatarMixerSlave.cpp | 15 ++++++++------- libraries/networking/src/LimitedNodeList.cpp | 6 ++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 17fadd8284..fe28c96340 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -442,13 +442,6 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin if (agentNode->getType() == NodeType::Agent && agentNode->getLinkedData()) { const AvatarMixerClientData* agentNodeData = reinterpret_cast(agentNode->getLinkedData()); - auto now = usecTimestampNow(); - auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getUUID()); - if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() - || (now - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { - sendIdentityPacket(agentNodeData, node); - } - AvatarSharedPointer otherAvatar = agentNodeData->getAvatarSharedPointer(); quint64 startAvatarDataPacking = usecTimestampNow(); @@ -465,6 +458,14 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); + auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getUUID()); + if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() + || (start - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { + qDebug() << "Sending identity packet for " << agentNode->getUUID() << " to " << node->getUUID(); + sendIdentityPacket(agentNodeData, node); + nodeData->setLastBroadcastTime(agentNode->getUUID(), start); + } + // figure out how large our avatar byte array can be to fit in the packet list // given that we need it and the avatar UUID and the size of the byte array (16 bit) // to fit in a segment of the packet list diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 0494efc7b4..9c0754cf26 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -446,7 +446,8 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, return _nodeSocket.writePacketList(std::move(packetList), *activeSocket); } else { - qCDebug(networking) << "LimitedNodeList::sendPacketList called without active socket for node. Not sending."; + qCDebug(networking) << "LimitedNodeList::sendPacketList called without active socket for node " + << destinationNode.getUUID() << ". Not sending."; return ERROR_SENDING_PACKET_BYTES; } } @@ -454,7 +455,8 @@ qint64 LimitedNodeList::sendPacketList(std::unique_ptr packetList, qint64 LimitedNodeList::sendPacket(std::unique_ptr packet, const Node& destinationNode, const HifiSockAddr& overridenSockAddr) { if (overridenSockAddr.isNull() && !destinationNode.getActiveSocket()) { - qCDebug(networking) << "LimitedNodeList::sendPacket called without active socket for node. Not sending."; + qCDebug(networking) << "LimitedNodeList::sendPacket called without active socket for node" + << destinationNode.getUUID() << ". Not sending."; return ERROR_SENDING_PACKET_BYTES; } From 76f1a7445e5c4cc7dfb9454d5d6f5d6d3f2ebc00 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:32:07 -0700 Subject: [PATCH 063/115] handle replicated bulk avatar data packets --- assignment-client/src/avatars/AvatarMixer.cpp | 37 +++++++++++++++++++ assignment-client/src/avatars/AvatarMixer.h | 1 + .../src/avatars/AvatarMixerSlave.cpp | 18 ++++----- libraries/networking/src/ReceivedMessage.cpp | 14 +++++++ libraries/networking/src/ReceivedMessage.h | 2 + 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 582c39b124..5aaf633ed9 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -59,6 +59,8 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : PacketType::ReplicatedKillAvatar }, this, "handleReplicatedPackets"); + packetReceiver.registerListener(PacketType::ReplicatedBulkAvatarData, this, "handleReplicatedBulkAvatarPacket"); + auto nodeList = DependencyManager::get(); connect(nodeList.data(), &NodeList::packetVersionMismatch, this, &AvatarMixer::handlePacketVersionMismatch); } @@ -91,6 +93,41 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag } +void AvatarMixer::handleReplicatedBulkAvatarPacket(QSharedPointer message) { + auto nodeList = DependencyManager::get(); + + while (message->getBytesLeftToRead()) { + // first, grab the node ID for this replicated avatar + auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); + + // make sure we have an upstream replicated node that matches + auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, + message->getSenderSockAddr(), message->getSenderSockAddr(), + DEFAULT_AGENT_PERMISSIONS, true); + + replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); + replicatedNode->setIsUpstream(true); + + // grab the size of the avatar byte array so we know how much to read + quint16 avatarByteArraySize; + message->readPrimitive(&avatarByteArraySize); + + // read the avatar byte array + auto avatarByteArray = message->read(avatarByteArraySize); + + // construct a "fake" avatar data received message from the byte array and packet list information + auto replicatedMessage = QSharedPointer::create(avatarByteArray, PacketType::AvatarData, + versionForPacketType(PacketType::AvatarData), + message->getSenderSockAddr(), nodeID); + + // queue up the replicated avatar data with the client data for the replicated node + auto start = usecTimestampNow(); + getOrCreateClientData(replicatedNode)->queuePacket(replicatedMessage, replicatedNode); + auto end = usecTimestampNow(); + _queueIncomingPacketElapsedTime += (end - start); + } +} + void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node& node) { // first, make sure that this is a packet from a node we are supposed to replicate if (node.isReplicated() diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 17a6db3b08..e0d073a281 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -47,6 +47,7 @@ private slots: void handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); void handleReplicatedPackets(QSharedPointer message); + void handleReplicatedBulkAvatarPacket(QSharedPointer message); void domainSettingsRequestComplete(); void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID); void start(); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index fe28c96340..a536eaabd3 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -452,9 +452,11 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin // so we always send a full update for this avatar quint64 start = usecTimestampNow(); AvatarDataPacket::HasFlags flagsOut; - QByteArray avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, {}, - flagsOut, false, false, - glm::vec3(0), nullptr); + + QVector emptyLastJointSendData { otherAvatar->getJointCount() }; + + QByteArray avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, emptyLastJointSendData, + flagsOut, false, false, glm::vec3(0), nullptr); quint64 end = usecTimestampNow(); _stats.toByteArrayElapsedTime += (end - start); @@ -477,17 +479,15 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin qCWarning(avatars) << "Replicated avatar data too large for" << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; - avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, {}, - flagsOut, true, false, - glm::vec3(0), nullptr); + avatarByteArray = otherAvatar->toByteArray(AvatarData::SendAllData, 0, emptyLastJointSendData, + flagsOut, true, false, glm::vec3(0), nullptr); if (avatarByteArray.size() > maxAvatarByteArraySize) { qCWarning(avatars) << "Replicated avatar data without facial data still too large for" << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; - avatarByteArray = otherAvatar->toByteArray(AvatarData::MinimumData, 0, {}, - flagsOut, true, false, - glm::vec3(0), nullptr); + avatarByteArray = otherAvatar->toByteArray(AvatarData::MinimumData, 0, emptyLastJointSendData, + flagsOut, true, false, glm::vec3(0), nullptr); } } diff --git a/libraries/networking/src/ReceivedMessage.cpp b/libraries/networking/src/ReceivedMessage.cpp index 2c5a11334b..6ca249fb22 100644 --- a/libraries/networking/src/ReceivedMessage.cpp +++ b/libraries/networking/src/ReceivedMessage.cpp @@ -42,6 +42,20 @@ ReceivedMessage::ReceivedMessage(NLPacket& packet) { } +ReceivedMessage::ReceivedMessage(QByteArray byteArray, PacketType packetType, PacketVersion packetVersion, + const HifiSockAddr& senderSockAddr, QUuid sourceID) : + _data(byteArray), + _headData(_data.mid(0, HEAD_DATA_SIZE)), + _numPackets(1), + _sourceID(sourceID), + _packetType(packetType), + _packetVersion(packetVersion), + _senderSockAddr(senderSockAddr), + _isComplete(true) +{ + +} + void ReceivedMessage::setFailed() { _failed = true; _isComplete = true; diff --git a/libraries/networking/src/ReceivedMessage.h b/libraries/networking/src/ReceivedMessage.h index 3acb7163e7..ae51e7592a 100644 --- a/libraries/networking/src/ReceivedMessage.h +++ b/libraries/networking/src/ReceivedMessage.h @@ -24,6 +24,8 @@ class ReceivedMessage : public QObject { public: ReceivedMessage(const NLPacketList& packetList); ReceivedMessage(NLPacket& packet); + ReceivedMessage(QByteArray byteArray, PacketType packetType, PacketVersion packetVersion, + const HifiSockAddr& senderSockAddr, QUuid sourceID = QUuid()); QByteArray getMessage() const { return _data; } const char* getRawMessage() const { return _data.constData(); } From 50f46dafa6fa2bdc463b97f5732588d78eb5b259 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:38:15 -0700 Subject: [PATCH 064/115] add a sequence number to replicated avatar data --- .../src/avatars/AvatarMixerSlave.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index a536eaabd3..8a3ec2dc5a 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -475,6 +475,9 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin maxAvatarByteArraySize -= NUM_BYTES_RFC4122_UUID; maxAvatarByteArraySize -= sizeof(quint16); + auto sequenceNumberSize = sizeof(agentNodeData->getLastReceivedSequenceNumber()); + maxAvatarByteArraySize -= sequenceNumberSize; + if (avatarByteArray.size() > maxAvatarByteArraySize) { qCWarning(avatars) << "Replicated avatar data too large for" << otherAvatar->getSessionUUID() << "-" << avatarByteArray.size() << "bytes"; @@ -492,11 +495,21 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin } if (avatarByteArray.size() <= maxAvatarByteArraySize) { + // increment the number of avatars sent to this reciever + nodeData->incrementNumAvatarsSentLastFrame(); + + // set the last sent sequence number for this sender on the receiver + nodeData->setLastBroadcastSequenceNumber(agentNode->getUUID(), + agentNodeData->getLastReceivedSequenceNumber()); + // start a new segment in the packet list for this avatar avatarPacketList->startSegment(); + // write the node's UUID, the size of the replicated avatar data, + // the sequence number of the replicated avatar data, and the replicated avatar data numAvatarDataBytes += avatarPacketList->write(agentNode->getUUID().toRfc4122()); - numAvatarDataBytes += avatarPacketList->writePrimitive((quint16) avatarByteArray.size()); + numAvatarDataBytes += avatarPacketList->writePrimitive((quint16) (avatarByteArray.size() + sequenceNumberSize)); + numAvatarDataBytes += avatarPacketList->writePrimitive(agentNodeData->getLastReceivedSequenceNumber()); numAvatarDataBytes += avatarPacketList->write(avatarByteArray); avatarPacketList->endSegment(); From 9085a0896a308f0e197b4a1fd8f05881ade83061 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:46:14 -0700 Subject: [PATCH 065/115] manually activate sockets for downstream servers --- assignment-client/src/audio/AudioMixerClientData.cpp | 2 +- assignment-client/src/avatars/AvatarMixer.cpp | 4 ++-- libraries/networking/src/ThreadedAssignment.cpp | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 848dc969d8..e976e4176d 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -169,7 +169,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c packet->write(message.getMessage()); } - nodeList->sendUnreliablePacket(*packet, downstreamNode->getPublicSocket()); + nodeList->sendUnreliablePacket(*packet, *downstreamNode); } }); } diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 5aaf633ed9..57c48ee6d4 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -146,7 +146,7 @@ void AvatarMixer::optionallyReplicatePacket(ReceivedMessage& message, const Node packet->write(message.getMessage()); } - nodeList->sendUnreliablePacket(*packet, node->getPublicSocket()); + nodeList->sendUnreliablePacket(*packet, *node); }); } } @@ -402,7 +402,7 @@ void AvatarMixer::nodeKilled(SharedNodePointer killedNode) { replicatedKillPacket->writePrimitive(KillAvatarReason::AvatarDisconnected); } - nodeList->sendUnreliablePacket(*replicatedKillPacket, node->getPublicSocket()); + nodeList->sendUnreliablePacket(*replicatedKillPacket, *node); } }); diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 24573e0ffc..3e679f643a 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -163,6 +163,10 @@ void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObjec // manually add the downstream node to our node list auto node = nodeList->addOrUpdateNode(QUuid::createUuid(), NodeType::downstreamType(nodeType), downstreamServerAddr, downstreamServerAddr); + + // manually activate the public socket for the downstream node + node->activatePublicSocket(); + callback(*node); } } From 8f7a3595f58f58560c668fc44f78340e89b85e25 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 15:49:03 -0700 Subject: [PATCH 066/115] only send display name changes back to directly connected agents --- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 57c48ee6d4..988ddfed4c 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -163,7 +163,7 @@ AvatarMixer::~AvatarMixer() { } void AvatarMixer::sendIdentityPacket(AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { - if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + if (destinationNode->getType() == NodeType::Agent && !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getAvatar().identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); From 0708daa6ccfeb401ef53601eeaabfa3894448004 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 16:10:08 -0700 Subject: [PATCH 067/115] Add separate sendReplicatedIdentityPacket --- assignment-client/src/avatars/AvatarMixer.cpp | 6 +++--- .../src/avatars/AvatarMixerSlave.cpp | 21 +++++++++++++++---- .../src/avatars/AvatarMixerSlave.h | 1 + 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 988ddfed4c..c88108fb36 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -76,14 +76,14 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); + // seek back in the message so the packet handler can start by reading the source ID + message->seek(0); + switch (message->getType()) { case PacketType::ReplicatedAvatarIdentity: handleAvatarIdentityPacket(message, replicatedNode); break; case PacketType::ReplicatedKillAvatar: - // seek back in the message so the kill packet handler can start by reading the source ID - message->seek(0); - handleKillAvatarPacket(message, replicatedNode); break; default: diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 8a3ec2dc5a..55975a790d 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -65,18 +65,31 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { _stats.processIncomingPacketsElapsedTime += (end - start); } - int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); + auto identityPackets = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); identityPackets->write(individualData); DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); _stats.numIdentityPackets++; return individualData.size(); } else { - return -1; + return 0; + } +} + +int AvatarMixerSlave::sendReplicatedIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { + if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); + individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious + auto identityPackets = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); + identityPackets->write(individualData); + DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + _stats.numIdentityPackets++; + return individualData.size(); + } else { + return 0; } } @@ -464,7 +477,7 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() || (start - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { qDebug() << "Sending identity packet for " << agentNode->getUUID() << " to " << node->getUUID(); - sendIdentityPacket(agentNodeData, node); + sendReplicatedIdentityPacket(agentNodeData, node); nodeData->setLastBroadcastTime(agentNode->getUUID(), start); } diff --git a/assignment-client/src/avatars/AvatarMixerSlave.h b/assignment-client/src/avatars/AvatarMixerSlave.h index c11ac3291c..509a8ec94b 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.h +++ b/assignment-client/src/avatars/AvatarMixerSlave.h @@ -92,6 +92,7 @@ public: private: int sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); + int sendReplicatedIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode); void broadcastAvatarDataToAgent(const SharedNodePointer& node); void broadcastAvatarDataToDownstreamMixer(const SharedNodePointer& node); From a476a5b82ea740ff34fa5f144acd4ad38d0b05a4 Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Tue, 13 Jun 2017 16:42:38 -0700 Subject: [PATCH 068/115] Change ReplicatedAvatarIdentity packet to be an unreliable packet --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 55975a790d..b6e76e2b57 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -69,7 +69,7 @@ int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); + auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); identityPackets->write(individualData); DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); _stats.numIdentityPackets++; @@ -83,9 +83,9 @@ int AvatarMixerSlave::sendReplicatedIdentityPacket(const AvatarMixerClientData* if (destinationNode->getType() == NodeType::DownstreamAvatarMixer || !destinationNode->isUpstream()) { QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious - auto identityPackets = NLPacketList::create(PacketType::ReplicatedAvatarIdentity, QByteArray(), true, true); - identityPackets->write(individualData); - DependencyManager::get()->sendPacketList(std::move(identityPackets), *destinationNode); + auto identityPacket = NLPacket::create(PacketType::ReplicatedAvatarIdentity); + identityPacket->write(individualData); + DependencyManager::get()->sendUnreliablePacket(*identityPacket, *destinationNode); _stats.numIdentityPackets++; return individualData.size(); } else { From 29842c67cc1705ae6d0446675426cf31ac0e1fc5 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Tue, 13 Jun 2017 18:04:59 -0700 Subject: [PATCH 069/115] use received message faking for cleaner replication in audio --- assignment-client/src/audio/AudioMixer.cpp | 21 +++++++++++++++- .../src/audio/AudioMixerClientData.cpp | 24 +++++-------------- libraries/audio/src/InboundAudioStream.cpp | 12 +++++++++- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index f8c2cf86e8..e397254441 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -125,7 +125,26 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); replicatedNode->setIsUpstream(true); - getOrCreateClientData(replicatedNode.data())->queuePacket(message, replicatedNode); + // construct a "fake" avatar data received message from the byte array and packet list information + auto audioData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID); + + PacketType rewrittenType; + + if (message->getType() == PacketType::ReplicatedMicrophoneAudioNoEcho) { + rewrittenType = PacketType::MicrophoneAudioNoEcho; + } else if (message->getType() == PacketType::ReplicatedMicrophoneAudioWithEcho) { + rewrittenType = PacketType::MicrophoneAudioWithEcho; + } else if (message->getType() == PacketType::ReplicatedInjectAudio) { + rewrittenType = PacketType::InjectAudio; + } else if (message->getType() == PacketType::ReplicatedSilentAudioFrame) { + rewrittenType = PacketType::SilentAudioFrame; + } + + auto replicatedMessage = QSharedPointer::create(audioData, rewrittenType, + versionForPacketType(rewrittenType), + message->getSenderSockAddr(), nodeID); + + getOrCreateClientData(replicatedNode.data())->queuePacket(replicatedMessage, replicatedNode); } void AudioMixer::handleMuteEnvironmentPacket(QSharedPointer message, SharedNodePointer sendingNode) { diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index e976e4176d..d4d098133d 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -160,10 +160,6 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c if (!isReplicatedPacket(message.getType())) { // since this packet will be non-sourced, we add the replicated node's ID here packet->write(node.getUUID().toRfc4122()); - - // we won't negotiate an audio format with the replicant, because we aren't a listener - // so pack the codec string here so that it can statelessly setup a decoder for this string when it needs - packet->writeString(_selectedCodecName); } packet->write(message.getMessage()); @@ -312,6 +308,7 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { // this is injected audio // grab the stream identifier for this injected audio message.seek(sizeof(quint16)); + QUuid streamIdentifier = QUuid::fromRfc4122(message.readWithoutCopy(NUM_BYTES_RFC4122_UUID)); bool isStereo; @@ -346,18 +343,6 @@ int AudioMixerClientData::parseData(ReceivedMessage& message) { // seek to the beginning of the packet so that the next reader is in the right spot message.seek(0); - if (packetType == PacketType::ReplicatedMicrophoneAudioWithEcho - || packetType == PacketType::ReplicatedMicrophoneAudioNoEcho - || packetType == PacketType::ReplicatedSilentAudioFrame - || packetType == PacketType::ReplicatedInjectAudio) { - - // skip past source ID for the replicated packet - message.seek(NUM_BYTES_RFC4122_UUID); - - // skip past the codec string - message.readString(); - } - // check the overflow count before we parse data auto overflowBefore = matchingStream->getOverflowCount(); auto parseResult = matchingStream->parseData(message); @@ -706,9 +691,9 @@ bool AudioMixerClientData::shouldIgnore(const SharedNodePointer self, const Shar } void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointer message) { - // first pull the codec string from the packet + // pull the codec string from the packet + message->seek(sizeof(quint16)); - // read the string for the codec auto codecString = message->readString(); qDebug() << "Manually setting codec for replicated agent" << uuidStringWithoutCurlyBraces(getNodeID()) @@ -718,4 +703,7 @@ void AudioMixerClientData::setupCodecForReplicatedAgent(QSharedPointerseek(0); } diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 65cccf1fe0..56353e14e3 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -127,7 +127,17 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { // parse the info after the seq number and before the audio data (the stream properties) int prePropertyPosition = message.getPosition(); int propertyBytes = parseStreamProperties(message.getType(), message.readWithoutCopy(message.getBytesLeftToRead()), networkFrames); - message.seek(prePropertyPosition + propertyBytes); + + if (message.getType() == PacketType::ReplicatedMicrophoneAudioNoEcho + || message.getType() == PacketType::ReplicatedMicrophoneAudioWithEcho + || message.getType() == PacketType::ReplicatedInjectAudio + || message.getType() == PacketType::ReplicatedSilentAudioFrame) { + message.seek(NUM_BYTES_RFC4122_UUID); + message.readString(); + message.read(sizeof(quint16) + prePropertyPosition + propertyBytes); + } else { + message.seek(prePropertyPosition + propertyBytes); + } // handle this packet based on its arrival status. switch (arrivalInfo._status) { From 2f307f418cf737172581196b2a33f80635f2ab0b Mon Sep 17 00:00:00 2001 From: seefo Date: Tue, 13 Jun 2017 15:40:37 -0700 Subject: [PATCH 070/115] Added support for dropdown to tables in domain settings --- .../resources/describe-settings.json | 13 ++++- .../resources/web/settings/js/settings.js | 49 +++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 0143414db8..e9327d6a66 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1360,8 +1360,19 @@ { "name": "server_type", "label": "Server Type", + "type": "select", "placeholder": "Audio Mixer", - "can_set": true + "can_set": true, + "options": [ + { + "value": "Audio Mixer", + "label": "Audio Mixer" + }, + { + "value": "Avatar Mixer", + "label": "Avatar Mixer" + } + ] } ] } diff --git a/domain-server/resources/web/settings/js/settings.js b/domain-server/resources/web/settings/js/settings.js index 8066223318..2c42699523 100644 --- a/domain-server/resources/web/settings/js/settings.js +++ b/domain-server/resources/web/settings/js/settings.js @@ -1163,6 +1163,8 @@ function makeTable(setting, keypath, setting_value) { var isNonDeletableRow = !setting.can_add_new_rows; + // console.log(setting); + _.each(setting.columns, function(col) { var colValue, colName; @@ -1183,6 +1185,24 @@ function makeTable(setting, keypath, setting_value) { "" + ""; + } else if (isArray && col.type === "select" ) { + + //console.log("adding row"); + //console.log(col); + //console.log(setting); + + html += + "" + + ""; + html += ""; + + } else if (isArray && col.type === "time" && col.editable) { html += "" + @@ -1281,6 +1301,23 @@ function makeTableHiddenInputs(setting, initialValues, categoryValue) { "" + ""; + } else if (col.type === "select") { + + console.log(col); + console.log(setting); + console.log(categoryValue); + + html += "" + html += ""; + html += ""; + } else { html += " 1 ? "." + key : ""); + if (isCheckbox) { - input.attr("name", setting_name + "[" + row_index + "]" + (num_columns > 1 ? "." + key : "")) + input.attr("name", newName) } else { - input.attr("name", setting_name + "[" + row_index + "]" + (num_columns > 1 ? "." + key : "")) + if(isDropdown) { + $(element).children("select").attr("data-hidden-input", newName); + } + input.attr("name", newName); } } else { // because the name of the setting in question requires the key // setup a hook to change the HTML name of the element whenever the key changes + var colName = $(element).attr("name"); keyInput.on('change', function(){ input.attr("name", setting_name + "." + $(this).val() + "." + colName); From 4080814df8cdc1466673f47941021c601f2edf49 Mon Sep 17 00:00:00 2001 From: seefo Date: Tue, 13 Jun 2017 16:32:51 -0700 Subject: [PATCH 071/115] Cleaned up dropdown support for tables --- .../resources/web/settings/js/settings.js | 41 +++++-------------- 1 file changed, 10 insertions(+), 31 deletions(-) diff --git a/domain-server/resources/web/settings/js/settings.js b/domain-server/resources/web/settings/js/settings.js index 2c42699523..3b8da0a849 100644 --- a/domain-server/resources/web/settings/js/settings.js +++ b/domain-server/resources/web/settings/js/settings.js @@ -1163,8 +1163,6 @@ function makeTable(setting, keypath, setting_value) { var isNonDeletableRow = !setting.can_add_new_rows; - // console.log(setting); - _.each(setting.columns, function(col) { var colValue, colName; @@ -1185,24 +1183,6 @@ function makeTable(setting, keypath, setting_value) { "" + ""; - } else if (isArray && col.type === "select" ) { - - //console.log("adding row"); - //console.log(col); - //console.log(setting); - - html += - "" + - ""; - html += ""; - - } else if (isArray && col.type === "time" && col.editable) { html += "" + @@ -1302,13 +1282,8 @@ function makeTableHiddenInputs(setting, initialValues, categoryValue) { "name='" + col.name + "'" + (defaultValue ? " checked" : "") + "/>" + ""; } else if (col.type === "select") { - - console.log(col); - console.log(setting); - console.log(categoryValue); - html += "" - html += "'" for(var i in col.options) { var option = col.options[i]; @@ -1316,8 +1291,7 @@ function makeTableHiddenInputs(setting, initialValues, categoryValue) { } html += ""; - html += ""; - + html += ""; } else { html += " 1 ? "." + key : ""); - + if (isCheckbox) { input.attr("name", newName) } else { - if(isDropdown) { + if(isDropdown) { $(element).children("select").attr("data-hidden-input", newName); } input.attr("name", newName); @@ -1467,7 +1441,6 @@ function addTableRow(row) { } else { // because the name of the setting in question requires the key // setup a hook to change the HTML name of the element whenever the key changes - var colName = $(element).attr("name"); keyInput.on('change', function(){ input.attr("name", setting_name + "." + $(this).val() + "." + colName); @@ -1478,6 +1451,12 @@ function addTableRow(row) { input.focus(); focusChanged = true; } + + // if we are adding a dropdown, we should go ahead and make its select + // element is visible + if(isDropdown) { + $(element).children("select").attr("style", ""); + } if (isCheckbox) { $(input).find("input").attr("data-changed", "true"); From 2d9e92d1218bfa60184ffc176cff5348b3357864 Mon Sep 17 00:00:00 2001 From: seefo Date: Tue, 13 Jun 2017 18:08:26 -0700 Subject: [PATCH 072/115] Made requested changes to PR10683 --- domain-server/resources/web/settings/js/settings.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/domain-server/resources/web/settings/js/settings.js b/domain-server/resources/web/settings/js/settings.js index 3b8da0a849..d35fbf0027 100644 --- a/domain-server/resources/web/settings/js/settings.js +++ b/domain-server/resources/web/settings/js/settings.js @@ -1285,7 +1285,7 @@ function makeTableHiddenInputs(setting, initialValues, categoryValue) { html += "" html += "'" for (var i in col.options) { - var option = col.options[i]; + var option = col.options[i]; html += ""; } - html += ""; - html += ""; + html += ""; + html += ""; } else { html += " Date: Wed, 14 Jun 2017 17:20:37 -0700 Subject: [PATCH 105/115] change new Node flag to isForcedNeverSilent --- domain-server/src/DomainServer.cpp | 2 +- libraries/networking/src/LimitedNodeList.cpp | 2 +- libraries/networking/src/NetworkPeer.h | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 7907d8679f..be75ad61ee 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2267,7 +2267,7 @@ void DomainServer::updateDownstreamNodes() { // manually add the downstream node to our node list auto node = nodeList->addOrUpdateNode(QUuid::createUuid(), downstreamNodeType, downstreamServerAddr, downstreamServerAddr); - node->setIsForcedAlive(true); + node->setIsForcedNeverSilent(true); qDebug() << "Adding downstream node:" << node->getUUID() << downstreamServerAddr; diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 994b5305e6..14a9d0b257 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -749,7 +749,7 @@ void LimitedNodeList::removeSilentNodes() { SharedNodePointer node = it->second; node->getMutex().lock(); - if (!node->isForcedAlive() + if (!node->isForcedNeverSilent() && (usecTimestampNow() - node->getLastHeardMicrostamp()) > (NODE_SILENCE_THRESHOLD_MSECS * USECS_PER_MSEC)) { // call the NodeHash erase to get rid of this node it = _nodeHash.unsafe_erase(it); diff --git a/libraries/networking/src/NetworkPeer.h b/libraries/networking/src/NetworkPeer.h index ad7076285f..48dd82ecb7 100644 --- a/libraries/networking/src/NetworkPeer.h +++ b/libraries/networking/src/NetworkPeer.h @@ -76,8 +76,8 @@ public: float getOutboundBandwidth() const; // in kbps float getInboundBandwidth() const; // in kbps - bool isForcedAlive() const { return _isForcedAlive; } - void setIsForcedAlive(bool isForcedAlive) { _isForcedAlive = isForcedAlive; } + bool isForcedNeverSilent() const { return _isForcedNeverSilent; } + void setIsForcedNeverSilent(bool isForcedNeverSilent) { _isForcedNeverSilent = isForcedNeverSilent; } friend QDataStream& operator<<(QDataStream& out, const NetworkPeer& peer); friend QDataStream& operator>>(QDataStream& in, NetworkPeer& peer); @@ -107,7 +107,7 @@ protected: int _connectionAttempts; - bool _isForcedAlive { false }; + bool _isForcedNeverSilent { false }; }; QDebug operator<<(QDebug debug, const NetworkPeer &peer); From 94abfc2d6b7e6ca58f352e0cab8f501ae55550e1 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 15 Jun 2017 11:47:13 -0700 Subject: [PATCH 106/115] nomenclature changes from replication to broadcast --- domain-server/resources/describe-settings.json | 14 +++++++------- domain-server/src/DomainServer.cpp | 6 +++--- domain-server/src/DomainServerSettingsManager.cpp | 7 ++++--- libraries/networking/src/ThreadedAssignment.cpp | 6 +++--- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 020262ca13..10da5b55fc 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1320,15 +1320,15 @@ ] }, { - "name": "replication", - "label": "Replication", + "name": "broadcasting", + "label": "Broadcasting", "settings": [ { "name": "users", - "label": "Replicated Users", + "label": "Broadcasted Users", "type": "table", "can_add_new_rows": true, - "help": "Users that are replicated to downstream servers", + "help": "Users that are broadcasted to downstream servers", "numbered": false, "columns": [ { @@ -1340,11 +1340,11 @@ }, { "name": "downstream_servers", - "label": "Downstream Servers", + "label": "Receiving Servers", "assignment-types": [0,1], "type": "table", "can_add_new_rows": true, - "help": "Downstream servers that are relayed data for replicated users", + "help": "Servers that receive data for broadcasted users", "numbered": false, "columns": [ { @@ -1364,7 +1364,7 @@ "placeholder": "Audio Mixer", "default": "Audio Mixer", "can_set": true, - "options": [ + "options": [ { "value": "Audio Mixer", "label": "Audio Mixer" diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 0c0f8a5e55..81f10d83c6 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2220,11 +2220,11 @@ void DomainServer::refreshStaticAssignmentAndAddToQueue(SharedAssignmentPointer& } void DomainServer::updateReplicatedNodes() { - static const QString REPLICATION_SETTINGS_KEY = "replication"; + static const QString BROADCASTING_SETTINGS_KEY = "broadcasting"; _replicatedUsernames.clear(); auto settings = _settingsManager.getSettingsMap(); - if (settings.contains(REPLICATION_SETTINGS_KEY)) { - auto replicationSettings = settings.value(REPLICATION_SETTINGS_KEY).toMap(); + if (settings.contains(BROADCASTING_SETTINGS_KEY)) { + auto replicationSettings = settings.value(BROADCASTING_SETTINGS_KEY).toMap(); if (replicationSettings.contains("users")) { auto usersSettings = replicationSettings.value("users").toList(); for (auto& username : usersSettings) { diff --git a/domain-server/src/DomainServerSettingsManager.cpp b/domain-server/src/DomainServerSettingsManager.cpp index 3d256bd2b9..9279648319 100644 --- a/domain-server/src/DomainServerSettingsManager.cpp +++ b/domain-server/src/DomainServerSettingsManager.cpp @@ -1197,7 +1197,7 @@ QJsonObject DomainServerSettingsManager::settingDescriptionFromGroup(const QJson bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject) { static const QString SECURITY_ROOT_KEY = "security"; static const QString AC_SUBNET_WHITELIST_KEY = "ac_subnet_whitelist"; - static const QString REPLICATION_KEY = "replication"; + static const QString BROADCASTING_KEY = "broadcasting"; auto& settingsVariant = _configMap.getConfig(); bool needRestart = false; @@ -1249,7 +1249,7 @@ bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJ if (!matchingDescriptionObject.isEmpty()) { updateSetting(rootKey, rootValue, *thisMap, matchingDescriptionObject); - if (rootKey != SECURITY_ROOT_KEY && rootKey != REPLICATION_KEY) { + if (rootKey != SECURITY_ROOT_KEY && rootKey != BROADCASTING_KEY) { needRestart = true; } } else { @@ -1265,7 +1265,8 @@ bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJ if (!matchingDescriptionObject.isEmpty()) { const QJsonValue& settingValue = rootValue.toObject()[settingKey]; updateSetting(settingKey, settingValue, *thisMap, matchingDescriptionObject); - if ((rootKey != SECURITY_ROOT_KEY && rootKey != REPLICATION_KEY) || settingKey == AC_SUBNET_WHITELIST_KEY) { + if ((rootKey != SECURITY_ROOT_KEY && rootKey != BROADCASTING_KEY) + || settingKey == AC_SUBNET_WHITELIST_KEY) { needRestart = true; } } else { diff --git a/libraries/networking/src/ThreadedAssignment.cpp b/libraries/networking/src/ThreadedAssignment.cpp index 3e679f643a..430775a7ac 100644 --- a/libraries/networking/src/ThreadedAssignment.cpp +++ b/libraries/networking/src/ThreadedAssignment.cpp @@ -135,10 +135,10 @@ void ThreadedAssignment::domainSettingsRequestFailed() { } void ThreadedAssignment::parseDownstreamServers(const QJsonObject& settingsObject, NodeType_t nodeType, DownstreamNodeFoundCallback callback) { - static const QString REPLICATION_GROUP_KEY = "replication"; + static const QString BROADCASTING_GROUP_KEY = "broadcasting"; static const QString DOWNSTREAM_SERVERS_SETTING_KEY = "downstream_servers"; - if (settingsObject.contains(REPLICATION_GROUP_KEY)) { - const QJsonObject replicationObject = settingsObject[REPLICATION_GROUP_KEY].toObject(); + if (settingsObject.contains(BROADCASTING_GROUP_KEY)) { + const QJsonObject replicationObject = settingsObject[BROADCASTING_GROUP_KEY].toObject(); const QJsonArray downstreamServers = replicationObject[DOWNSTREAM_SERVERS_SETTING_KEY].toArray(); From 6cf4ff1dafe54a9a3c5fae656cdf5c23d51a915f Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 15 Jun 2017 13:44:44 -0700 Subject: [PATCH 107/115] remove debug and fix peek of session ID --- assignment-client/src/avatars/AvatarMixerSlave.cpp | 1 - libraries/avatars/src/AvatarHashMap.cpp | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 17d222e9e3..4d5e507923 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -479,7 +479,6 @@ void AvatarMixerSlave::broadcastAvatarDataToDownstreamMixer(const SharedNodePoin auto lastBroadcastTime = nodeData->getLastBroadcastTime(agentNode->getUUID()); if (lastBroadcastTime <= agentNodeData->getIdentityChangeTimestamp() || (start - lastBroadcastTime) >= REBROADCAST_IDENTITY_TO_DOWNSTREAM_EVERY_US) { - qDebug() << "Sending identity packet for " << agentNode->getUUID() << " to " << node->getUUID(); sendReplicatedIdentityPacket(agentNodeData, node); nodeData->setLastBroadcastTime(agentNode->getUUID(), start); } diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index cfbf2a8806..023bb39201 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -128,7 +128,7 @@ AvatarSharedPointer AvatarHashMap::parseAvatarData(QSharedPointer message, SharedNodePointer sendingNode) { // peek the avatar UUID from the incoming packet - QUuid identityUUID = message->peek(NUM_BYTES_RFC4122_UUID); + QUuid identityUUID = QUuid::fromRfc4122(message->peek(NUM_BYTES_RFC4122_UUID)); // make sure this isn't for an ignored avatar auto nodeList = DependencyManager::get(); From 8f154321a4727cbfde9d156cf5ab574aaf01b099 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 15 Jun 2017 14:25:10 -0700 Subject: [PATCH 108/115] never process null avatar ID in avatar manager --- libraries/avatars/src/AvatarHashMap.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libraries/avatars/src/AvatarHashMap.cpp b/libraries/avatars/src/AvatarHashMap.cpp index 023bb39201..06ead63de1 100644 --- a/libraries/avatars/src/AvatarHashMap.cpp +++ b/libraries/avatars/src/AvatarHashMap.cpp @@ -130,6 +130,11 @@ void AvatarHashMap::processAvatarIdentityPacket(QSharedPointer // peek the avatar UUID from the incoming packet QUuid identityUUID = QUuid::fromRfc4122(message->peek(NUM_BYTES_RFC4122_UUID)); + if (identityUUID.isNull()) { + qCDebug(avatars) << "Refusing to process identity packet for null avatar ID"; + return; + } + // make sure this isn't for an ignored avatar auto nodeList = DependencyManager::get(); static auto EMPTY = QUuid(); From 330a27b3a089bc64db1edd9adc16076fbc068aee Mon Sep 17 00:00:00 2001 From: Ryan Huffman Date: Thu, 15 Jun 2017 14:25:21 -0700 Subject: [PATCH 109/115] Fix removing all downstream nodes in settings not killing nodes --- domain-server/src/DomainServer.cpp | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index be75ad61ee..9a69e3077e 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2227,11 +2227,12 @@ static const QString REPLICATION_SETTINGS_KEY = "replication"; void DomainServer::updateDownstreamNodes() { auto settings = _settingsManager.getSettingsMap(); if (settings.contains(REPLICATION_SETTINGS_KEY)) { + auto nodeList = DependencyManager::get(); + std::vector downstreamNodesInSettings; auto replicationSettings = settings.value(REPLICATION_SETTINGS_KEY).toMap(); if (replicationSettings.contains("downstream_servers")) { auto serversSettings = replicationSettings.value("downstream_servers").toList(); - auto nodeList = DependencyManager::get(); std::vector knownDownstreamNodes; nodeList->eachNode([&](const SharedNodePointer& otherNode) { if (NodeType::isDownstream(otherNode->getType())) { @@ -2239,8 +2240,6 @@ void DomainServer::updateDownstreamNodes() { } }); - std::vector downstreamNodesInSettings; - for (auto& server : serversSettings) { auto downstreamServer = server.toMap(); @@ -2277,20 +2276,20 @@ void DomainServer::updateDownstreamNodes() { } } - std::vector nodesToKill; - nodeList->eachNode([&](const SharedNodePointer& otherNode) { - if (NodeType::isDownstream(otherNode->getType())) { - bool nodeInSettings = find(downstreamNodesInSettings.cbegin(), downstreamNodesInSettings.cend(), - otherNode->getPublicSocket()) != downstreamNodesInSettings.cend(); - if (!nodeInSettings) { - qDebug() << "Removing downstream node:" << otherNode->getUUID() << otherNode->getPublicSocket(); - nodesToKill.push_back(otherNode); - } + } + std::vector nodesToKill; + nodeList->eachNode([&](const SharedNodePointer& otherNode) { + if (NodeType::isDownstream(otherNode->getType())) { + bool nodeInSettings = find(downstreamNodesInSettings.cbegin(), downstreamNodesInSettings.cend(), + otherNode->getPublicSocket()) != downstreamNodesInSettings.cend(); + if (!nodeInSettings) { + qDebug() << "Removing downstream node:" << otherNode->getUUID() << otherNode->getPublicSocket(); + nodesToKill.push_back(otherNode); } - }); - for (auto& node : nodesToKill) { - handleKillNode(node); } + }); + for (auto& node : nodesToKill) { + handleKillNode(node); } } } From 0565cfa2b7f41a6cf0f0616607b87e9035b9221d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 15 Jun 2017 14:59:12 -0700 Subject: [PATCH 110/115] fix for array additions in DS settings --- .../resources/web/settings/js/settings.js | 49 +++++++++++-------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/domain-server/resources/web/settings/js/settings.js b/domain-server/resources/web/settings/js/settings.js index 2947983ef9..3196213e4a 100644 --- a/domain-server/resources/web/settings/js/settings.js +++ b/domain-server/resources/web/settings/js/settings.js @@ -39,7 +39,8 @@ var Settings = { ACCESS_TOKEN_SELECTOR: '[name="metaverse.access_token"]', PLACES_TABLE_ID: 'places-table', FORM_ID: 'settings-form', - INVALID_ROW_CLASS: 'invalid-input' + INVALID_ROW_CLASS: 'invalid-input', + DATA_ROW_INDEX: 'data-row-index' }; var viewHelpers = { @@ -223,10 +224,10 @@ $(document).ready(function(){ // set focus to the first input in the new row $target.closest('table').find('tr.inputs input:first').focus(); } - + var tableRows = sibling.parent(); var tableBody = tableRows.parent(); - + // if theres no more siblings, we should jump to a new row if (sibling.next().length == 0 && tableRows.nextAll().length == 1) { tableBody.find("." + Settings.ADD_ROW_BUTTON_CLASS).click(); @@ -1005,7 +1006,7 @@ function saveSettings() { var password = formJSON["security"]["http_password"]; var verify_password = formJSON["security"]["verify_http_password"]; - // if they've only emptied out the default password field, we should go ahead and acknowledge + // if they've only emptied out the default password field, we should go ahead and acknowledge // the verify password field if (password != undefined && verify_password == undefined) { verify_password = ""; @@ -1158,8 +1159,9 @@ function makeTable(setting, keypath, setting_value) { } html += ""; + (isCategorized ? ("data-category='" + categoryValue + "'") : "") + " " + + (isArray ? "" : "name='" + keypath + "." + rowIndexOrName + "'") + + (isArray ? Settings.DATA_ROW_INDEX + "='" + (row_num - 1) + "'" : "" ) + ">"; if (setting.numbered === true) { html += "" + row_num + "" @@ -1292,12 +1294,12 @@ function makeTableHiddenInputs(setting, initialValues, categoryValue) { } else if (col.type === "select") { html += "" html += ""; html += ""; } else { @@ -1398,6 +1400,15 @@ function addTableRow(row) { var setting_name = table.attr("name"); row.addClass(Settings.DATA_ROW_CLASS + " " + Settings.NEW_ROW_CLASS); + // if this is an array, add the row index (which is the index of the last row + 1) + // as a data attribute to the row + var row_index = 0; + if (isArray) { + var previous_row_index = parseInt(row.siblings('.' + Settings.DATA_ROW_CLASS + ':last').attr(Settings.DATA_ROW_INDEX), 10); + row_index = previous_row_index + 1; + row.attr(Settings.DATA_ROW_INDEX, row_index); + } + var focusChanged = false; _.each(row.children(), function(element) { @@ -1430,7 +1441,6 @@ function addTableRow(row) { var isDropdown = input.hasClass("table-dropdown"); if (isArray) { - var row_index = row.siblings('.' + Settings.DATA_ROW_CLASS).length var key = $(element).attr('name'); // are there multiple columns or just one? @@ -1438,16 +1448,13 @@ function addTableRow(row) { var num_columns = row.children('.' + Settings.DATA_COL_CLASS).length var newName = setting_name + "[" + row_index + "]" + (num_columns > 1 ? "." + key : ""); - if (isCheckbox) { - input.attr("name", newName) - } else { - if (isDropdown) { - // default values for hidden inputs inside child selects gets cleared so we need to remind it - var selectElement = $(element).children("select"); - selectElement.attr("data-hidden-input", newName); - $(element).children("input").val(selectElement.val()); - } - input.attr("name", newName); + input.attr("name", newName); + + if (isDropdown) { + // default values for hidden inputs inside child selects gets cleared so we need to remind it + var selectElement = $(element).children("select"); + selectElement.attr("data-hidden-input", newName); + $(element).children("input").val(selectElement.val()); } } else { // because the name of the setting in question requires the key @@ -1462,10 +1469,10 @@ function addTableRow(row) { input.focus(); focusChanged = true; } - + // if we are adding a dropdown, we should go ahead and make its select // element is visible - if (isDropdown) { + if (isDropdown) { $(element).children("select").attr("style", ""); } From d7724b90afa7de507fe45f7bf77aad6f7e249fa8 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 15 Jun 2017 15:01:03 -0700 Subject: [PATCH 111/115] address code review comments --- assignment-client/src/audio/AudioMixer.cpp | 2 +- assignment-client/src/avatars/AvatarMixer.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index b5faf71bde..76ce2061c0 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -103,7 +103,7 @@ AudioMixer::AudioMixer(ReceivedMessage& message) : ); connect(nodeList.data(), &NodeList::nodeKilled, this, &AudioMixer::handleNodeKilled); - connect(nodeList.data(), &NodeList::nodeAdded, this, [this](SharedNodePointer node) { + connect(nodeList.data(), &NodeList::nodeAdded, this, [this](const SharedNodePointer& node) { if (node->getType() == NodeType::DownstreamAudioMixer) { node->activatePublicSocket(); } diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 631145aee4..1ec3ed54c4 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -63,7 +63,7 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : auto nodeList = DependencyManager::get(); connect(nodeList.data(), &NodeList::packetVersionMismatch, this, &AvatarMixer::handlePacketVersionMismatch); - connect(nodeList.data(), &NodeList::nodeAdded, this, [this](SharedNodePointer node) { + connect(nodeList.data(), &NodeList::nodeAdded, this, [this](const SharedNodePointer& node) { if (node->getType() == NodeType::DownstreamAvatarMixer) { getOrCreateClientData(node); node->activatePublicSocket(); From 36b49901148c19f03d331133f60cbc7e608d7e83 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 15 Jun 2017 17:19:05 -0700 Subject: [PATCH 112/115] fix for table additions when there is no initial row --- domain-server/resources/web/settings/js/settings.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/domain-server/resources/web/settings/js/settings.js b/domain-server/resources/web/settings/js/settings.js index 3196213e4a..181e5baa21 100644 --- a/domain-server/resources/web/settings/js/settings.js +++ b/domain-server/resources/web/settings/js/settings.js @@ -1404,8 +1404,14 @@ function addTableRow(row) { // as a data attribute to the row var row_index = 0; if (isArray) { - var previous_row_index = parseInt(row.siblings('.' + Settings.DATA_ROW_CLASS + ':last').attr(Settings.DATA_ROW_INDEX), 10); - row_index = previous_row_index + 1; + var previous_row = row.siblings('.' + Settings.DATA_ROW_CLASS + ':last'); + + if (previous_row.length > 0) { + row_index = parseInt(previous_row.attr(Settings.DATA_ROW_INDEX), 10) + 1; + } else { + row_index = 0; + } + row.attr(Settings.DATA_ROW_INDEX, row_index); } From c332244f4f7ebe525f0d7ce973c28805f413d45d Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Thu, 15 Jun 2017 17:34:20 -0700 Subject: [PATCH 113/115] fix race for isUpstream and node hole punch --- assignment-client/src/audio/AudioMixer.cpp | 3 +-- assignment-client/src/avatars/AvatarMixer.cpp | 3 +-- libraries/networking/src/LimitedNodeList.cpp | 12 ++++++++---- libraries/networking/src/LimitedNodeList.h | 5 +++-- libraries/networking/src/Node.cpp | 8 ++------ libraries/networking/src/Node.h | 1 - libraries/networking/src/NodeList.cpp | 2 +- 7 files changed, 16 insertions(+), 18 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 76ce2061c0..d0afc7b0d4 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -126,9 +126,8 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess auto replicatedNode = nodeList->addOrUpdateNode(nodeID, NodeType::Agent, message->getSenderSockAddr(), message->getSenderSockAddr(), - DEFAULT_AGENT_PERMISSIONS, true); + true, true); replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); - replicatedNode->setIsUpstream(true); // construct a "fake" audio received message from the byte array and packet list information auto audioData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID); diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 4998a3b520..c4e59b6324 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -75,10 +75,9 @@ SharedNodePointer addOrUpdateReplicatedNode(const QUuid& nodeID, const HifiSockA auto replicatedNode = DependencyManager::get()->addOrUpdateNode(nodeID, NodeType::Agent, senderSockAddr, senderSockAddr, - DEFAULT_AGENT_PERMISSIONS, true); + true, true); replicatedNode->setLastHeardMicrostamp(usecTimestampNow()); - replicatedNode->setIsUpstream(true); return replicatedNode; } diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 14a9d0b257..d2bae28820 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -570,9 +570,8 @@ void LimitedNodeList::handleNodeKill(const SharedNodePointer& node) { SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, - const NodePermissions& permissions, - bool isReplicated, - const QUuid& connectionSecret) { + bool isReplicated, bool isUpstream, + const QUuid& connectionSecret, const NodePermissions& permissions) { QReadLocker readLocker(&_nodeMutex); NodeHash::const_iterator it = _nodeHash.find(uuid); @@ -584,11 +583,16 @@ SharedNodePointer LimitedNodeList::addOrUpdateNode(const QUuid& uuid, NodeType_t matchingNode->setPermissions(permissions); matchingNode->setConnectionSecret(connectionSecret); matchingNode->setIsReplicated(isReplicated); + matchingNode->setIsUpstream(isUpstream); return matchingNode; } else { // we didn't have this node, so add them - Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket, permissions, isReplicated, connectionSecret); + Node* newNode = new Node(uuid, nodeType, publicSocket, localSocket); + newNode->setIsReplicated(isReplicated); + newNode->setIsUpstream(isUpstream); + newNode->setConnectionSecret(connectionSecret); + newNode->setPermissions(permissions); // move the newly constructed node to the LNL thread newNode->moveToThread(thread()); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 5e2c88ed94..01e0ef2dc0 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -145,8 +145,9 @@ public: SharedNodePointer addOrUpdateNode(const QUuid& uuid, NodeType_t nodeType, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, - const NodePermissions& permissions = DEFAULT_AGENT_PERMISSIONS, - bool isReplicated = false, const QUuid& connectionSecret = QUuid()); + bool isReplicated = false, bool isUpstream = false, + const QUuid& connectionSecret = QUuid(), + const NodePermissions& permissions = DEFAULT_AGENT_PERMISSIONS); static bool parseSTUNResponse(udt::BasePacket* packet, QHostAddress& newPublicAddress, uint16_t& newPublicPort); bool hasCompletedInitialSTUN() const { return _hasCompletedInitialSTUN; } diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index f74be8adcd..ea1b6e0eb5 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -73,17 +73,13 @@ NodeType_t NodeType::fromString(QString type) { Node::Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, - const HifiSockAddr& localSocket, const NodePermissions& permissions, bool isReplicated, - const QUuid& connectionSecret, QObject* parent) : + const HifiSockAddr& localSocket, QObject* parent) : NetworkPeer(uuid, publicSocket, localSocket, parent), _type(type), - _connectionSecret(connectionSecret), - _isReplicated(isReplicated), _pingMs(-1), // "Uninitialized" _clockSkewUsec(0), _mutex(), - _clockSkewMovingPercentile(30, 0.8f), // moving 80th percentile of 30 samples - _permissions(permissions) + _clockSkewMovingPercentile(30, 0.8f) // moving 80th percentile of 30 samples { // Update socket's object name setType(_type); diff --git a/libraries/networking/src/Node.h b/libraries/networking/src/Node.h index 33c9e2c205..c20ff5a395 100644 --- a/libraries/networking/src/Node.h +++ b/libraries/networking/src/Node.h @@ -40,7 +40,6 @@ public: Node(const QUuid& uuid, NodeType_t type, const HifiSockAddr& publicSocket, const HifiSockAddr& localSocket, - const NodePermissions& permissions, bool isReplicated, const QUuid& connectionSecret = QUuid(), QObject* parent = nullptr); bool operator==(const Node& otherNode) const { return _uuid == otherNode._uuid; } diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 179b6e202d..f502cb2a3c 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -667,7 +667,7 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { packetStream >> connectionUUID; SharedNodePointer node = addOrUpdateNode(nodeUUID, nodeType, nodePublicSocket, - nodeLocalSocket, permissions, isReplicated, connectionUUID); + nodeLocalSocket, isReplicated, false, connectionUUID, permissions); // nodes that are downstream of our own type are kept alive when we hear about them from the domain server if (node->getType() == NodeType::downstreamType(_ownerType)) { From 2933a20a0c52231a1012c3449064c2827fec4aa9 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Mon, 19 Jun 2017 10:43:45 -0700 Subject: [PATCH 114/115] some cleanup for audio/avatar replicated packet checking --- assignment-client/src/audio/AudioMixer.cpp | 14 ++------- .../src/audio/AudioMixerClientData.cpp | 29 +++++++------------ .../src/audio/AudioMixerClientData.h | 1 - assignment-client/src/avatars/AvatarMixer.cpp | 18 ++++++++++-- .../networking/src/udt/PacketHeaders.cpp | 9 ++++++ libraries/networking/src/udt/PacketHeaders.h | 2 ++ 6 files changed, 39 insertions(+), 34 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index d0afc7b0d4..ab4b20b9cc 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -132,18 +132,10 @@ void AudioMixer::queueReplicatedAudioPacket(QSharedPointer mess // construct a "fake" audio received message from the byte array and packet list information auto audioData = message->getMessage().mid(NUM_BYTES_RFC4122_UUID); - PacketType rewrittenType; + PacketType rewrittenType = REPLICATED_PACKET_MAPPING.key(message->getType()); - if (message->getType() == PacketType::ReplicatedMicrophoneAudioNoEcho) { - rewrittenType = PacketType::MicrophoneAudioNoEcho; - } else if (message->getType() == PacketType::ReplicatedMicrophoneAudioWithEcho) { - rewrittenType = PacketType::MicrophoneAudioWithEcho; - } else if (message->getType() == PacketType::ReplicatedInjectAudio) { - rewrittenType = PacketType::InjectAudio; - } else if (message->getType() == PacketType::ReplicatedSilentAudioFrame) { - rewrittenType = PacketType::SilentAudioFrame; - } else { - return; + if (rewrittenType == PacketType::Unknown) { + qDebug() << "Cannot unwrap replicated packet type not present in REPLICATED_PACKET_WRAPPING"; } auto replicatedMessage = QSharedPointer::create(audioData, rewrittenType, diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 7641000a7a..6e11257c7f 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -124,27 +124,18 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c auto nodeList = DependencyManager::get(); // now make sure it's a packet type that we want to replicate - PacketType mirroredType; - switch (message.getType()) { - case PacketType::MicrophoneAudioNoEcho: - case PacketType::ReplicatedMicrophoneAudioNoEcho: - mirroredType = PacketType::ReplicatedMicrophoneAudioNoEcho; - break; - case PacketType::MicrophoneAudioWithEcho: - case PacketType::ReplicatedMicrophoneAudioWithEcho: - mirroredType = PacketType::ReplicatedMicrophoneAudioWithEcho; - break; - case PacketType::InjectAudio: - case PacketType::ReplicatedInjectAudio: - mirroredType = PacketType::ReplicatedInjectAudio; - break; - case PacketType::SilentAudioFrame: - case PacketType::ReplicatedSilentAudioFrame: - mirroredType = PacketType::ReplicatedSilentAudioFrame; - break; - default: + // first check if it is an original type that we should replicate + PacketType mirroredType = REPLICATED_PACKET_MAPPING.value(message.getType()); + + if (mirroredType == PacketType::Unknown) { + // if it wasn't check if it is a replicated type that we should re-replicate + if (REPLICATED_PACKET_MAPPING.key(message.getType()) != PacketType::Unknown) { + mirroredType = message.getType(); + } else { + qDebug() << "Packet passed to optionallyReplicatePacket was not a replicatable type - returning"; return; + } } std::unique_ptr packet; diff --git a/assignment-client/src/audio/AudioMixerClientData.h b/assignment-client/src/audio/AudioMixerClientData.h index b397ab0b8e..76a9cd6aa7 100644 --- a/assignment-client/src/audio/AudioMixerClientData.h +++ b/assignment-client/src/audio/AudioMixerClientData.h @@ -26,7 +26,6 @@ #include "PositionalAudioStream.h" #include "AvatarAudioStream.h" - class AudioMixerClientData : public NodeData { Q_OBJECT public: diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index c4e59b6324..136d5f2e8e 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -127,9 +127,21 @@ void AvatarMixer::handleReplicatedBulkAvatarPacket(QSharedPointer packet; diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index 7985df58bf..fb416cac4d 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -44,6 +44,15 @@ const QSet NON_SOURCED_PACKETS = QSet() << PacketType::ReplicatedInjectAudio << PacketType::ReplicatedSilentAudioFrame << PacketType::ReplicatedAvatarIdentity << PacketType::ReplicatedKillAvatar << PacketType::ReplicatedBulkAvatarData; +const QHash REPLICATED_PACKET_MAPPING { + { PacketType::MicrophoneAudioNoEcho, PacketType::ReplicatedMicrophoneAudioNoEcho }, + { PacketType::MicrophoneAudioWithEcho, PacketType::ReplicatedMicrophoneAudioWithEcho }, + { PacketType::InjectAudio, PacketType::ReplicatedInjectAudio }, + { PacketType::SilentAudioFrame, PacketType::ReplicatedSilentAudioFrame }, + { PacketType::AvatarIdentity, PacketType::ReplicatedAvatarIdentity }, + { PacketType::KillAvatar, PacketType::ReplicatedKillAvatar }, +}; + PacketVersion versionForPacketType(PacketType packetType) { switch (packetType) { case PacketType::DomainList: diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index c080ab8e19..2944c1ce93 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -128,6 +128,8 @@ public: using PacketType = PacketTypeEnum::Value; +extern const QHash REPLICATED_PACKET_MAPPING; + const int NUM_BYTES_MD5_HASH = 16; typedef char PacketVersion; From aeb3f443f81da91a689bd31e9580761dd3a863b4 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Wed, 21 Jun 2017 16:14:14 -0700 Subject: [PATCH 115/115] address code review comments --- .../src/audio/AudioMixerClientData.cpp | 3 +-- assignment-client/src/avatars/AvatarMixer.cpp | 6 ++--- assignment-client/src/avatars/AvatarMixer.h | 2 +- .../src/avatars/AvatarMixerSlave.cpp | 4 +-- domain-server/src/DomainServer.cpp | 26 ++++++++++++------- libraries/avatars/src/AvatarData.cpp | 24 ++++++----------- libraries/avatars/src/AvatarData.h | 5 ++-- libraries/networking/src/LimitedNodeList.h | 5 ---- libraries/networking/src/NetworkPeer.h | 5 ++++ 9 files changed, 37 insertions(+), 43 deletions(-) diff --git a/assignment-client/src/audio/AudioMixerClientData.cpp b/assignment-client/src/audio/AudioMixerClientData.cpp index 6e11257c7f..bf19a02d9a 100644 --- a/assignment-client/src/audio/AudioMixerClientData.cpp +++ b/assignment-client/src/audio/AudioMixerClientData.cpp @@ -121,8 +121,6 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c // first, make sure that this is a packet from a node we are supposed to replicate if (node.isReplicated()) { - auto nodeList = DependencyManager::get(); - // now make sure it's a packet type that we want to replicate // first check if it is an original type that we should replicate @@ -139,6 +137,7 @@ void AudioMixerClientData::optionallyReplicatePacket(ReceivedMessage& message, c } std::unique_ptr packet; + auto nodeList = DependencyManager::get(); // enumerate the downstream audio mixers and send them the replicated version of this packet nodeList->unsafeEachNode([&](const SharedNodePointer& downstreamNode) { diff --git a/assignment-client/src/avatars/AvatarMixer.cpp b/assignment-client/src/avatars/AvatarMixer.cpp index 136d5f2e8e..f1e30f4442 100644 --- a/assignment-client/src/avatars/AvatarMixer.cpp +++ b/assignment-client/src/avatars/AvatarMixer.cpp @@ -57,7 +57,7 @@ AvatarMixer::AvatarMixer(ReceivedMessage& message) : packetReceiver.registerListenerForTypes({ PacketType::ReplicatedAvatarIdentity, PacketType::ReplicatedKillAvatar - }, this, "handleReplicatedPackets"); + }, this, "handleReplicatedPacket"); packetReceiver.registerListener(PacketType::ReplicatedBulkAvatarData, this, "handleReplicatedBulkAvatarPacket"); @@ -82,7 +82,7 @@ SharedNodePointer addOrUpdateReplicatedNode(const QUuid& nodeID, const HifiSockA return replicatedNode; } -void AvatarMixer::handleReplicatedPackets(QSharedPointer message) { +void AvatarMixer::handleReplicatedPacket(QSharedPointer message) { auto nodeList = DependencyManager::get(); auto nodeID = QUuid::fromRfc4122(message->peek(NUM_BYTES_RFC4122_UUID)); @@ -96,8 +96,6 @@ void AvatarMixer::handleReplicatedPackets(QSharedPointer messag } void AvatarMixer::handleReplicatedBulkAvatarPacket(QSharedPointer message) { - auto nodeList = DependencyManager::get(); - while (message->getBytesLeftToRead()) { // first, grab the node ID for this replicated avatar auto nodeID = QUuid::fromRfc4122(message->readWithoutCopy(NUM_BYTES_RFC4122_UUID)); diff --git a/assignment-client/src/avatars/AvatarMixer.h b/assignment-client/src/avatars/AvatarMixer.h index 42414dab55..58d4487652 100644 --- a/assignment-client/src/avatars/AvatarMixer.h +++ b/assignment-client/src/avatars/AvatarMixer.h @@ -46,7 +46,7 @@ private slots: void handleNodeIgnoreRequestPacket(QSharedPointer message, SharedNodePointer senderNode); void handleRadiusIgnoreRequestPacket(QSharedPointer packet, SharedNodePointer sendingNode); void handleRequestsDomainListDataPacket(QSharedPointer message, SharedNodePointer senderNode); - void handleReplicatedPackets(QSharedPointer message); + void handleReplicatedPacket(QSharedPointer message); void handleReplicatedBulkAvatarPacket(QSharedPointer message); void domainSettingsRequestComplete(); void handlePacketVersionMismatch(PacketType type, const HifiSockAddr& senderSockAddr, const QUuid& senderUUID); diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp index 4d5e507923..8092518454 100644 --- a/assignment-client/src/avatars/AvatarMixerSlave.cpp +++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp @@ -67,7 +67,7 @@ void AvatarMixerSlave::processIncomingPackets(const SharedNodePointer& node) { int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { if (destinationNode->getType() == NodeType::Agent && !destinationNode->isUpstream()) { - QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(true); + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious auto identityPackets = NLPacketList::create(PacketType::AvatarIdentity, QByteArray(), true, true); identityPackets->write(individualData); @@ -81,7 +81,7 @@ int AvatarMixerSlave::sendIdentityPacket(const AvatarMixerClientData* nodeData, int AvatarMixerSlave::sendReplicatedIdentityPacket(const AvatarMixerClientData* nodeData, const SharedNodePointer& destinationNode) { if (destinationNode->getType() == NodeType::DownstreamAvatarMixer) { - QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(true); + QByteArray individualData = nodeData->getConstAvatarData()->identityByteArray(); individualData.replace(0, NUM_BYTES_RFC4122_UUID, nodeData->getNodeID().toRfc4122()); // FIXME, this looks suspicious auto identityPacket = NLPacket::create(PacketType::ReplicatedAvatarIdentity); identityPacket->write(individualData); diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 856a568ca7..1f903ea1bc 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -2277,6 +2277,10 @@ void DomainServer::updateDownstreamNodes() { } } + + // enumerate the nodes to determine which are no longer downstream for this domain + // collect them in a vector to separately remove them with handleKillNode (since eachNode has a read lock and + // we cannot recursively take the write lock required by handleKillNode) std::vector nodesToKill; nodeList->eachNode([&](const SharedNodePointer& otherNode) { if (NodeType::isDownstream(otherNode->getType())) { @@ -2288,6 +2292,7 @@ void DomainServer::updateDownstreamNodes() { } } }); + for (auto& node : nodesToKill) { handleKillNode(node); } @@ -2296,12 +2301,11 @@ void DomainServer::updateDownstreamNodes() { void DomainServer::updateReplicatedNodes() { // Make sure we have downstream nodes in our list - // TODO Move this to a different function - _replicatedUsernames.clear(); auto settings = _settingsManager.getSettingsMap(); static const QString REPLICATED_USERS_KEY = "users"; _replicatedUsernames.clear(); + if (settings.contains(BROADCASTING_SETTINGS_KEY)) { auto replicationSettings = settings.value(BROADCASTING_SETTINGS_KEY).toMap(); if (replicationSettings.contains(REPLICATED_USERS_KEY)) { @@ -2319,9 +2323,6 @@ void DomainServer::updateReplicatedNodes() { auto shouldReplicate = shouldReplicateNode(*otherNode); auto isReplicated = otherNode->isReplicated(); if (isReplicated && !shouldReplicate) { - qDebug() << "Setting node to NOT be replicated:" << otherNode->getUUID(); - } else if (!isReplicated && shouldReplicate) { - qDebug() << "Setting node to replicated:" << otherNode->getUUID(); qDebug() << "Setting node to NOT be replicated:" << otherNode->getPermissions().getVerifiedUserName() << otherNode->getUUID(); } else if (!isReplicated && shouldReplicate) { @@ -2334,11 +2335,16 @@ void DomainServer::updateReplicatedNodes() { } bool DomainServer::shouldReplicateNode(const Node& node) { - QString verifiedUsername = node.getPermissions().getVerifiedUserName(); - // Both the verified username and usernames in _replicatedUsernames are lowercase, so - // comparisons here are case-insensitive. - auto it = find(_replicatedUsernames.cbegin(), _replicatedUsernames.cend(), verifiedUsername); - return it != _replicatedUsernames.end() && node.getType() == NodeType::Agent; + if (node.getType() == NodeType::Agent) { + QString verifiedUsername = node.getPermissions().getVerifiedUserName(); + + // Both the verified username and usernames in _replicatedUsernames are lowercase, so + // comparisons here are case-insensitive. + auto it = find(_replicatedUsernames.cbegin(), _replicatedUsernames.cend(), verifiedUsername); + return it != _replicatedUsernames.end(); + } else { + return false; + } }; void DomainServer::nodeAdded(SharedNodePointer node) { diff --git a/libraries/avatars/src/AvatarData.cpp b/libraries/avatars/src/AvatarData.cpp index 6ec2b45c89..eb4a02cb62 100644 --- a/libraries/avatars/src/AvatarData.cpp +++ b/libraries/avatars/src/AvatarData.cpp @@ -1496,13 +1496,13 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide udt::SequenceNumber incomingSequenceNumber(incomingSequenceNumberType); if (!_hasProcessedFirstIdentity) { - _lastIncomingSequenceNumber = incomingSequenceNumber - 1; + _lastSequenceNumber = incomingSequenceNumber - 1; _hasProcessedFirstIdentity = true; qCDebug(avatars) << "Processing first identity packet for" << avatarSessionID << "-" << (udt::SequenceNumber::Type) incomingSequenceNumber; } - if (incomingSequenceNumber > _lastIncomingSequenceNumber) { + if (incomingSequenceNumber > _lastSequenceNumber) { Identity identity; packetStream >> identity.skeletonModelURL @@ -1512,7 +1512,7 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide >> identity.avatarEntityData; // set the store identity sequence number to match the incoming identity - _lastIncomingSequenceNumber = incomingSequenceNumber; + _lastSequenceNumber = incomingSequenceNumber; if (_firstSkeletonCheck || (identity.skeletonModelURL != cannonicalSkeletonModelURL(emptyURL))) { setSkeletonModelURL(identity.skeletonModelURL); @@ -1552,34 +1552,26 @@ void AvatarData::processAvatarIdentity(const QByteArray& identityData, bool& ide << "identity.skeletonModelURL:" << identity.skeletonModelURL << "identity.displayName:" << identity.displayName << "identity.sessionDisplayName:" << identity.sessionDisplayName; -#endif - } else { -#ifdef WANT_DEBUG + qCDebug(avatars) << "Refusing to process identity for" << uuidStringWithoutCurlyBraces(avatarSessionID) << "since" - << (udt::SequenceNumber::Type) _lastIncomingSequenceNumber + << (udt::SequenceNumber::Type) _lastSequenceNumber << "is >=" << (udt::SequenceNumber::Type) incomingSequenceNumber; #endif } } -QByteArray AvatarData::identityByteArray(bool shouldForwardIncomingSequenceNumber) const { +QByteArray AvatarData::identityByteArray() const { QByteArray identityData; QDataStream identityStream(&identityData, QIODevice::Append); const QUrl& urlToSend = cannonicalSkeletonModelURL(emptyURL); // depends on _skeletonModelURL - // we use the boolean flag to determine if this is an identity byte array for a mixer to send to an agent - // or an agent to send to a mixer - // when mixers send identity packets to agents, they simply forward along the last incoming sequence number they received // whereas agents send a fresh outgoing sequence number when identity data has changed - udt::SequenceNumber identitySequenceNumber = - shouldForwardIncomingSequenceNumber ? _lastIncomingSequenceNumber : _lastOutgoingSequenceNumber; - _avatarEntitiesLock.withReadLock([&] { identityStream << getSessionUUID() - << (udt::SequenceNumber::Type) identitySequenceNumber + << (udt::SequenceNumber::Type) _lastSequenceNumber << urlToSend << _attachmentData << _displayName @@ -1763,7 +1755,7 @@ void AvatarData::sendIdentityPacket() { if (_identityDataChanged) { // if the identity data has changed, push the sequence number forwards - ++_lastOutgoingSequenceNumber; + ++_lastSequenceNumber; } QByteArray identityData = identityByteArray(); diff --git a/libraries/avatars/src/AvatarData.h b/libraries/avatars/src/AvatarData.h index 44d910b571..8d09f936b5 100644 --- a/libraries/avatars/src/AvatarData.h +++ b/libraries/avatars/src/AvatarData.h @@ -539,7 +539,7 @@ public: void processAvatarIdentity(const QByteArray& identityData, bool& identityChanged, bool& displayNameChanged, bool& skeletonModelUrlChanged); - QByteArray identityByteArray(bool shouldForwardIncomingSequenceNumber = false) const; + QByteArray identityByteArray() const; const QUrl& getSkeletonModelURL() const { return _skeletonModelURL; } const QString& getDisplayName() const { return _displayName; } @@ -781,8 +781,7 @@ protected: float _audioAverageLoudness { 0.0f }; bool _identityDataChanged { false }; - udt::SequenceNumber _lastIncomingSequenceNumber { 0 }; - udt::SequenceNumber _lastOutgoingSequenceNumber { 0 }; + udt::SequenceNumber _lastSequenceNumber { 0 }; bool _hasProcessedFirstIdentity { false }; float _density; diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 01e0ef2dc0..ab61c71952 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -289,9 +289,6 @@ public: void sendFakedHandshakeRequestToNode(SharedNodePointer node); #endif - void setMirrorSocket(const HifiSockAddr& mirrorSocket) { _mirrorSocket = mirrorSocket; } - const HifiSockAddr& getMirrorSocket() { return _mirrorSocket; } - public slots: void reset(); void eraseAllNodes(); @@ -401,8 +398,6 @@ protected: } - HifiSockAddr _mirrorSocket; - private slots: void flagTimeForConnectionStep(ConnectionStep connectionStep, quint64 timestamp); void possiblyTimeoutSTUNAddressLookup(); diff --git a/libraries/networking/src/NetworkPeer.h b/libraries/networking/src/NetworkPeer.h index 48dd82ecb7..9842768b37 100644 --- a/libraries/networking/src/NetworkPeer.h +++ b/libraries/networking/src/NetworkPeer.h @@ -76,6 +76,11 @@ public: float getOutboundBandwidth() const; // in kbps float getInboundBandwidth() const; // in kbps + // Typically the LimitedNodeList removes nodes after they are "silent" + // meaning that we have not received any packets (including simple keepalive pings) from them for a set interval. + // The _isForcedNeverSilent flag tells the LimitedNodeList that a Node should never be killed by removeSilentNodes() + // even if its the timestamp of when it was last heard from has never been updated. + bool isForcedNeverSilent() const { return _isForcedNeverSilent; } void setIsForcedNeverSilent(bool isForcedNeverSilent) { _isForcedNeverSilent = isForcedNeverSilent; }