From 8cd5ec3b54e4e5b0f1cdebf2bf20833e8ebcac2b Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 6 Jun 2014 11:50:34 -0700 Subject: [PATCH 1/9] started work on sending nack packets from client --- .../src/octree/OctreeSendThread.cpp | 4 +- interface/src/Application.cpp | 59 ++++++++++++++++++- interface/src/Application.h | 6 ++ libraries/octree/src/OctreeSceneStats.cpp | 15 +++++ libraries/octree/src/OctreeSceneStats.h | 4 ++ 5 files changed, 85 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index befea80380..032b45fcdf 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -288,12 +288,12 @@ int OctreeSendThread::handlePacketSend(OctreeQueryNode* nodeData, int& trueBytes int OctreeSendThread::resendNackedPackets(OctreeQueryNode* nodeData) { - const int maxPacketsSent = 10; + const int MAX_PACKETS_RESEND = 10; int packetsSent = 0; const QByteArray* packet; - while (nodeData->hasNextNackedPacket() && packetsSent < maxPacketsSent) { + while (nodeData->hasNextNackedPacket() && packetsSent < MAX_PACKETS_RESEND) { packet = nodeData->getNextNackedPacket(); // packet will be NULL if it's not in nodeData's packet history if (packet) { diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index de3c354b27..358bb1bde3 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -167,7 +167,8 @@ Application::Application(int& argc, char** argv, QElapsedTimer &startup_time) : _applicationOverlay(), _runningScriptsWidget(new RunningScriptsWidget(_window)), _runningScriptsWidgetWasVisible(false), - _trayIcon(new QSystemTrayIcon(_window)) + _trayIcon(new QSystemTrayIcon(_window)), + _lastNackTime(usecTimestampNow()) { // read the ApplicationInfo.ini file for Name/Version/Domain information QSettings applicationInfo(Application::resourcesPath() + "info/ApplicationInfo.ini", QSettings::IniFormat); @@ -2091,7 +2092,63 @@ void Application::updateMyAvatar(float deltaTime) { _lastQueriedViewFrustum = _viewFrustum; } } + +// sent a nack packet containing missing sequence numbers of received packets +{ + quint64 now = usecTimestampNow(); + quint64 sinceLastNack = now - _lastNackTime; + const quint64 TOO_LONG_SINCE_LAST_NACK = 100 * MSECS_PER_SECOND; + if (sinceLastNack > TOO_LONG_SINCE_LAST_NACK) { + _lastNackTime = now; + + //_octreeServerSceneStats- + } } +} + + +void Application::sendNack() { + /* + // now that we know the node ID, let's add these stats to the stats for that node... + _octreeSceneStatsLock.lockForWrite(); + if (_octreeServerSceneStats.find(nodeUUID) != _octreeServerSceneStats.end()) { + OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; + stats.trackIncomingOctreePacket(packet, wasStatsPacket, sendingNode->getClockSkewUsec()); + } + _octreeSceneStatsLock.unlock(); + */ + + char packet[MAX_PACKET_SIZE]; + NodeList* nodeList = NodeList::getInstance(); + + // iterates thru all nodes in NodeList + foreach(const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { + + char* dataAt = packet; + int bytesRemaining = MAX_PACKET_SIZE; + + int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack, node->getUUID()); + dataAt += numBytesPacketHeader; + bytesRemaining -= numBytesPacketHeader; + + + + uint16_t numSequenceNumbers; + + + + + OctreeSceneStats& stats = _octreeServerSceneStats[node->getUUID()]; + int numSequenceNumbersAvailable = stats.getNumSequenceNumberToNack(); + + + // make sure we still have an active socket + nodeList->writeUnverifiedDatagram(reinterpret_cast(queryPacket), packetLength, node); + } +} + + + void Application::queryOctree(NodeType_t serverType, PacketType packetType, NodeToJurisdictionMap& jurisdictions) { diff --git a/interface/src/Application.h b/interface/src/Application.h index f3d9c0fd27..670ec35238 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -411,6 +411,10 @@ private: static void attachNewHeadToNode(Node *newNode); static void* networkReceive(void* args); // network receive thread + void sendNack(); + + + MainWindow* _window; GLCanvas* _glWidget; // our GLCanvas has a couple extra features @@ -580,6 +584,8 @@ private: bool _runningScriptsWidgetWasVisible; QSystemTrayIcon* _trayIcon; + +quint64 _lastNackTime; }; #endif // hifi_Application_h diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 9580ae6d13..a25b72bedd 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -46,6 +46,7 @@ OctreeSceneStats::OctreeSceneStats() : _incomingReallyLate(0), _incomingPossibleDuplicate(0), _missingSequenceNumbers(), +_sequenceNumbersToNack(), _incomingFlightTimeAverage(samples), _jurisdictionRoot(NULL) { @@ -158,6 +159,7 @@ void OctreeSceneStats::copyFromOther(const OctreeSceneStats& other) { _incomingPossibleDuplicate = other._incomingPossibleDuplicate; _missingSequenceNumbers = other._missingSequenceNumbers; +_missingSequenceNumbersToNack = other._missingSequenceNumbersToNack; } @@ -926,6 +928,7 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, qDebug() << "found it in _missingSequenceNumbers"; } _missingSequenceNumbers.remove(sequence); +_sequenceNumbersToNack.remove(sequence); _incomingLikelyLost--; _incomingRecovered++; } else { @@ -955,6 +958,7 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, _incomingLikelyLost += missing; for(unsigned int missingSequence = expected; missingSequence < sequence; missingSequence++) { _missingSequenceNumbers << missingSequence; +_sequenceNumbersToNack << missingSequence; } } } @@ -982,9 +986,20 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, qDebug() << "pruning really old missing sequence:" << missingItem; } _missingSequenceNumbers.remove(missingItem); +_sequenceNumbersToNack.remove(missingItem); } } } } +bool OctreeSceneStats::getNumSequenceNumberToNack() const { + return _sequenceNumbersToNack.size(); +} + +uint16_t OctreeSceneStats::getNextSequenceNumberToNack() { + QSet::Iterator it = _sequenceNumbersToNack.begin(); + uint16_t sequenceNumber = *it; + _sequenceNumbersToNack.remove(sequenceNumber); + return sequenceNumber; +} \ No newline at end of file diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index ef22fd7c1c..e6664083b0 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -172,6 +172,9 @@ public: quint32 getIncomingPossibleDuplicate() const { return _incomingPossibleDuplicate; } float getIncomingFlightTimeAverage() { return _incomingFlightTimeAverage.getAverage(); } +bool getNumSequenceNumberToNack() const; +uint16_t getNextSequenceNumberToNack(); + private: void copyFromOther(const OctreeSceneStats& other); @@ -273,6 +276,7 @@ private: quint32 _incomingReallyLate; /// out of order and later than MAX_MISSING_SEQUENCE_OLD_AGE late quint32 _incomingPossibleDuplicate; /// out of order possibly a duplicate QSet _missingSequenceNumbers; +QSet _sequenceNumbersToNack; SimpleMovingAverage _incomingFlightTimeAverage; // features related items From 194493ab9b9f1b13843b1f3dc29c047932e92212 Mon Sep 17 00:00:00 2001 From: Stephen Birarda Date: Fri, 6 Jun 2014 12:00:47 -0700 Subject: [PATCH 2/9] remove MMX math to fix stereo discrepancies --- assignment-client/src/audio/AudioMixer.cpp | 23 ++++++++-------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/assignment-client/src/audio/AudioMixer.cpp b/assignment-client/src/audio/AudioMixer.cpp index 61dee6c82b..b3909660e2 100644 --- a/assignment-client/src/audio/AudioMixer.cpp +++ b/assignment-client/src/audio/AudioMixer.cpp @@ -310,21 +310,14 @@ void AudioMixer::addBufferToMixForListeningNodeWithBuffer(PositionalAudioRingBuf // stereo buffer - do attenuation but no sample delay for spatialization for (int s = 0; s < NETWORK_BUFFER_LENGTH_SAMPLES_STEREO; s += 4) { // use MMX to clamp four additions at a time - - __m64 bufferSamples = _mm_set_pi16(_clientSamples[s], _clientSamples[s + 1], - _clientSamples[s + 2], _clientSamples[s + 3]); - __m64 addSamples = _mm_set_pi16(nextOutputStart[s] * attenuationCoefficient, - nextOutputStart[s + 1] * attenuationCoefficient, - nextOutputStart[s + 2] * attenuationCoefficient, - nextOutputStart[s + 3] * attenuationCoefficient); - - __m64 mmxResult = _mm_adds_pi16(bufferSamples, addSamples); - int16_t* shortResults = reinterpret_cast(&mmxResult); - - _clientSamples[s] = shortResults[3]; - _clientSamples[s + 1] = shortResults[2]; - _clientSamples[s + 2] = shortResults[1]; - _clientSamples[s + 3] = shortResults[0]; + _clientSamples[s] = glm::clamp(_clientSamples[s] + (int) (nextOutputStart[s] * attenuationCoefficient), + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _clientSamples[s + 1] = glm::clamp(_clientSamples[s + 1] + (int) (nextOutputStart[s + 1] * attenuationCoefficient), + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _clientSamples[s + 2] = glm::clamp(_clientSamples[s + 2] + (int) (nextOutputStart[s + 2] * attenuationCoefficient), + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); + _clientSamples[s + 3] = glm::clamp(_clientSamples[s + 3] + (int) (nextOutputStart[s + 3] * attenuationCoefficient), + MIN_SAMPLE_VALUE, MAX_SAMPLE_VALUE); } } } From 5c4748556206e940758b46d41625b25deb962f5c Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 6 Jun 2014 14:27:51 -0700 Subject: [PATCH 3/9] client-side nack sending complete; ready to test --- interface/src/Application.cpp | 55 ++++++++++++----------- libraries/octree/src/OctreeSceneStats.cpp | 2 +- libraries/octree/src/OctreeSceneStats.h | 2 +- 3 files changed, 32 insertions(+), 27 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 358bb1bde3..7333960689 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2100,23 +2100,13 @@ void Application::updateMyAvatar(float deltaTime) { const quint64 TOO_LONG_SINCE_LAST_NACK = 100 * MSECS_PER_SECOND; if (sinceLastNack > TOO_LONG_SINCE_LAST_NACK) { _lastNackTime = now; - - //_octreeServerSceneStats- + sendNack(); } } } void Application::sendNack() { - /* - // now that we know the node ID, let's add these stats to the stats for that node... - _octreeSceneStatsLock.lockForWrite(); - if (_octreeServerSceneStats.find(nodeUUID) != _octreeServerSceneStats.end()) { - OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; - stats.trackIncomingOctreePacket(packet, wasStatsPacket, sendingNode->getClockSkewUsec()); - } - _octreeSceneStatsLock.unlock(); - */ char packet[MAX_PACKET_SIZE]; NodeList* nodeList = NodeList::getInstance(); @@ -2124,26 +2114,41 @@ void Application::sendNack() { // iterates thru all nodes in NodeList foreach(const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { - char* dataAt = packet; - int bytesRemaining = MAX_PACKET_SIZE; + if (node->getActiveSocket() && + ( node->getType() == NodeType::VoxelServer + || node->getType() == NodeType::ParticleServer + || node->getType() == NodeType::ModelServer) + ) { - int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack, node->getUUID()); - dataAt += numBytesPacketHeader; - bytesRemaining -= numBytesPacketHeader; - - + OctreeSceneStats& stats = _octreeServerSceneStats[node->getUUID()]; - uint16_t numSequenceNumbers; + char* dataAt = packet; + int bytesRemaining = MAX_PACKET_SIZE; + + // pack header + int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack, node->getUUID()); + dataAt += numBytesPacketHeader; + bytesRemaining -= numBytesPacketHeader; + + int numPacketsRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(OCTREE_PACKET_SEQUENCE); + // calculate and pack number of sequence numbers + uint16_t numSequenceNumbers = min(stats.getNumSequenceNumbersToNack(), numPacketsRoomFor); + uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; + *numSequenceNumbersAt = numSequenceNumbers; + dataAt += sizeof(uint16_t); + // pack sequence numbers + for (int i = 0; i < numSequenceNumbers; i++) { + OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; + *sequenceNumberAt = stats.getNextSequenceNumberToNack(); + dataAt += sizeof(OCTREE_PACKET_SEQUENCE); + } - OctreeSceneStats& stats = _octreeServerSceneStats[node->getUUID()]; - int numSequenceNumbersAvailable = stats.getNumSequenceNumberToNack(); - - - // make sure we still have an active socket - nodeList->writeUnverifiedDatagram(reinterpret_cast(queryPacket), packetLength, node); + // make sure we still have an active socket???? + nodeList->writeUnverifiedDatagram(packet, dataAt - packet, node); + } } } diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index a25b72bedd..01332fbc46 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -993,7 +993,7 @@ _sequenceNumbersToNack.remove(missingItem); } -bool OctreeSceneStats::getNumSequenceNumberToNack() const { +int OctreeSceneStats::getNumSequenceNumbersToNack() const { return _sequenceNumbersToNack.size(); } diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index e6664083b0..ca9bf7c74b 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -172,7 +172,7 @@ public: quint32 getIncomingPossibleDuplicate() const { return _incomingPossibleDuplicate; } float getIncomingFlightTimeAverage() { return _incomingFlightTimeAverage.getAverage(); } -bool getNumSequenceNumberToNack() const; +int getNumSequenceNumbersToNack() const; uint16_t getNextSequenceNumberToNack(); private: From bd2148d4b7eef48d46c6171f801521a4eb1dead8 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 6 Jun 2014 14:33:53 -0700 Subject: [PATCH 4/9] added write lock/unlock ...for _octreeSceneStatsLock --- interface/src/Application.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7333960689..6d6c7d64ab 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2120,6 +2120,8 @@ void Application::sendNack() { || node->getType() == NodeType::ModelServer) ) { + _octreeSceneStatsLock.lockForWrite(); + OctreeSceneStats& stats = _octreeServerSceneStats[node->getUUID()]; char* dataAt = packet; @@ -2146,6 +2148,8 @@ void Application::sendNack() { dataAt += sizeof(OCTREE_PACKET_SEQUENCE); } + _octreeSceneStatsLock.unlock(); + // make sure we still have an active socket???? nodeList->writeUnverifiedDatagram(packet, dataAt - packet, node); } From 0b1bffa83ca55957d174aa7e7fc0aa3e7916c07d Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 6 Jun 2014 15:03:54 -0700 Subject: [PATCH 5/9] added random packet drops; ready to test --- .../src/octree/OctreeSendThread.cpp | 7 +++---- interface/src/Application.cpp | 7 ++++++- libraries/networking/src/LimitedNodeList.cpp | 17 +++++++++++++++++ libraries/networking/src/LimitedNodeList.h | 3 +++ libraries/octree/src/OctreeSceneStats.cpp | 2 +- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 032b45fcdf..e5c47f5d2d 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -181,7 +181,7 @@ int OctreeSendThread::handlePacketSend(OctreeQueryNode* nodeData, int& trueBytes // actually send it OctreeServer::didCallWriteDatagram(this); - NodeList::getInstance()->writeDatagram((char*) statsMessage, statsMessageLength, _node); +NodeList::getInstance()->writeDatagram2(nodeData->getSequenceNumber(), (char*) statsMessage, statsMessageLength, _node); packetSent = true; } else { // not enough room in the packet, send two packets @@ -215,8 +215,7 @@ int OctreeSendThread::handlePacketSend(OctreeQueryNode* nodeData, int& trueBytes packetsSent++; OctreeServer::didCallWriteDatagram(this); - NodeList::getInstance()->writeDatagram((char*) nodeData->getPacket(), nodeData->getPacketLength(), _node); - +NodeList::getInstance()->writeDatagram2(nodeData->getSequenceNumber(), (char*)nodeData->getPacket(), nodeData->getPacketLength(), _node); packetSent = true; thisWastedBytes = MAX_PACKET_SIZE - nodeData->getPacketLength(); @@ -245,7 +244,7 @@ int OctreeSendThread::handlePacketSend(OctreeQueryNode* nodeData, int& trueBytes if (nodeData->isPacketWaiting() && !nodeData->isShuttingDown()) { // just send the voxel packet OctreeServer::didCallWriteDatagram(this); - NodeList::getInstance()->writeDatagram((char*) nodeData->getPacket(), nodeData->getPacketLength(), _node); +NodeList::getInstance()->writeDatagram2(nodeData->getSequenceNumber(), (char*)nodeData->getPacket(), nodeData->getPacketLength(), _node); packetSent = true; int thisWastedBytes = MAX_PACKET_SIZE - nodeData->getPacketLength(); diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 6d6c7d64ab..75980af934 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2097,7 +2097,7 @@ void Application::updateMyAvatar(float deltaTime) { { quint64 now = usecTimestampNow(); quint64 sinceLastNack = now - _lastNackTime; - const quint64 TOO_LONG_SINCE_LAST_NACK = 100 * MSECS_PER_SECOND; + const quint64 TOO_LONG_SINCE_LAST_NACK = 250 * MSECS_PER_SECOND; if (sinceLastNack > TOO_LONG_SINCE_LAST_NACK) { _lastNackTime = now; sendNack(); @@ -2108,6 +2108,8 @@ void Application::updateMyAvatar(float deltaTime) { void Application::sendNack() { +printf("\n\t sendNack()...\n"); + char packet[MAX_PACKET_SIZE]; NodeList* nodeList = NodeList::getInstance(); @@ -2142,11 +2144,14 @@ void Application::sendNack() { dataAt += sizeof(uint16_t); // pack sequence numbers +printf("\t\t packed %d seq #s:", numSequenceNumbers); for (int i = 0; i < numSequenceNumbers; i++) { OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; *sequenceNumberAt = stats.getNextSequenceNumberToNack(); dataAt += sizeof(OCTREE_PACKET_SEQUENCE); +printf(" %d,", *sequenceNumberAt); } +printf("\n"); _octreeSceneStatsLock.unlock(); diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index c0d7941edf..f9630c9102 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -271,6 +271,23 @@ qint64 LimitedNodeList::writeDatagram(const char* data, qint64 size, const Share return writeDatagram(QByteArray(data, size), destinationNode, overridenSockAddr); } +qint64 LimitedNodeList::writeDatagram2(int seq, const char* data, qint64 size, const SharedNodePointer& destinationNode, + const HifiSockAddr& overridenSockAddr) { + + qint64 ret; + + if (randFloat() < 0.8f) { + ret = writeDatagram(QByteArray(data, size), destinationNode, overridenSockAddr); + } + else { + printf("\t\t\t dropped packet seq = %d!!!\n", seq); + } + + + return ret; +} + + qint64 LimitedNodeList::writeUnverifiedDatagram(const char* data, qint64 size, const SharedNodePointer& destinationNode, const HifiSockAddr& overridenSockAddr) { return writeUnverifiedDatagram(QByteArray(data, size), destinationNode, overridenSockAddr); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index a4bc8022bf..b3481885c8 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -72,6 +72,9 @@ public: qint64 writeDatagram(const char* data, qint64 size, const SharedNodePointer& destinationNode, const HifiSockAddr& overridenSockAddr = HifiSockAddr()); +qint64 writeDatagram2(int seq, const char* data, qint64 size, const SharedNodePointer& destinationNode, + const HifiSockAddr& overridenSockAddr = HifiSockAddr()); + qint64 writeUnverifiedDatagram(const char* data, qint64 size, const SharedNodePointer& destinationNode, const HifiSockAddr& overridenSockAddr = HifiSockAddr()); diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 01332fbc46..a93b2d0374 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -159,7 +159,7 @@ void OctreeSceneStats::copyFromOther(const OctreeSceneStats& other) { _incomingPossibleDuplicate = other._incomingPossibleDuplicate; _missingSequenceNumbers = other._missingSequenceNumbers; -_missingSequenceNumbersToNack = other._missingSequenceNumbersToNack; +_sequenceNumbersToNack = other._sequenceNumbersToNack; } From b39559d86086f3b78fff66e5476553c315ba9a28 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 6 Jun 2014 17:44:30 -0700 Subject: [PATCH 6/9] packet recovery seems to be working Seems to impact FPS a lot. OctreeSceneStats is not being locked (causes freezing, need to find out why). --- .../src/octree/OctreeQueryNode.cpp | 5 --- .../src/octree/OctreeQueryNode.h | 4 ++- .../src/octree/OctreeSendThread.cpp | 3 +- assignment-client/src/octree/OctreeServer.cpp | 35 ++++++++++--------- interface/src/Application.cpp | 16 +++++---- libraries/networking/src/LimitedNodeList.cpp | 4 +-- libraries/networking/src/PacketHeaders.h | 3 +- 7 files changed, 37 insertions(+), 33 deletions(-) diff --git a/assignment-client/src/octree/OctreeQueryNode.cpp b/assignment-client/src/octree/OctreeQueryNode.cpp index 208be5951b..cf01e9a864 100644 --- a/assignment-client/src/octree/OctreeQueryNode.cpp +++ b/assignment-client/src/octree/OctreeQueryNode.cpp @@ -377,11 +377,6 @@ void OctreeQueryNode::packetSent(const QByteArray& packet) { _sequenceNumber++; } - -void OctreeQueryNode::addSequenceNumbersToResend(const QList& sequenceNumbers) { - _nackedSequenceNumbers.append(sequenceNumbers); -} - bool OctreeQueryNode::hasNextNackedPacket() const { return !_nackedSequenceNumbers.isEmpty(); } diff --git a/assignment-client/src/octree/OctreeQueryNode.h b/assignment-client/src/octree/OctreeQueryNode.h index e1707cc899..b79367503c 100644 --- a/assignment-client/src/octree/OctreeQueryNode.h +++ b/assignment-client/src/octree/OctreeQueryNode.h @@ -109,7 +109,9 @@ public: OCTREE_PACKET_SEQUENCE getSequenceNumber() const { return _sequenceNumber; } - void addSequenceNumbersToResend(const QList& sequenceNumbers); + void addNackedSequenceNumber(OCTREE_PACKET_SEQUENCE sequenceNumber) { + _nackedSequenceNumbers.append(sequenceNumber); + } bool hasNextNackedPacket() const; const QByteArray* getNextNackedPacket(); diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index e5c47f5d2d..cb8a773a64 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -305,7 +305,8 @@ int OctreeSendThread::resendNackedPackets(OctreeQueryNode* nodeData) { _totalWastedBytes += MAX_PACKET_SIZE - packet->size(); // ??? } } - + +if (packetsSent > 0) printf("\t\t re-sent %d packets!\n", packetsSent); return packetsSent; } diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index 46eb7aac16..8cf1649cdb 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -832,10 +832,9 @@ void OctreeServer::readPendingDatagrams() { PacketType packetType = packetTypeForPacket(receivedPacket); SharedNodePointer matchingNode = nodeList->sendingNodeForPacket(receivedPacket); if (packetType == getMyQueryMessageType()) { - // If we got a query packet, then we're talking to an agent, and we // need to make sure we have it in our nodeList. - if (matchingNode) { + if (matchingNode) { nodeList->updateNodeWithDataFromPacket(matchingNode, receivedPacket); OctreeQueryNode* nodeData = (OctreeQueryNode*)matchingNode->getLinkedData(); if (nodeData && !nodeData->isOctreeSendThreadInitalized()) { @@ -852,25 +851,29 @@ void OctreeServer::readPendingDatagrams() { } else if (packetType == PacketTypeOctreeDataNack) { // parse packet for sequence numbers that need to be resent -int numBytesPacketHeader = numBytesForPacketHeader(receivedPacket); -const unsigned char* dataAt = reinterpret_cast(receivedPacket.data()) + numBytesPacketHeader; -uint16_t numSequenceNumbers = (*(uint16_t*)dataAt); -dataAt += sizeof(uint16_t); +if (matchingNode) { -// read sequence numbers -QList sequenceNumbers; -for (int i = 0; i < numSequenceNumbers; i++) { - OCTREE_PACKET_SEQUENCE sequenceNumber = (*(OCTREE_PACKET_SEQUENCE*)dataAt); - sequenceNumbers.append(sequenceNumber); - dataAt += sizeof(OCTREE_PACKET_SEQUENCE); + OctreeQueryNode* nodeData = (OctreeQueryNode*)matchingNode->getLinkedData(); -printf("\t\t\t nacked packet: seq = %d\n", sequenceNumber); + int numBytesPacketHeader = numBytesForPacketHeader(receivedPacket); + const unsigned char* dataAt = reinterpret_cast(receivedPacket.data()) + numBytesPacketHeader; + + uint16_t numSequenceNumbers = (*(uint16_t*)dataAt); + dataAt += sizeof(uint16_t); + +printf("\t received nack packet containing %d seq nums\n", numSequenceNumbers); + + // read sequence numbers + for (int i = 0; i < numSequenceNumbers; i++) { + OCTREE_PACKET_SEQUENCE sequenceNumber = (*(OCTREE_PACKET_SEQUENCE*)dataAt); + nodeData->addNackedSequenceNumber(sequenceNumber); + dataAt += sizeof(OCTREE_PACKET_SEQUENCE); + +printf("\t seq = %d\n", sequenceNumber); + } } -OctreeQueryNode* nodeData = (OctreeQueryNode*)matchingNode->getLinkedData(); // move this or something -nodeData->addSequenceNumbersToResend(sequenceNumbers); - } else if (packetType == PacketTypeJurisdictionRequest) { diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 07bef852f2..a28343eb96 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2110,8 +2110,6 @@ void Application::updateMyAvatar(float deltaTime) { void Application::sendNack() { -printf("\n\t sendNack()...\n"); - char packet[MAX_PACKET_SIZE]; NodeList* nodeList = NodeList::getInstance(); @@ -2124,15 +2122,18 @@ printf("\n\t sendNack()...\n"); || node->getType() == NodeType::ModelServer) ) { - _octreeSceneStatsLock.lockForWrite(); + //_octreeSceneStatsLock.lockForWrite(); OctreeSceneStats& stats = _octreeServerSceneStats[node->getUUID()]; + int numSequenceNumbersAvailable = stats.getNumSequenceNumbersToNack(); + if (numSequenceNumbersAvailable == 0) + continue; char* dataAt = packet; int bytesRemaining = MAX_PACKET_SIZE; // pack header - int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack, node->getUUID()); + int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack); dataAt += numBytesPacketHeader; bytesRemaining -= numBytesPacketHeader; @@ -2140,12 +2141,13 @@ printf("\n\t sendNack()...\n"); // calculate and pack number of sequence numbers - uint16_t numSequenceNumbers = min(stats.getNumSequenceNumbersToNack(), numPacketsRoomFor); + uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numPacketsRoomFor); uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; *numSequenceNumbersAt = numSequenceNumbers; dataAt += sizeof(uint16_t); // pack sequence numbers +printf("\n\t sending nack...\n"); printf("\t\t packed %d seq #s:", numSequenceNumbers); for (int i = 0; i < numSequenceNumbers; i++) { OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; @@ -2155,10 +2157,10 @@ printf(" %d,", *sequenceNumberAt); } printf("\n"); - _octreeSceneStatsLock.unlock(); + //_octreeSceneStatsLock.unlock(); // make sure we still have an active socket???? - nodeList->writeUnverifiedDatagram(packet, dataAt - packet, node); + nodeList->writeUnverifiedDatagram(packet, dataAt-packet, node); } } } diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index f9630c9102..15cebc26ac 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -274,13 +274,13 @@ qint64 LimitedNodeList::writeDatagram(const char* data, qint64 size, const Share qint64 LimitedNodeList::writeDatagram2(int seq, const char* data, qint64 size, const SharedNodePointer& destinationNode, const HifiSockAddr& overridenSockAddr) { - qint64 ret; + qint64 ret = -1; if (randFloat() < 0.8f) { ret = writeDatagram(QByteArray(data, size), destinationNode, overridenSockAddr); } else { - printf("\t\t\t dropped packet seq = %d!!!\n", seq); + printf("dropped packet seq = %d --------------------------\n", seq); } diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index a73ffe8564..eac2f88870 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -75,7 +75,8 @@ const QSet NON_VERIFIED_PACKETS = QSet() << PacketTypeDomainServerRequireDTLS << PacketTypeDomainConnectRequest << PacketTypeDomainList << PacketTypeDomainListRequest << PacketTypeDomainOAuthRequest << PacketTypeCreateAssignment << PacketTypeRequestAssignment << PacketTypeStunResponse - << PacketTypeNodeJsonStats << PacketTypeVoxelQuery << PacketTypeParticleQuery << PacketTypeModelQuery; + << PacketTypeNodeJsonStats << PacketTypeVoxelQuery << PacketTypeParticleQuery << PacketTypeModelQuery +<< PacketTypeOctreeDataNack; const int NUM_BYTES_MD5_HASH = 16; const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID; From 52860908128fe0052a18981ace38b2ec06da2a46 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 6 Jun 2014 17:52:48 -0700 Subject: [PATCH 7/9] removed interface print statements --- interface/src/Application.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index a28343eb96..87e82b21a2 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2147,15 +2147,15 @@ void Application::sendNack() { dataAt += sizeof(uint16_t); // pack sequence numbers -printf("\n\t sending nack...\n"); -printf("\t\t packed %d seq #s:", numSequenceNumbers); +//printf("\n\t sending nack...\n"); +//printf("\t\t packed %d seq #s:", numSequenceNumbers); for (int i = 0; i < numSequenceNumbers; i++) { OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; *sequenceNumberAt = stats.getNextSequenceNumberToNack(); dataAt += sizeof(OCTREE_PACKET_SEQUENCE); -printf(" %d,", *sequenceNumberAt); +//printf(" %d,", *sequenceNumberAt); } -printf("\n"); +//printf("\n"); //_octreeSceneStatsLock.unlock(); From 3bc457d4d2ea981f3c12bb8605bfe078f9c180bf Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 9 Jun 2014 11:14:02 -0700 Subject: [PATCH 8/9] added octree scene stats locking and unlocking --- .../src/octree/OctreeQueryNode.cpp | 8 +-- .../src/octree/OctreeSendThread.cpp | 3 - interface/src/Application.cpp | 69 ++++++++++++------- libraries/octree/src/OctreeSceneStats.h | 4 +- 4 files changed, 49 insertions(+), 35 deletions(-) diff --git a/assignment-client/src/octree/OctreeQueryNode.cpp b/assignment-client/src/octree/OctreeQueryNode.cpp index cf01e9a864..5cfe5501e1 100644 --- a/assignment-client/src/octree/OctreeQueryNode.cpp +++ b/assignment-client/src/octree/OctreeQueryNode.cpp @@ -382,11 +382,9 @@ bool OctreeQueryNode::hasNextNackedPacket() const { } const QByteArray* OctreeQueryNode::getNextNackedPacket() { - if (!_nackedSequenceNumbers.isEmpty()) { - const QByteArray* nextPacket = _sentPacketHistory.getPacket(_nackedSequenceNumbers.first()); - _nackedSequenceNumbers.pop_front(); - return nextPacket; // could be null + // could return null if packet is not in the history + return _sentPacketHistory.getPacket(_nackedSequenceNumbers.takeFirst()); } return NULL; -} \ No newline at end of file +} diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index cb8a773a64..6f39cd28dc 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -288,18 +288,15 @@ NodeList::getInstance()->writeDatagram2(nodeData->getSequenceNumber(), (char*)no int OctreeSendThread::resendNackedPackets(OctreeQueryNode* nodeData) { const int MAX_PACKETS_RESEND = 10; - int packetsSent = 0; const QByteArray* packet; while (nodeData->hasNextNackedPacket() && packetsSent < MAX_PACKETS_RESEND) { packet = nodeData->getNextNackedPacket(); - // packet will be NULL if it's not in nodeData's packet history if (packet) { NodeList::getInstance()->writeDatagram(*packet, _node); packetsSent++; - // ?????? _totalBytes += packet->size(); _totalPackets++; _totalWastedBytes += MAX_PACKET_SIZE - packet->size(); // ??? diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 87e82b21a2..f8152bb62b 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2107,6 +2107,21 @@ void Application::updateMyAvatar(float deltaTime) { } } +/*/ Attempt to identify the sender from it's address. + if (sendingNode) { + QUuid nodeUUID = sendingNode->getUUID(); + + // now that we know the node ID, let's add these stats to the stats for that node... + _octreeSceneStatsLock.lockForWrite(); + if (_octreeServerSceneStats.find(nodeUUID) != _octreeServerSceneStats.end()) { + OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; + stats.trackIncomingOctreePacket(packet, wasStatsPacket, sendingNode->getClockSkewUsec()); + } + _octreeSceneStatsLock.unlock(); + } + */ + + void Application::sendNack() { @@ -2122,13 +2137,6 @@ void Application::sendNack() { || node->getType() == NodeType::ModelServer) ) { - //_octreeSceneStatsLock.lockForWrite(); - - OctreeSceneStats& stats = _octreeServerSceneStats[node->getUUID()]; - int numSequenceNumbersAvailable = stats.getNumSequenceNumbersToNack(); - if (numSequenceNumbersAvailable == 0) - continue; - char* dataAt = packet; int bytesRemaining = MAX_PACKET_SIZE; @@ -2136,28 +2144,39 @@ void Application::sendNack() { int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack); dataAt += numBytesPacketHeader; bytesRemaining -= numBytesPacketHeader; - - int numPacketsRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(OCTREE_PACKET_SEQUENCE); + int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(OCTREE_PACKET_SEQUENCE); - // calculate and pack number of sequence numbers - uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numPacketsRoomFor); - uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; - *numSequenceNumbersAt = numSequenceNumbers; - dataAt += sizeof(uint16_t); + QUuid nodeUUID = node->getUUID(); - // pack sequence numbers -//printf("\n\t sending nack...\n"); -//printf("\t\t packed %d seq #s:", numSequenceNumbers); - for (int i = 0; i < numSequenceNumbers; i++) { - OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; - *sequenceNumberAt = stats.getNextSequenceNumberToNack(); - dataAt += sizeof(OCTREE_PACKET_SEQUENCE); -//printf(" %d,", *sequenceNumberAt); + _octreeSceneStatsLock.lockForWrite(); + if (_octreeServerSceneStats.find(nodeUUID) != _octreeServerSceneStats.end()) { + OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; + + int numSequenceNumbersAvailable = stats.getNumSequenceNumbersToNack(); + if (numSequenceNumbersAvailable == 0) { + _octreeSceneStatsLock.unlock(); + continue; + } + + // calculate and pack number of sequence numbers + uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); + uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; + *numSequenceNumbersAt = numSequenceNumbers; + dataAt += sizeof(uint16_t); + + // pack sequence numbers + //printf("\n\t sending nack...\n"); + //printf("\t\t packed %d seq #s:", numSequenceNumbers); + for (int i = 0; i < numSequenceNumbers; i++) { + OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; + *sequenceNumberAt = stats.getNextSequenceNumberToNack(); + dataAt += sizeof(OCTREE_PACKET_SEQUENCE); + //printf(" %d,", *sequenceNumberAt); + } + //printf("\n"); } -//printf("\n"); - - //_octreeSceneStatsLock.unlock(); + _octreeSceneStatsLock.unlock(); // make sure we still have an active socket???? nodeList->writeUnverifiedDatagram(packet, dataAt-packet, node); diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index ca9bf7c74b..ac9b1606c1 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -275,8 +275,8 @@ private: quint32 _incomingLate; /// out of order later than expected quint32 _incomingReallyLate; /// out of order and later than MAX_MISSING_SEQUENCE_OLD_AGE late quint32 _incomingPossibleDuplicate; /// out of order possibly a duplicate - QSet _missingSequenceNumbers; -QSet _sequenceNumbersToNack; + QSet _missingSequenceNumbers; +QSet _sequenceNumbersToNack; SimpleMovingAverage _incomingFlightTimeAverage; // features related items From 529dd827a2d613aa6c6b3fd74d9dfcbeaf9de72e Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 9 Jun 2014 12:00:19 -0700 Subject: [PATCH 9/9] moved around code in sendNack() a bit --- interface/src/Application.cpp | 65 ++++++++++++---------- libraries/networking/src/PacketHeaders.cpp | 2 +- libraries/octree/src/OctreeSceneStats.h | 4 +- 3 files changed, 38 insertions(+), 33 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index f8152bb62b..3ebfe728de 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2137,6 +2137,25 @@ void Application::sendNack() { || node->getType() == NodeType::ModelServer) ) { + QUuid nodeUUID = node->getUUID(); + + + _octreeSceneStatsLock.lockForWrite(); + + // retreive octree scene stats of this node + if (_octreeServerSceneStats.find(nodeUUID) == _octreeServerSceneStats.end()) { + _octreeSceneStatsLock.unlock(); + continue; + } + OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; + + // check if there are any sequence numbers that need to be nacked + int numSequenceNumbersAvailable = stats.getNumSequenceNumbersToNack(); + if (numSequenceNumbersAvailable == 0) { + _octreeSceneStatsLock.unlock(); + continue; + } + char* dataAt = packet; int bytesRemaining = MAX_PACKET_SIZE; @@ -2146,40 +2165,26 @@ void Application::sendNack() { bytesRemaining -= numBytesPacketHeader; int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(OCTREE_PACKET_SEQUENCE); + // calculate and pack the number of sequence numbers + uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); + uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; + *numSequenceNumbersAt = numSequenceNumbers; + dataAt += sizeof(uint16_t); - QUuid nodeUUID = node->getUUID(); - - _octreeSceneStatsLock.lockForWrite(); - if (_octreeServerSceneStats.find(nodeUUID) != _octreeServerSceneStats.end()) { - OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; - - int numSequenceNumbersAvailable = stats.getNumSequenceNumbersToNack(); - if (numSequenceNumbersAvailable == 0) { - _octreeSceneStatsLock.unlock(); - continue; - } - - // calculate and pack number of sequence numbers - uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); - uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; - *numSequenceNumbersAt = numSequenceNumbers; - dataAt += sizeof(uint16_t); - - // pack sequence numbers - //printf("\n\t sending nack...\n"); - //printf("\t\t packed %d seq #s:", numSequenceNumbers); - for (int i = 0; i < numSequenceNumbers; i++) { - OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; - *sequenceNumberAt = stats.getNextSequenceNumberToNack(); - dataAt += sizeof(OCTREE_PACKET_SEQUENCE); - //printf(" %d,", *sequenceNumberAt); - } - //printf("\n"); + // pack sequence numbers +//printf("\n\t sending nack...\n"); +//printf("\t\t packed %d seq #s:", numSequenceNumbers); + for (int i = 0; i < numSequenceNumbers; i++) { + OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; + *sequenceNumberAt = stats.getNextSequenceNumberToNack(); + dataAt += sizeof(OCTREE_PACKET_SEQUENCE); +//printf(" %d,", *sequenceNumberAt); } +//printf("\n"); + _octreeSceneStatsLock.unlock(); - // make sure we still have an active socket???? - nodeList->writeUnverifiedDatagram(packet, dataAt-packet, node); + nodeList->writeUnverifiedDatagram(packet, dataAt - packet, node); } } } diff --git a/libraries/networking/src/PacketHeaders.cpp b/libraries/networking/src/PacketHeaders.cpp index 7278f1fbcb..e2bc46b3be 100644 --- a/libraries/networking/src/PacketHeaders.cpp +++ b/libraries/networking/src/PacketHeaders.cpp @@ -108,7 +108,7 @@ int populatePacketHeader(char* packet, PacketType type, const QUuid& connectionU position += NUM_BYTES_RFC4122_UUID; if (!NON_VERIFIED_PACKETS.contains(type)) { - // pack 16 bytes of zeros where the md5 hash will be placed one data is packed + // pack 16 bytes of zeros where the md5 hash will be placed once data is packed memset(position, 0, NUM_BYTES_MD5_HASH); position += NUM_BYTES_MD5_HASH; } diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index ac9b1606c1..ca9bf7c74b 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -275,8 +275,8 @@ private: quint32 _incomingLate; /// out of order later than expected quint32 _incomingReallyLate; /// out of order and later than MAX_MISSING_SEQUENCE_OLD_AGE late quint32 _incomingPossibleDuplicate; /// out of order possibly a duplicate - QSet _missingSequenceNumbers; -QSet _sequenceNumbersToNack; + QSet _missingSequenceNumbers; +QSet _sequenceNumbersToNack; SimpleMovingAverage _incomingFlightTimeAverage; // features related items