From 7955979599cced0a45d9cbf9da89ec643f073094 Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 12 Jun 2014 09:17:12 -0700 Subject: [PATCH 01/55] added _missingSequenceNumbers tracking to OctreeInboundPacketProcessor --- .../octree/OctreeInboundPacketProcessor.cpp | 83 ++++++++++++++----- .../src/octree/OctreeInboundPacketProcessor.h | 9 +- 2 files changed, 71 insertions(+), 21 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 545c502036..52407f78d5 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -123,13 +123,13 @@ void OctreeInboundPacketProcessor::processPacket(const SharedNodePointer& sendin qDebug() << "sender has no known nodeUUID."; } } - trackInboundPackets(nodeUUID, sequence, transitTime, editsInPacket, processTime, lockWaitTime); + trackInboundPacket(nodeUUID, sequence, transitTime, editsInPacket, processTime, lockWaitTime); } else { qDebug("unknown packet ignored... packetType=%d", packetType); } } -void OctreeInboundPacketProcessor::trackInboundPackets(const QUuid& nodeUUID, int sequence, quint64 transitTime, +void OctreeInboundPacketProcessor::trackInboundPacket(const QUuid& nodeUUID, unsigned short int sequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { _totalTransitTime += transitTime; @@ -142,31 +142,74 @@ void OctreeInboundPacketProcessor::trackInboundPackets(const QUuid& nodeUUID, in // see if this is the first we've heard of this node... if (_singleSenderStats.find(nodeUUID) == _singleSenderStats.end()) { SingleSenderStats stats; - - stats._totalTransitTime += transitTime; - stats._totalProcessTime += processTime; - stats._totalLockWaitTime += lockWaitTime; - stats._totalElementsInPacket += editsInPacket; - stats._totalPackets++; - + stats.trackInboundPacket(sequence, transitTime, editsInPacket, processTime, lockWaitTime); _singleSenderStats[nodeUUID] = stats; } else { SingleSenderStats& stats = _singleSenderStats[nodeUUID]; - stats._totalTransitTime += transitTime; - stats._totalProcessTime += processTime; - stats._totalLockWaitTime += lockWaitTime; - stats._totalElementsInPacket += editsInPacket; - stats._totalPackets++; + stats.trackInboundPacket(sequence, transitTime, editsInPacket, processTime, lockWaitTime); } } +SingleSenderStats::SingleSenderStats() + : _totalTransitTime(0), + _totalProcessTime(0), + _totalLockWaitTime(0), + _totalElementsInPacket(0), + _totalPackets(0), + _missingSequenceNumbers() +{ -SingleSenderStats::SingleSenderStats() { - _totalTransitTime = 0; - _totalProcessTime = 0; - _totalLockWaitTime = 0; - _totalElementsInPacket = 0; - _totalPackets = 0; } +void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, + int editsInPacket, quint64 processTime, quint64 lockWaitTime) { + const int MAX_REASONABLE_SEQUENCE_GAP = 1000; + const int MAX_MISSING_SEQUENCE_SIZE = 100; + + unsigned short int expectedSequence = _totalPackets == 0 ? incomingSequence : _incomingLastSequence + 1; + + if (incomingSequence == expectedSequence) { // on time + _incomingLastSequence = incomingSequence; + } + else { + // ignore packet if sequence number gap is unreasonable + if (std::abs(incomingSequence - expectedSequence) > MAX_REASONABLE_SEQUENCE_GAP) { + qDebug() << "ignoring unreasonable packet... sequence:" << incomingSequence + << "_incomingLastSequence:" << _incomingLastSequence; + return; + } + + if (incomingSequence > expectedSequence) { // early + + // add all sequence numbers that were skipped to the missing sequence numbers list + for (int missingSequence = expectedSequence; missingSequence < incomingSequence; missingSequence++) { + _missingSequenceNumbers.insert(missingSequence); + } + _incomingLastSequence = incomingSequence; + + } else { // late + + // remove this from missing sequence number if it's in there + _missingSequenceNumbers.remove(incomingSequence); + + // do not update _incomingLastSequence + } + } + + // prune missing sequence list if it gets too big + if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE_SIZE) { + foreach(unsigned short int missingSequence, _missingSequenceNumbers) { + if (missingSequence <= std::max(0, _incomingLastSequence - MAX_REASONABLE_SEQUENCE_GAP)) { + _missingSequenceNumbers.remove(missingSequence); + } + } + } + + // update other stats + _totalTransitTime += transitTime; + _totalProcessTime += processTime; + _totalLockWaitTime += lockWaitTime; + _totalElementsInPacket += editsInPacket; + _totalPackets++; +} \ No newline at end of file diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index f637a9e7c9..82471bcddf 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -32,12 +32,19 @@ public: { return _totalElementsInPacket == 0 ? 0 : _totalProcessTime / _totalElementsInPacket; } quint64 getAverageLockWaitTimePerElement() const { return _totalElementsInPacket == 0 ? 0 : _totalLockWaitTime / _totalElementsInPacket; } + + void trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, + int editsInPacket, quint64 processTime, quint64 lockWaitTime); + quint64 _totalTransitTime; quint64 _totalProcessTime; quint64 _totalLockWaitTime; quint64 _totalElementsInPacket; quint64 _totalPackets; + + unsigned short int _incomingLastSequence; + QSet _missingSequenceNumbers; }; typedef std::map NodeToSenderStatsMap; @@ -69,7 +76,7 @@ protected: virtual void processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet); private: - void trackInboundPackets(const QUuid& nodeUUID, int sequence, quint64 transitTime, + void trackInboundPacket(const QUuid& nodeUUID, unsigned short int sequence, quint64 transitTime, int voxelsInPacket, quint64 processTime, quint64 lockWaitTime); OctreeServer* _myServer; From 15bd0878c407ede95f724009adadc81deeb7b4fd Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 12 Jun 2014 11:37:05 -0700 Subject: [PATCH 02/55] added code for AC to send nack packets; no locking yet --- .../octree/OctreeInboundPacketProcessor.cpp | 2 +- .../src/octree/OctreeInboundPacketProcessor.h | 2 + .../src/octree/OctreeSendThread.cpp | 70 +++++++++++++++++++ .../src/octree/OctreeSendThread.h | 2 + assignment-client/src/octree/OctreeServer.h | 2 + libraries/networking/src/PacketHeaders.h | 3 +- 6 files changed, 79 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 52407f78d5..c1c564b58c 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -212,4 +212,4 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, _totalLockWaitTime += lockWaitTime; _totalElementsInPacket += editsInPacket; _totalPackets++; -} \ No newline at end of file +} diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 82471bcddf..3e3f5f2dcb 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -35,6 +35,7 @@ public: void trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime); + const QSet& getMissingSequenceNumbers() const { return _missingSequenceNumbers; } quint64 _totalTransitTime; @@ -71,6 +72,7 @@ public: void resetStats(); NodeToSenderStatsMap& getSingleSenderStats() { return _singleSenderStats; } + const NodeToSenderStatsMap& getSingleSenderStats() const { return _singleSenderStats; } protected: virtual void processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet); diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index cb149b1d96..6b1119fca1 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -85,6 +85,7 @@ bool OctreeSendThread::process() { if (nodeData && !nodeData->isShuttingDown()) { bool viewFrustumChanged = nodeData->updateCurrentViewFrustum(); packetDistributor(nodeData, viewFrustumChanged); + sendNack(nodeData); } } } @@ -111,6 +112,75 @@ bool OctreeSendThread::process() { return isStillRunning(); // keep running till they terminate us } + +int OctreeSendThread::sendNack(OctreeQueryNode* nodeData) { + + // if we're shutting down, then exit early + if (nodeData->isShuttingDown()) { + return 0; + } + + const OctreeInboundPacketProcessor* myServerPacketProcessor = _myServer->getInboundPacketProcessor(); + + // if there are packets from _node that are waiting to be processed, + // don't send a NACK since the missing packets may be among those waiting packets. + if (myServerPacketProcessor->hasPacketsToProcessFrom(_node)) { + return 0; + } + + // lock unlock required ????????? prolly: _singleSenderStats may have our node's entry deleted during this or something + // maybe just make a copy instead!! + + // lock + + const QSet missingSequenceNumbersFromNode = myServerPacketProcessor + ->getSingleSenderStats().at(_node->getUUID()).getMissingSequenceNumbers(); + + // check if there are any sequence numbers that need to be nacked + int numSequenceNumbersAvailable = missingSequenceNumbersFromNode.size(); + if (numSequenceNumbersAvailable == 0) { + //unlock + return 0; + } + + // construct nack packet + + char packet[MAX_PACKET_SIZE]; + + char* dataAt = packet; + int bytesRemaining = MAX_PACKET_SIZE; + + // pack header + int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeEditNack); + dataAt += numBytesPacketHeader; + bytesRemaining -= numBytesPacketHeader; + int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(unsigned short int); + + // calculate and pack the number of sequence numbers + uint16_t numSequenceNumbers = std::min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); + uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; + *numSequenceNumbersAt = numSequenceNumbers; + dataAt += sizeof(uint16_t); + + // pack sequence numbers + QSet::const_iterator begin = missingSequenceNumbersFromNode.begin(), end = missingSequenceNumbersFromNode.end(); + for (QSet::const_iterator i = begin; i != end; i++) { + unsigned short int* sequenceNumberAt = (unsigned short int*)dataAt; + *sequenceNumberAt = *i; + dataAt += sizeof(unsigned short int); + } + + // send it + OctreeServer::didCallWriteDatagram(this); + NodeList::getInstance()->writeDatagram((char*)packet, dataAt - packet, _node); + + return 1; +} + + + + + quint64 OctreeSendThread::_usleepTime = 0; quint64 OctreeSendThread::_usleepCalls = 0; diff --git a/assignment-client/src/octree/OctreeSendThread.h b/assignment-client/src/octree/OctreeSendThread.h index d8eed27802..041b44577c 100644 --- a/assignment-client/src/octree/OctreeSendThread.h +++ b/assignment-client/src/octree/OctreeSendThread.h @@ -50,6 +50,8 @@ private: int handlePacketSend(OctreeQueryNode* nodeData, int& trueBytesSent, int& truePacketsSent); int packetDistributor(OctreeQueryNode* nodeData, bool viewFrustumChanged); + + int sendNack(OctreeQueryNode* nodeData); OctreePacketData _packetData; diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 5595d139be..39536c0981 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -51,6 +51,8 @@ public: int getPacketsPerClientPerSecond() const { return getPacketsPerClientPerInterval() * INTERVALS_PER_SECOND; } int getPacketsTotalPerInterval() const { return _packetsTotalPerInterval; } int getPacketsTotalPerSecond() const { return getPacketsTotalPerInterval() * INTERVALS_PER_SECOND; } + + const OctreeInboundPacketProcessor* getInboundPacketProcessor() const { return _octreeInboundPacketProcessor; } static int getCurrentClientCount() { return _clientCount; } static void clientConnected() { _clientCount++; } diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index 8ac5333d10..d7c6bdead6 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -66,7 +66,8 @@ enum PacketType { PacketTypeModelAddOrEdit, PacketTypeModelErase, PacketTypeModelAddResponse, - PacketTypeOctreeDataNack + PacketTypeOctreeDataNack, + PacketTypeOctreeEditNack, }; typedef char PacketVersion; From c542da97076ba0d27f4e191ae5d8249dbf8afe7e Mon Sep 17 00:00:00 2001 From: wangyix Date: Thu, 12 Jun 2014 12:03:22 -0700 Subject: [PATCH 03/55] added locking on _singleSenderStats; untested! --- .../src/octree/OctreeInboundPacketProcessor.cpp | 8 +++++++- .../src/octree/OctreeInboundPacketProcessor.h | 3 +++ assignment-client/src/octree/OctreeSendThread.cpp | 13 ++++++------- assignment-client/src/octree/OctreeServer.h | 2 +- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index c1c564b58c..1912103bf0 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -25,7 +25,9 @@ OctreeInboundPacketProcessor::OctreeInboundPacketProcessor(OctreeServer* myServe _totalProcessTime(0), _totalLockWaitTime(0), _totalElementsInPacket(0), - _totalPackets(0) + _totalPackets(0), + _singleSenderStats(), + _singleSenderStatsLock() { } @@ -36,7 +38,9 @@ void OctreeInboundPacketProcessor::resetStats() { _totalElementsInPacket = 0; _totalPackets = 0; + _singleSenderStatsLock.lockForWrite(); _singleSenderStats.clear(); + _singleSenderStatsLock.unlock(); } @@ -143,7 +147,9 @@ void OctreeInboundPacketProcessor::trackInboundPacket(const QUuid& nodeUUID, uns if (_singleSenderStats.find(nodeUUID) == _singleSenderStats.end()) { SingleSenderStats stats; stats.trackInboundPacket(sequence, transitTime, editsInPacket, processTime, lockWaitTime); + _singleSenderStatsLock.lockForWrite(); _singleSenderStats[nodeUUID] = stats; + _singleSenderStatsLock.unlock(); } else { SingleSenderStats& stats = _singleSenderStats[nodeUUID]; stats.trackInboundPacket(sequence, transitTime, editsInPacket, processTime, lockWaitTime); diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 3e3f5f2dcb..c9760dcc5c 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -71,6 +71,8 @@ public: void resetStats(); + void lockSingleSenderStatsForRead() { _singleSenderStatsLock.lockForRead(); } + void unlockSingleSenderStats() { _singleSenderStatsLock.unlock(); } NodeToSenderStatsMap& getSingleSenderStats() { return _singleSenderStats; } const NodeToSenderStatsMap& getSingleSenderStats() const { return _singleSenderStats; } @@ -91,5 +93,6 @@ private: quint64 _totalPackets; NodeToSenderStatsMap _singleSenderStats; + QReadWriteLock _singleSenderStatsLock; }; #endif // hifi_OctreeInboundPacketProcessor_h diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 6b1119fca1..51696e35ea 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -120,7 +120,7 @@ int OctreeSendThread::sendNack(OctreeQueryNode* nodeData) { return 0; } - const OctreeInboundPacketProcessor* myServerPacketProcessor = _myServer->getInboundPacketProcessor(); + OctreeInboundPacketProcessor* myServerPacketProcessor = _myServer->getInboundPacketProcessor(); // if there are packets from _node that are waiting to be processed, // don't send a NACK since the missing packets may be among those waiting packets. @@ -128,18 +128,15 @@ int OctreeSendThread::sendNack(OctreeQueryNode* nodeData) { return 0; } - // lock unlock required ????????? prolly: _singleSenderStats may have our node's entry deleted during this or something - // maybe just make a copy instead!! + myServerPacketProcessor->lockSingleSenderStatsForRead(); - // lock - - const QSet missingSequenceNumbersFromNode = myServerPacketProcessor + const QSet& missingSequenceNumbersFromNode = myServerPacketProcessor ->getSingleSenderStats().at(_node->getUUID()).getMissingSequenceNumbers(); // check if there are any sequence numbers that need to be nacked int numSequenceNumbersAvailable = missingSequenceNumbersFromNode.size(); if (numSequenceNumbersAvailable == 0) { - //unlock + myServerPacketProcessor->unlockSingleSenderStats(); return 0; } @@ -170,6 +167,8 @@ int OctreeSendThread::sendNack(OctreeQueryNode* nodeData) { dataAt += sizeof(unsigned short int); } + myServerPacketProcessor->unlockSingleSenderStats(); + // send it OctreeServer::didCallWriteDatagram(this); NodeList::getInstance()->writeDatagram((char*)packet, dataAt - packet, _node); diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 39536c0981..22b2da0682 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -52,7 +52,7 @@ public: int getPacketsTotalPerInterval() const { return _packetsTotalPerInterval; } int getPacketsTotalPerSecond() const { return getPacketsTotalPerInterval() * INTERVALS_PER_SECOND; } - const OctreeInboundPacketProcessor* getInboundPacketProcessor() const { return _octreeInboundPacketProcessor; } + OctreeInboundPacketProcessor* getInboundPacketProcessor() { return _octreeInboundPacketProcessor; } static int getCurrentClientCount() { return _clientCount; } static void clientConnected() { _clientCount++; } From 1491216962d21fb6d273156af2a9ee21793b2592 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 13 Jun 2014 09:58:45 -0700 Subject: [PATCH 04/55] Revert "added locking on _singleSenderStats; untested!" This reverts commit c542da97076ba0d27f4e191ae5d8249dbf8afe7e. --- .../src/octree/OctreeInboundPacketProcessor.cpp | 8 +------- .../src/octree/OctreeInboundPacketProcessor.h | 3 --- assignment-client/src/octree/OctreeSendThread.cpp | 13 +++++++------ assignment-client/src/octree/OctreeServer.h | 2 +- 4 files changed, 9 insertions(+), 17 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 1912103bf0..c1c564b58c 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -25,9 +25,7 @@ OctreeInboundPacketProcessor::OctreeInboundPacketProcessor(OctreeServer* myServe _totalProcessTime(0), _totalLockWaitTime(0), _totalElementsInPacket(0), - _totalPackets(0), - _singleSenderStats(), - _singleSenderStatsLock() + _totalPackets(0) { } @@ -38,9 +36,7 @@ void OctreeInboundPacketProcessor::resetStats() { _totalElementsInPacket = 0; _totalPackets = 0; - _singleSenderStatsLock.lockForWrite(); _singleSenderStats.clear(); - _singleSenderStatsLock.unlock(); } @@ -147,9 +143,7 @@ void OctreeInboundPacketProcessor::trackInboundPacket(const QUuid& nodeUUID, uns if (_singleSenderStats.find(nodeUUID) == _singleSenderStats.end()) { SingleSenderStats stats; stats.trackInboundPacket(sequence, transitTime, editsInPacket, processTime, lockWaitTime); - _singleSenderStatsLock.lockForWrite(); _singleSenderStats[nodeUUID] = stats; - _singleSenderStatsLock.unlock(); } else { SingleSenderStats& stats = _singleSenderStats[nodeUUID]; stats.trackInboundPacket(sequence, transitTime, editsInPacket, processTime, lockWaitTime); diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index c9760dcc5c..3e3f5f2dcb 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -71,8 +71,6 @@ public: void resetStats(); - void lockSingleSenderStatsForRead() { _singleSenderStatsLock.lockForRead(); } - void unlockSingleSenderStats() { _singleSenderStatsLock.unlock(); } NodeToSenderStatsMap& getSingleSenderStats() { return _singleSenderStats; } const NodeToSenderStatsMap& getSingleSenderStats() const { return _singleSenderStats; } @@ -93,6 +91,5 @@ private: quint64 _totalPackets; NodeToSenderStatsMap _singleSenderStats; - QReadWriteLock _singleSenderStatsLock; }; #endif // hifi_OctreeInboundPacketProcessor_h diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 51696e35ea..6b1119fca1 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -120,7 +120,7 @@ int OctreeSendThread::sendNack(OctreeQueryNode* nodeData) { return 0; } - OctreeInboundPacketProcessor* myServerPacketProcessor = _myServer->getInboundPacketProcessor(); + const OctreeInboundPacketProcessor* myServerPacketProcessor = _myServer->getInboundPacketProcessor(); // if there are packets from _node that are waiting to be processed, // don't send a NACK since the missing packets may be among those waiting packets. @@ -128,15 +128,18 @@ int OctreeSendThread::sendNack(OctreeQueryNode* nodeData) { return 0; } - myServerPacketProcessor->lockSingleSenderStatsForRead(); + // lock unlock required ????????? prolly: _singleSenderStats may have our node's entry deleted during this or something + // maybe just make a copy instead!! - const QSet& missingSequenceNumbersFromNode = myServerPacketProcessor + // lock + + const QSet missingSequenceNumbersFromNode = myServerPacketProcessor ->getSingleSenderStats().at(_node->getUUID()).getMissingSequenceNumbers(); // check if there are any sequence numbers that need to be nacked int numSequenceNumbersAvailable = missingSequenceNumbersFromNode.size(); if (numSequenceNumbersAvailable == 0) { - myServerPacketProcessor->unlockSingleSenderStats(); + //unlock return 0; } @@ -167,8 +170,6 @@ int OctreeSendThread::sendNack(OctreeQueryNode* nodeData) { dataAt += sizeof(unsigned short int); } - myServerPacketProcessor->unlockSingleSenderStats(); - // send it OctreeServer::didCallWriteDatagram(this); NodeList::getInstance()->writeDatagram((char*)packet, dataAt - packet, _node); diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 22b2da0682..39536c0981 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -52,7 +52,7 @@ public: int getPacketsTotalPerInterval() const { return _packetsTotalPerInterval; } int getPacketsTotalPerSecond() const { return getPacketsTotalPerInterval() * INTERVALS_PER_SECOND; } - OctreeInboundPacketProcessor* getInboundPacketProcessor() { return _octreeInboundPacketProcessor; } + const OctreeInboundPacketProcessor* getInboundPacketProcessor() const { return _octreeInboundPacketProcessor; } static int getCurrentClientCount() { return _clientCount; } static void clientConnected() { _clientCount++; } From 3d4fae4b3f611bbc4ce98dade9c1845a92b55779 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 13 Jun 2014 09:58:56 -0700 Subject: [PATCH 05/55] Revert "added code for AC to send nack packets; no locking yet" This reverts commit 15bd0878c407ede95f724009adadc81deeb7b4fd. --- .../octree/OctreeInboundPacketProcessor.cpp | 2 +- .../src/octree/OctreeInboundPacketProcessor.h | 2 - .../src/octree/OctreeSendThread.cpp | 70 ------------------- .../src/octree/OctreeSendThread.h | 2 - assignment-client/src/octree/OctreeServer.h | 2 - libraries/networking/src/PacketHeaders.h | 3 +- 6 files changed, 2 insertions(+), 79 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index c1c564b58c..52407f78d5 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -212,4 +212,4 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, _totalLockWaitTime += lockWaitTime; _totalElementsInPacket += editsInPacket; _totalPackets++; -} +} \ No newline at end of file diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 3e3f5f2dcb..82471bcddf 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -35,7 +35,6 @@ public: void trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime); - const QSet& getMissingSequenceNumbers() const { return _missingSequenceNumbers; } quint64 _totalTransitTime; @@ -72,7 +71,6 @@ public: void resetStats(); NodeToSenderStatsMap& getSingleSenderStats() { return _singleSenderStats; } - const NodeToSenderStatsMap& getSingleSenderStats() const { return _singleSenderStats; } protected: virtual void processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet); diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 6b1119fca1..cb149b1d96 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -85,7 +85,6 @@ bool OctreeSendThread::process() { if (nodeData && !nodeData->isShuttingDown()) { bool viewFrustumChanged = nodeData->updateCurrentViewFrustum(); packetDistributor(nodeData, viewFrustumChanged); - sendNack(nodeData); } } } @@ -112,75 +111,6 @@ bool OctreeSendThread::process() { return isStillRunning(); // keep running till they terminate us } - -int OctreeSendThread::sendNack(OctreeQueryNode* nodeData) { - - // if we're shutting down, then exit early - if (nodeData->isShuttingDown()) { - return 0; - } - - const OctreeInboundPacketProcessor* myServerPacketProcessor = _myServer->getInboundPacketProcessor(); - - // if there are packets from _node that are waiting to be processed, - // don't send a NACK since the missing packets may be among those waiting packets. - if (myServerPacketProcessor->hasPacketsToProcessFrom(_node)) { - return 0; - } - - // lock unlock required ????????? prolly: _singleSenderStats may have our node's entry deleted during this or something - // maybe just make a copy instead!! - - // lock - - const QSet missingSequenceNumbersFromNode = myServerPacketProcessor - ->getSingleSenderStats().at(_node->getUUID()).getMissingSequenceNumbers(); - - // check if there are any sequence numbers that need to be nacked - int numSequenceNumbersAvailable = missingSequenceNumbersFromNode.size(); - if (numSequenceNumbersAvailable == 0) { - //unlock - return 0; - } - - // construct nack packet - - char packet[MAX_PACKET_SIZE]; - - char* dataAt = packet; - int bytesRemaining = MAX_PACKET_SIZE; - - // pack header - int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeEditNack); - dataAt += numBytesPacketHeader; - bytesRemaining -= numBytesPacketHeader; - int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(unsigned short int); - - // calculate and pack the number of sequence numbers - uint16_t numSequenceNumbers = std::min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); - uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; - *numSequenceNumbersAt = numSequenceNumbers; - dataAt += sizeof(uint16_t); - - // pack sequence numbers - QSet::const_iterator begin = missingSequenceNumbersFromNode.begin(), end = missingSequenceNumbersFromNode.end(); - for (QSet::const_iterator i = begin; i != end; i++) { - unsigned short int* sequenceNumberAt = (unsigned short int*)dataAt; - *sequenceNumberAt = *i; - dataAt += sizeof(unsigned short int); - } - - // send it - OctreeServer::didCallWriteDatagram(this); - NodeList::getInstance()->writeDatagram((char*)packet, dataAt - packet, _node); - - return 1; -} - - - - - quint64 OctreeSendThread::_usleepTime = 0; quint64 OctreeSendThread::_usleepCalls = 0; diff --git a/assignment-client/src/octree/OctreeSendThread.h b/assignment-client/src/octree/OctreeSendThread.h index 041b44577c..d8eed27802 100644 --- a/assignment-client/src/octree/OctreeSendThread.h +++ b/assignment-client/src/octree/OctreeSendThread.h @@ -50,8 +50,6 @@ private: int handlePacketSend(OctreeQueryNode* nodeData, int& trueBytesSent, int& truePacketsSent); int packetDistributor(OctreeQueryNode* nodeData, bool viewFrustumChanged); - - int sendNack(OctreeQueryNode* nodeData); OctreePacketData _packetData; diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 39536c0981..5595d139be 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -51,8 +51,6 @@ public: int getPacketsPerClientPerSecond() const { return getPacketsPerClientPerInterval() * INTERVALS_PER_SECOND; } int getPacketsTotalPerInterval() const { return _packetsTotalPerInterval; } int getPacketsTotalPerSecond() const { return getPacketsTotalPerInterval() * INTERVALS_PER_SECOND; } - - const OctreeInboundPacketProcessor* getInboundPacketProcessor() const { return _octreeInboundPacketProcessor; } static int getCurrentClientCount() { return _clientCount; } static void clientConnected() { _clientCount++; } diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index d7c6bdead6..8ac5333d10 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -66,8 +66,7 @@ enum PacketType { PacketTypeModelAddOrEdit, PacketTypeModelErase, PacketTypeModelAddResponse, - PacketTypeOctreeDataNack, - PacketTypeOctreeEditNack, + PacketTypeOctreeDataNack }; typedef char PacketVersion; From b210b07b81c8a12ad243e7af7ed9f65080115ef3 Mon Sep 17 00:00:00 2001 From: wangyix Date: Fri, 13 Jun 2014 11:08:15 -0700 Subject: [PATCH 06/55] rollovers are now handled in SingleSenderStats::trackInboundPacket --- .../octree/OctreeInboundPacketProcessor.cpp | 34 +++++++++++++++---- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 52407f78d5..931814d4f8 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -156,6 +156,7 @@ SingleSenderStats::SingleSenderStats() _totalLockWaitTime(0), _totalElementsInPacket(0), _totalPackets(0), + _incomingLastSequence(0), _missingSequenceNumbers() { @@ -168,23 +169,42 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, const int MAX_MISSING_SEQUENCE_SIZE = 100; unsigned short int expectedSequence = _totalPackets == 0 ? incomingSequence : _incomingLastSequence + 1; - + if (incomingSequence == expectedSequence) { // on time _incomingLastSequence = incomingSequence; } - else { - // ignore packet if sequence number gap is unreasonable - if (std::abs(incomingSequence - expectedSequence) > MAX_REASONABLE_SEQUENCE_GAP) { + else { // out of order + + const int UINT16_RANGE = 65536; + + int incoming = (int)incomingSequence; + int expected = (int)expectedSequence; + + // check if the gap between incoming and expected is reasonable, taking possible rollover into consideration + int absGap = std::abs(incoming - expected); + if (absGap >= UINT16_RANGE - MAX_REASONABLE_SEQUENCE_GAP) { + // rollover likely occurred between incoming and expected. + // correct the larger of the two so that it's within [-65536, -1] while the other remains within [0, 65535] + if (incoming > expected) { + incoming -= UINT16_RANGE; + } else { + expected -= UINT16_RANGE; + } + } else if (absGap > MAX_REASONABLE_SEQUENCE_GAP) { + // ignore packet if gap is unreasonable qDebug() << "ignoring unreasonable packet... sequence:" << incomingSequence << "_incomingLastSequence:" << _incomingLastSequence; return; } + + // now that rollover has been corrected for (if it occurred), incoming and expected can be + // compared to each other directly, though one of them might be negative - if (incomingSequence > expectedSequence) { // early + if (incoming > expected) { // early // add all sequence numbers that were skipped to the missing sequence numbers list - for (int missingSequence = expectedSequence; missingSequence < incomingSequence; missingSequence++) { - _missingSequenceNumbers.insert(missingSequence); + for (int missingSequence = expected; missingSequence < incoming; missingSequence++) { + _missingSequenceNumbers.insert(missingSequence < 0 ? missingSequence + UINT16_RANGE : missingSequence); } _incomingLastSequence = incomingSequence; From 7db05f2c8e822635d28f31af8c63095ef26518ac Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 13 Jun 2014 16:25:38 -0700 Subject: [PATCH 07/55] Working on JSON encoding. --- libraries/metavoxels/src/Bitstream.cpp | 223 +++++++++++++++++++++++++ libraries/metavoxels/src/Bitstream.h | 54 +++++- 2 files changed, 276 insertions(+), 1 deletion(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 80aa07b026..1266e8efa4 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1657,6 +1657,49 @@ const TypeStreamer* Bitstream::createInvalidTypeStreamer() { return streamer; } +void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { + if (!_typeStreamerNames.contains(streamer->getName())) { + _typeStreamerNames.insert(streamer->getName()); + + // start with a placeholder; then remove/replace with actual metadata + int index = _typeStreamers.size(); + _typeStreamers.append(QJsonValue()); + QJsonValue metadata = streamer->getJSONMetadata(*this); + if (metadata.isNull()) { + _typeStreamers.removeAt(index); + } else { + _typeStreamers.replace(index, metadata); + } + } +} + +void JSONWriter::addObjectStreamer(const ObjectStreamer* streamer) { + if (!_objectStreamerNames.contains(streamer->getName())) { + _objectStreamerNames.insert(streamer->getName()); + + // start with a placeholder; then replace with actual metadata + int index = _objectStreamers.size(); + _objectStreamers.append(QJsonValue()); + _objectStreamers.replace(index, streamer->getJSONMetadata(*this)); + } +} + +void JSONWriter::addSharedObject(const SharedObjectPointer& object) { + if (!_sharedObjectIDs.contains(object->getID())) { + _sharedObjectIDs.insert(object->getID()); + + // start with a placeholder; then replace with actual object + int index = _sharedObjects.size(); + _sharedObjects.append(QJsonValue()); + + QJsonObject sharedObject; + sharedObject.insert("id", object->getID()); + sharedObject.insert("originID", object->getOriginID()); + + _sharedObjects.replace(index, sharedObject); + } +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } @@ -1699,6 +1742,33 @@ void MappedObjectStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonObject MappedObjectStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(_metaObject->className())); + QJsonArray properties; + foreach (const StreamerPropertyPair& property, _properties) { + QJsonObject object; + writer.addTypeStreamer(property.first.data()); + object.insert("type", QString(property.first->getName())); + object.insert("name", QString(property.second.name())); + properties.append(object); + } + metadata.insert("properties", properties); + return metadata; +} + +QJsonObject MappedObjectStreamer::getJSONData(JSONWriter& writer, const QObject* object) const { + QJsonObject data; + writer.addObjectStreamer(this); + data.insert("class", QString(_metaObject->className())); + QJsonArray properties; + foreach (const StreamerPropertyPair& property, _properties) { + properties.append(property.first->getJSONData(writer, property.second.read(object))); + } + data.insert("properties", properties); + return data; +} + void MappedObjectStreamer::write(Bitstream& out, const QObject* object) const { foreach (const StreamerPropertyPair& property, _properties) { property.first->write(out, property.second.read(object)); @@ -1775,6 +1845,34 @@ void GenericObjectStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonObject GenericObjectStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(_name)); + QJsonArray properties; + foreach (const StreamerNamePair& property, _properties) { + QJsonObject object; + writer.addTypeStreamer(property.first.data()); + object.insert("type", QString(property.first->getName())); + object.insert("name", QString(property.second)); + properties.append(object); + } + metadata.insert("properties", properties); + return metadata; +} + +QJsonObject GenericObjectStreamer::getJSONData(JSONWriter& writer, const QObject* object) const { + QJsonObject data; + writer.addObjectStreamer(this); + data.insert("class", QString(_name)); + QJsonArray properties; + const QVariantList& values = static_cast(object)->getValues(); + for (int i = 0; i < _properties.size(); i++) { + properties.append(_properties.at(i).first->getJSONData(writer, values.at(i))); + } + data.insert("properties", properties); + return data; +} + void GenericObjectStreamer::write(Bitstream& out, const QObject* object) const { const QVariantList& values = static_cast(object)->getValues(); for (int i = 0; i < _properties.size(); i++) { @@ -1876,6 +1974,55 @@ void TypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue TypeStreamer::getJSONMetadata(JSONWriter& writer) const { + Category category = getCategory(); + switch (category) { + case STREAMABLE_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("STREAMABLE")); + QJsonArray fields; + foreach (const MetaField& metaField, getMetaFields()) { + QJsonObject field; + writer.addTypeStreamer(metaField.getStreamer()); + field.insert("type", QString(metaField.getStreamer()->getName())); + field.insert("name", QString(metaField.getName())); + fields.append(field); + } + metadata.insert("fields", fields); + return metadata; + } + case LIST_CATEGORY: + case SET_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString(category == LIST_CATEGORY ? "LIST" : "SET")); + const TypeStreamer* valueStreamer = getValueStreamer(); + writer.addTypeStreamer(valueStreamer); + metadata.insert("valueType", QString(valueStreamer->getName())); + return metadata; + } + case MAP_CATEGORY: { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("MAP")); + const TypeStreamer* keyStreamer = getKeyStreamer(); + writer.addTypeStreamer(keyStreamer); + metadata.insert("keyType", QString(keyStreamer->getName())); + const TypeStreamer* valueStreamer = getValueStreamer(); + writer.addTypeStreamer(valueStreamer); + metadata.insert("valueType", QString(valueStreamer->getName())); + return metadata; + } + default: + return QJsonValue(); + } +} + +QJsonValue TypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return QJsonValue(); +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2045,6 +2192,22 @@ void EnumTypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue EnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("ENUM")); + QJsonArray values; + QMetaEnum metaEnum = getMetaEnum(); + for (int i = 0; i < metaEnum.keyCount(); i++) { + QJsonObject value; + value.insert("key", QString(metaEnum.key(i))); + value.insert("value", metaEnum.value(i)); + values.append(value); + } + metadata.insert("values", values); + return metadata; +} + TypeStreamer::Category EnumTypeStreamer::getCategory() const { return ENUM_CATEGORY; } @@ -2194,6 +2357,21 @@ void GenericEnumTypeStreamer::writeMetadata(Bitstream& out, bool full) const { } } +QJsonValue GenericEnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("ENUM")); + QJsonArray values; + foreach (const NameIntPair& value, _values) { + QJsonObject object; + object.insert("key", QString(value.first)); + object.insert("value", value.second); + values.append(object); + } + metadata.insert("values", values); + return metadata; +} + void GenericEnumTypeStreamer::write(Bitstream& out, const QVariant& value) const { int intValue = value.toInt(); out.write(&intValue, _bits); @@ -2270,6 +2448,22 @@ void GenericStreamableTypeStreamer::writeMetadata(Bitstream& out, bool full) con } } +QJsonValue GenericStreamableTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("STREAMABLE")); + QJsonArray fields; + foreach (const StreamerNamePair& field, _fields) { + QJsonObject object; + writer.addTypeStreamer(field.first.data()); + object.insert("type", QString(field.first->getName())); + object.insert("name", QString(field.second)); + fields.append(object); + } + metadata.insert("fields", fields); + return metadata; +} + void GenericStreamableTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); for (int i = 0; i < _fields.size(); i++) { @@ -2334,6 +2528,15 @@ void GenericListTypeStreamer::writeMetadata(Bitstream& out, bool full) const { out << _valueStreamer.data(); } +QJsonValue GenericListTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("LIST")); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + void GenericListTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); out << values.size(); @@ -2376,6 +2579,15 @@ GenericSetTypeStreamer::GenericSetTypeStreamer(const QByteArray& name, const Typ GenericListTypeStreamer(name, valueStreamer) { } +QJsonValue GenericSetTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("SET")); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + TypeStreamer::Category GenericSetTypeStreamer::getCategory() const { return SET_CATEGORY; } @@ -2437,6 +2649,17 @@ void GenericMapTypeStreamer::writeMetadata(Bitstream& out, bool full) const { out << _keyStreamer.data() << _valueStreamer.data(); } +QJsonValue GenericMapTypeStreamer::getJSONMetadata(JSONWriter& writer) const { + QJsonObject metadata; + metadata.insert("name", QString(getName())); + metadata.insert("category", QString("MAP")); + writer.addTypeStreamer(_keyStreamer.data()); + metadata.insert("keyType", QString(_keyStreamer->getName())); + writer.addTypeStreamer(_valueStreamer.data()); + metadata.insert("valueType", QString(_valueStreamer->getName())); + return metadata; +} + void GenericMapTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantPairList values = value.value(); out << values.size(); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 1589473b0e..732d1f8f7b 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -13,6 +13,8 @@ #define hifi_Bitstream_h #include +#include +#include #include #include #include @@ -766,6 +768,41 @@ template inline Bitstream& Bitstream::operator>>(QHash& return *this; } +/// Tracks state when writing to JSON. +class JSONWriter { +public: + + void addTypeStreamer(const TypeStreamer* streamer); + void addObjectStreamer(const ObjectStreamer* streamer); + void addSharedObject(const SharedObjectPointer& object); + +private: + + QSet _typeStreamerNames; + QJsonArray _typeStreamers; + + QSet _objectStreamerNames; + QJsonArray _objectStreamers; + + QSet _sharedObjectIDs; + QJsonArray _sharedObjects; +}; + +/// Tracks state when reading from JSON. +class JSONReader { +public: + + TypeStreamerPointer getTypeStreamer(const QByteArray& name) const { return _typeStreamers.value(name); } + ObjectStreamerPointer getObjectStreamer(const QByteArray& name) const { return _objectStreamers.value(name); } + SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } + +private: + + QHash _typeStreamers; + QHash _objectStreamers; + QHash _sharedObjects; +}; + typedef QPair StreamerPropertyPair; /// Contains the information required to stream an object. @@ -780,6 +817,8 @@ public: virtual const char* getName() const = 0; virtual const QVector& getProperties() const; virtual void writeMetadata(Bitstream& out, bool full) const = 0; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const = 0; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const = 0; virtual void write(Bitstream& out, const QObject* object) const = 0; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const = 0; virtual QObject* read(Bitstream& in, QObject* object = NULL) const = 0; @@ -802,6 +841,8 @@ public: virtual const char* getName() const; virtual const QVector& getProperties() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -822,6 +863,8 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; + virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -913,6 +956,9 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual bool equal(const QVariant& first, const QVariant& second) const; virtual void write(Bitstream& out, const QVariant& value) const; @@ -990,6 +1036,7 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual Category getCategory() const; virtual int getBits() const; virtual QMetaEnum getMetaEnum() const; @@ -1051,6 +1098,7 @@ public: GenericEnumTypeStreamer(const QByteArray& name, const QVector& values, int bits, const QByteArray& hash); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1099,6 +1147,7 @@ public: GenericStreamableTypeStreamer(const QByteArray& name, const QVector& fields, const QByteArray& hash); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1169,11 +1218,12 @@ public: GenericListTypeStreamer(const QByteArray& name, const TypeStreamerPointer& valueStreamer); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; -private: +protected: TypeStreamerPointer _valueStreamer; }; @@ -1206,6 +1256,7 @@ public: GenericSetTypeStreamer(const QByteArray& name, const TypeStreamerPointer& valueStreamer); + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual Category getCategory() const; }; @@ -1250,6 +1301,7 @@ public: const TypeStreamerPointer& valueStreamer); virtual void writeMetadata(Bitstream& out, bool full) const; + virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; From aac7459f42846e0c924e24a78d25c9a670b0841b Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Fri, 13 Jun 2014 18:32:57 -0700 Subject: [PATCH 08/55] More work on JSON encoding. --- libraries/metavoxels/src/Bitstream.cpp | 180 +++++++++++++++++++++++- libraries/metavoxels/src/Bitstream.h | 48 ++++++- tests/metavoxels/src/MetavoxelTests.cpp | 12 ++ 3 files changed, 232 insertions(+), 8 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 1266e8efa4..aa5ed3fa5e 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1657,6 +1657,109 @@ const TypeStreamer* Bitstream::createInvalidTypeStreamer() { return streamer; } +JSONWriter& JSONWriter::operator<<(bool value) { + _contents.append(value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(int value) { + _contents.append(value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(uint value) { + _contents.append((int)value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(float value) { + _contents.append((double)value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QByteArray& value) { + _contents.append(QString(value.toPercentEncoding())); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QColor& value) { + _contents.append(value.name()); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QScriptValue& value) { + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QString& value) { + _contents.append(value); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QUrl& value) { + _contents.append(value.toString()); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const glm::vec3& value) { + QJsonArray array; + array.append(value.x); + array.append(value.y); + array.append(value.z); + _contents.append(array); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const glm::quat& value) { + QJsonArray array; + array.append(value.x); + array.append(value.y); + array.append(value.z); + array.append(value.w); + _contents.append(array); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QMetaObject* metaObject) { + if (!metaObject) { + _contents.append(QJsonValue()); + return *this; + } + const ObjectStreamer* streamer = Bitstream::getObjectStreamers().value(metaObject); + addObjectStreamer(streamer); + _contents.append(QString(streamer->getName())); + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QVariant& value) { + if (!value.isValid()) { + _contents.append(QJsonValue()); + return *this; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamers().value(value.userType()); + if (streamer) { + _contents.append(streamer->getJSONVariantData(*this, value)); + } else { + qWarning() << "Non-streamable type:" << value.typeName(); + } + return *this; +} + +JSONWriter& JSONWriter::operator<<(const SharedObjectPointer& object) { + if (object) { + addSharedObject(object); + _contents.append(object->getID()); + } else { + _contents.append(0); + } + return *this; +} + +JSONWriter& JSONWriter::operator<<(const QObject* object) { + _contents.append(getData(object)); + return *this; +} + void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { if (!_typeStreamerNames.contains(streamer->getName())) { _typeStreamerNames.insert(streamer->getName()); @@ -1695,11 +1798,31 @@ void JSONWriter::addSharedObject(const SharedObjectPointer& object) { QJsonObject sharedObject; sharedObject.insert("id", object->getID()); sharedObject.insert("originID", object->getOriginID()); - + sharedObject.insert("data", getData(object.data())); _sharedObjects.replace(index, sharedObject); } } +QJsonDocument JSONWriter::getDocument() const { + QJsonObject top; + top.insert("contents", _contents); + top.insert("objects", _sharedObjects); + top.insert("classes", _objectStreamers); + top.insert("types", _typeStreamers); + return QJsonDocument(top); +} + +QJsonValue JSONWriter::getData(const QObject* object) { + if (!object) { + return QJsonValue(); + } + const QMetaObject* metaObject = object->metaObject(); + const ObjectStreamer* streamer = (metaObject == &GenericSharedObject::staticMetaObject) ? + static_cast(object)->getStreamer().data() : + Bitstream::getObjectStreamers().value(metaObject); + return streamer->getJSONData(*this, object); +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } @@ -2023,6 +2146,14 @@ QJsonValue TypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) return QJsonValue(); } +QJsonValue TypeStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& value) const { + writer.addTypeStreamer(this); + QJsonObject data; + data.insert("type", QString(getName())); + data.insert("value", getJSONData(writer, value)); + return data; +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2208,6 +2339,10 @@ QJsonValue EnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { return metadata; } +QJsonValue EnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return value.toInt(); +} + TypeStreamer::Category EnumTypeStreamer::getCategory() const { return ENUM_CATEGORY; } @@ -2372,6 +2507,10 @@ QJsonValue GenericEnumTypeStreamer::getJSONMetadata(JSONWriter& writer) const { return metadata; } +QJsonValue GenericEnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + return value.toInt(); +} + void GenericEnumTypeStreamer::write(Bitstream& out, const QVariant& value) const { int intValue = value.toInt(); out.write(&intValue, _bits); @@ -2464,6 +2603,15 @@ QJsonValue GenericStreamableTypeStreamer::getJSONMetadata(JSONWriter& writer) co return metadata; } +QJsonValue GenericStreamableTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantList values = value.toList(); + QJsonArray array; + for (int i = 0; i < _fields.size(); i++) { + array.append(_fields.at(i).first->getJSONData(writer, values.at(i))); + } + return array; +} + void GenericStreamableTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); for (int i = 0; i < _fields.size(); i++) { @@ -2537,6 +2685,15 @@ QJsonValue GenericListTypeStreamer::getJSONMetadata(JSONWriter& writer) const { return metadata; } +QJsonValue GenericListTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantList values = value.toList(); + QJsonArray array; + foreach (const QVariant& element, values) { + array.append(_valueStreamer->getJSONData(writer, element)); + } + return array; +} + void GenericListTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); out << values.size(); @@ -2660,6 +2817,18 @@ QJsonValue GenericMapTypeStreamer::getJSONMetadata(JSONWriter& writer) const { return metadata; } +QJsonValue GenericMapTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& value) const { + QVariantPairList values = value.value(); + QJsonArray array; + foreach (const QVariantPair& pair, values) { + QJsonArray pairArray; + pairArray.append(_keyStreamer->getJSONData(writer, pair.first)); + pairArray.append(_valueStreamer->getJSONData(writer, pair.second)); + array.append(pairArray); + } + return array; +} + void GenericMapTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantPairList values = value.value(); out << values.size(); @@ -2685,6 +2854,15 @@ TypeStreamer::Category GenericMapTypeStreamer::getCategory() const { return MAP_CATEGORY; } +QJsonValue GenericValueStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& value) const { + GenericValue genericValue = value.value(); + writer.addTypeStreamer(genericValue.getStreamer().data()); + QJsonObject data; + data.insert("type", QString(genericValue.getStreamer()->getName())); + data.insert("value", genericValue.getStreamer()->getJSONData(writer, genericValue.getValue())); + return data; +} + void GenericValueStreamer::writeVariant(Bitstream& out, const QVariant& value) const { GenericValue genericValue = value.value(); out << genericValue.getStreamer().data(); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 732d1f8f7b..159633c810 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -38,6 +39,7 @@ class Attribute; class AttributeValue; class Bitstream; class GenericValue; +class JSONWriter; class ObjectReader; class ObjectStreamer; class OwnedAttributeValue; @@ -447,6 +449,8 @@ private slots: private: + friend class JSONWriter; + ObjectStreamerPointer readGenericObjectStreamer(const QByteArray& name); TypeStreamerPointer readGenericTypeStreamer(const QByteArray& name, int category); @@ -771,21 +775,44 @@ template inline Bitstream& Bitstream::operator>>(QHash& /// Tracks state when writing to JSON. class JSONWriter { public: - - void addTypeStreamer(const TypeStreamer* streamer); - void addObjectStreamer(const ObjectStreamer* streamer); + + JSONWriter& operator<<(bool value); + JSONWriter& operator<<(int value); + JSONWriter& operator<<(uint value); + JSONWriter& operator<<(float value); + JSONWriter& operator<<(const QByteArray& value); + JSONWriter& operator<<(const QColor& value); + JSONWriter& operator<<(const QScriptValue& value); + JSONWriter& operator<<(const QString& value); + JSONWriter& operator<<(const QUrl& value); + JSONWriter& operator<<(const glm::vec3& value); + JSONWriter& operator<<(const glm::quat& value); + JSONWriter& operator<<(const QMetaObject* metaObject); + + JSONWriter& operator<<(const QVariant& value); + JSONWriter& operator<<(const SharedObjectPointer& object); + JSONWriter& operator<<(const QObject* object); + void addSharedObject(const SharedObjectPointer& object); + void addObjectStreamer(const ObjectStreamer* streamer); + void addTypeStreamer(const TypeStreamer* streamer); + + QJsonDocument getDocument() const; private: - QSet _typeStreamerNames; - QJsonArray _typeStreamers; + QJsonValue getData(const QObject* object); + + QJsonArray _contents; + + QSet _sharedObjectIDs; + QJsonArray _sharedObjects; QSet _objectStreamerNames; QJsonArray _objectStreamers; - QSet _sharedObjectIDs; - QJsonArray _sharedObjects; + QSet _typeStreamerNames; + QJsonArray _typeStreamers; }; /// Tracks state when reading from JSON. @@ -958,6 +985,7 @@ public: virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; virtual bool equal(const QVariant& first, const QVariant& second) const; @@ -1037,6 +1065,7 @@ public: virtual const char* getName() const; virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual Category getCategory() const; virtual int getBits() const; virtual QMetaEnum getMetaEnum() const; @@ -1099,6 +1128,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1148,6 +1178,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1219,6 +1250,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1302,6 +1334,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1316,6 +1349,7 @@ private: class GenericValueStreamer : public SimpleTypeStreamer { public: + QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; virtual void writeVariant(Bitstream& out, const QVariant& value) const; }; diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 6bd99a6c82..618178383f 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -243,6 +243,18 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { return true; } + if (metadataType != Bitstream::FULL_METADATA) { + return false; + } + + // now write to JSON + JSONWriter jsonWriter; + jsonWriter << testObjectReadA; + jsonWriter << testObjectReadB; + jsonWriter << messageRead; + qDebug() << jsonWriter.getDocument().toJson(); + qDebug(); + return false; } From f4bd2a8beb31a88a564a7d93977c7b15b4a81a9d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 15 Jun 2014 16:23:54 -0700 Subject: [PATCH 09/55] I think that basically covers the JSON writing portion. --- libraries/metavoxels/src/Bitstream.cpp | 176 ++++++++++++++++--------- libraries/metavoxels/src/Bitstream.h | 83 +++++++++--- 2 files changed, 181 insertions(+), 78 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index aa5ed3fa5e..b88f536d1a 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1657,107 +1657,172 @@ const TypeStreamer* Bitstream::createInvalidTypeStreamer() { return streamer; } -JSONWriter& JSONWriter::operator<<(bool value) { - _contents.append(value); - return *this; +QJsonValue JSONWriter::getData(bool value) { + return value; } -JSONWriter& JSONWriter::operator<<(int value) { - _contents.append(value); - return *this; +QJsonValue JSONWriter::getData(int value) { + return value; } -JSONWriter& JSONWriter::operator<<(uint value) { - _contents.append((int)value); - return *this; +QJsonValue JSONWriter::getData(uint value) { + return (int)value; } -JSONWriter& JSONWriter::operator<<(float value) { - _contents.append((double)value); - return *this; +QJsonValue JSONWriter::getData(float value) { + return (double)value; } -JSONWriter& JSONWriter::operator<<(const QByteArray& value) { - _contents.append(QString(value.toPercentEncoding())); - return *this; +QJsonValue JSONWriter::getData(const QByteArray& value) { + return QString(value.toPercentEncoding()); } -JSONWriter& JSONWriter::operator<<(const QColor& value) { - _contents.append(value.name()); - return *this; +QJsonValue JSONWriter::getData(const QColor& value) { + return value.name(); } -JSONWriter& JSONWriter::operator<<(const QScriptValue& value) { - return *this; +QJsonValue JSONWriter::getData(const QScriptValue& value) { + QJsonObject object; + if (value.isUndefined()) { + object.insert("type", QString("UNDEFINED")); + + } else if (value.isNull()) { + object.insert("type", QString("NULL")); + + } else if (value.isBool()) { + object.insert("type", QString("BOOL")); + object.insert("value", value.toBool()); + + } else if (value.isNumber()) { + object.insert("type", QString("NUMBER")); + object.insert("value", value.toNumber()); + + } else if (value.isString()) { + object.insert("type", QString("STRING")); + object.insert("value", value.toString()); + + } else if (value.isVariant()) { + object.insert("type", QString("VARIANT")); + object.insert("value", getData(value.toVariant())); + + } else if (value.isQObject()) { + object.insert("type", QString("QOBJECT")); + object.insert("value", getData(value.toQObject())); + + } else if (value.isQMetaObject()) { + object.insert("type", QString("QMETAOBJECT")); + object.insert("value", getData(value.toQMetaObject())); + + } else if (value.isDate()) { + object.insert("type", QString("DATE")); + object.insert("value", getData(value.toDateTime())); + + } else if (value.isRegExp()) { + object.insert("type", QString("REGEXP")); + object.insert("value", getData(value.toRegExp())); + + } else if (value.isArray()) { + object.insert("type", QString("ARRAY")); + QJsonArray array; + int length = value.property(ScriptCache::getInstance()->getLengthString()).toInt32(); + for (int i = 0; i < length; i++) { + array.append(getData(value.property(i))); + } + object.insert("value", array); + + } else if (value.isObject()) { + object.insert("type", QString("OBJECT")); + QJsonObject valueObject; + for (QScriptValueIterator it(value); it.hasNext(); ) { + it.next(); + valueObject.insert(it.name(), getData(it.value())); + } + object.insert("value", valueObject); + + } else { + object.insert("type", QString("INVALID")); + } + return object; } -JSONWriter& JSONWriter::operator<<(const QString& value) { - _contents.append(value); - return *this; +QJsonValue JSONWriter::getData(const QString& value) { + return value; } -JSONWriter& JSONWriter::operator<<(const QUrl& value) { - _contents.append(value.toString()); - return *this; +QJsonValue JSONWriter::getData(const QUrl& value) { + return value.toString(); } -JSONWriter& JSONWriter::operator<<(const glm::vec3& value) { +QJsonValue JSONWriter::getData(const QDateTime& value) { + return (qsreal)value.toMSecsSinceEpoch(); +} + +QJsonValue JSONWriter::getData(const QRegExp& value) { + QJsonObject object; + object.insert("pattern", value.pattern()); + object.insert("caseSensitivity", (int)value.caseSensitivity()); + object.insert("patternSyntax", (int)value.patternSyntax()); + object.insert("minimal", value.isMinimal()); + return object; +} + +QJsonValue JSONWriter::getData(const glm::vec3& value) { QJsonArray array; array.append(value.x); array.append(value.y); array.append(value.z); - _contents.append(array); - return *this; + return array; } -JSONWriter& JSONWriter::operator<<(const glm::quat& value) { +QJsonValue JSONWriter::getData(const glm::quat& value) { QJsonArray array; array.append(value.x); array.append(value.y); array.append(value.z); array.append(value.w); - _contents.append(array); - return *this; + return array; } -JSONWriter& JSONWriter::operator<<(const QMetaObject* metaObject) { +QJsonValue JSONWriter::getData(const QMetaObject* metaObject) { if (!metaObject) { - _contents.append(QJsonValue()); - return *this; + return QJsonValue(); } const ObjectStreamer* streamer = Bitstream::getObjectStreamers().value(metaObject); addObjectStreamer(streamer); - _contents.append(QString(streamer->getName())); - return *this; + return QString(streamer->getName()); } -JSONWriter& JSONWriter::operator<<(const QVariant& value) { +QJsonValue JSONWriter::getData(const QVariant& value) { if (!value.isValid()) { - _contents.append(QJsonValue()); - return *this; + return QJsonValue(); } const TypeStreamer* streamer = Bitstream::getTypeStreamers().value(value.userType()); if (streamer) { - _contents.append(streamer->getJSONVariantData(*this, value)); + return streamer->getJSONVariantData(*this, value); } else { qWarning() << "Non-streamable type:" << value.typeName(); + return QJsonValue(); } - return *this; } -JSONWriter& JSONWriter::operator<<(const SharedObjectPointer& object) { +QJsonValue JSONWriter::getData(const SharedObjectPointer& object) { if (object) { addSharedObject(object); - _contents.append(object->getID()); + return object->getID(); } else { - _contents.append(0); + return 0; } - return *this; } -JSONWriter& JSONWriter::operator<<(const QObject* object) { - _contents.append(getData(object)); - return *this; +QJsonValue JSONWriter::getData(const QObject* object) { + if (!object) { + return QJsonValue(); + } + const QMetaObject* metaObject = object->metaObject(); + const ObjectStreamer* streamer = (metaObject == &GenericSharedObject::staticMetaObject) ? + static_cast(object)->getStreamer().data() : + Bitstream::getObjectStreamers().value(metaObject); + return streamer->getJSONData(*this, object); } void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { @@ -1798,7 +1863,7 @@ void JSONWriter::addSharedObject(const SharedObjectPointer& object) { QJsonObject sharedObject; sharedObject.insert("id", object->getID()); sharedObject.insert("originID", object->getOriginID()); - sharedObject.insert("data", getData(object.data())); + sharedObject.insert("data", getData(static_cast(object.data()))); _sharedObjects.replace(index, sharedObject); } } @@ -1812,17 +1877,6 @@ QJsonDocument JSONWriter::getDocument() const { return QJsonDocument(top); } -QJsonValue JSONWriter::getData(const QObject* object) { - if (!object) { - return QJsonValue(); - } - const QMetaObject* metaObject = object->metaObject(); - const ObjectStreamer* streamer = (metaObject == &GenericSharedObject::staticMetaObject) ? - static_cast(object)->getStreamer().data() : - Bitstream::getObjectStreamers().value(metaObject); - return streamer->getJSONData(*this, object); -} - ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 159633c810..23943444d9 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -776,22 +776,32 @@ template inline Bitstream& Bitstream::operator>>(QHash& class JSONWriter { public: - JSONWriter& operator<<(bool value); - JSONWriter& operator<<(int value); - JSONWriter& operator<<(uint value); - JSONWriter& operator<<(float value); - JSONWriter& operator<<(const QByteArray& value); - JSONWriter& operator<<(const QColor& value); - JSONWriter& operator<<(const QScriptValue& value); - JSONWriter& operator<<(const QString& value); - JSONWriter& operator<<(const QUrl& value); - JSONWriter& operator<<(const glm::vec3& value); - JSONWriter& operator<<(const glm::quat& value); - JSONWriter& operator<<(const QMetaObject* metaObject); + QJsonValue getData(bool value); + QJsonValue getData(int value); + QJsonValue getData(uint value); + QJsonValue getData(float value); + QJsonValue getData(const QByteArray& value); + QJsonValue getData(const QColor& value); + QJsonValue getData(const QScriptValue& value); + QJsonValue getData(const QString& value); + QJsonValue getData(const QUrl& value); + QJsonValue getData(const QDateTime& value); + QJsonValue getData(const QRegExp& value); + QJsonValue getData(const glm::vec3& value); + QJsonValue getData(const glm::quat& value); + QJsonValue getData(const QMetaObject* value); + QJsonValue getData(const QVariant& value); + QJsonValue getData(const SharedObjectPointer& value); + QJsonValue getData(const QObject* value); - JSONWriter& operator<<(const QVariant& value); - JSONWriter& operator<<(const SharedObjectPointer& object); - JSONWriter& operator<<(const QObject* object); + template QJsonValue getData(const T& value); + + template QJsonValue getData(const QList& list); + template QJsonValue getData(const QVector& list); + template QJsonValue getData(const QSet& set); + template QJsonValue getData(const QHash& hash); + + template JSONWriter& operator<<(const T& value) { _contents.append(getData(value)); return *this; } void addSharedObject(const SharedObjectPointer& object); void addObjectStreamer(const ObjectStreamer* streamer); @@ -801,8 +811,6 @@ public: private: - QJsonValue getData(const QObject* object); - QJsonArray _contents; QSet _sharedObjectIDs; @@ -815,6 +823,45 @@ private: QJsonArray _typeStreamers; }; +template inline QJsonValue JSONWriter::getData(const T& value) { + return QJsonValue(); +} + +template inline QJsonValue JSONWriter::getData(const QList& list) { + QJsonArray array; + foreach (const T& value, list) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QVector& vector) { + QJsonArray array; + foreach (const T& value, vector) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QSet& set) { + QJsonArray array; + foreach (const T& value, set) { + array.append(getData(value)); + } + return array; +} + +template inline QJsonValue JSONWriter::getData(const QHash& hash) { + QJsonArray array; + for (typename QHash::const_iterator it = hash.constBegin(); it != hash.constEnd(); it++) { + QJsonArray pair; + pair.append(getData(it.key())); + pair.append(getData(it.value())); + array.append(pair); + } + return array; +} + /// Tracks state when reading from JSON. class JSONReader { public: @@ -1042,6 +1089,8 @@ QDebug& operator<<(QDebug& debug, const QMetaObject* metaObject); template class SimpleTypeStreamer : public TypeStreamer { public: + virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const { + return writer.getData(value.value()); } virtual bool equal(const QVariant& first, const QVariant& second) const { return first.value() == second.value(); } virtual void write(Bitstream& out, const QVariant& value) const { out << value.value(); } virtual QVariant read(Bitstream& in) const { T value; in >> value; return QVariant::fromValue(value); } From 3ec127afde9eaf23fae65779f6c7633ae19623a9 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 15 Jun 2014 16:46:43 -0700 Subject: [PATCH 10/55] Spoke a little too soon. --- libraries/metavoxels/src/Bitstream.h | 6 +++++- tools/mtc/src/main.cpp | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 23943444d9..e9599e9e87 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -1418,6 +1418,7 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; @@ -1427,6 +1428,7 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ __attribute__((unused)) static const int* _TypePtr##X = &X::Type; @@ -1437,6 +1439,7 @@ public: Bitstream& operator>>(Bitstream& in, X& obj); \ template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ + template<> QJsonValue JSONWriter::getData(const X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; \ @@ -1447,7 +1450,8 @@ public: Bitstream& operator<<(Bitstream& out, const S::N& obj); \ Bitstream& operator>>(Bitstream& in, S::N& obj); \ template<> inline void Bitstream::writeRawDelta(const S::N& value, const S::N& reference) { *this << value; } \ - template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } + template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } \ + template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } #define IMPLEMENT_ENUM_METATYPE(S, N) \ static int S##N##MetaTypeId = registerEnumMetaType(&S::staticMetaObject, #N); \ diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index 096ade4625..eadccdb310 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -217,6 +217,19 @@ void generateOutput (QTextStream& out, const QList& streamables) { } out << "}\n"; + out << "template<> QJsonValue JSONWriter::getData(const " << name << "& value) {\n"; + out << " QJsonArray array;\n"; + foreach (const QString& base, str.clazz.bases) { + out << " foreach (const QJsonValue& element, getData(static_cast(value)).toArray()) {\n"; + out << " array.append(element);\n"; + out << " }\n"; + } + foreach (const Field& field, str.fields) { + out << " array.append(getData(value." << field.name << "));\n"; + } + out << " return array;\n"; + out << "}\n"; + out << "bool operator==(const " << name << "& first, const " << name << "& second) {\n"; if (str.clazz.bases.isEmpty() && str.fields.isEmpty()) { out << " return true"; From 24ec441df76cc498643b03221b5d1f175c81c745 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Sun, 15 Jun 2014 17:57:09 -0700 Subject: [PATCH 11/55] Starting to work on JSON reading. --- libraries/metavoxels/src/Bitstream.cpp | 43 ++++++++++++++++++++++++++ libraries/metavoxels/src/Bitstream.h | 10 +++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index b88f536d1a..91e2cd8723 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1877,6 +1877,49 @@ QJsonDocument JSONWriter::getDocument() const { return QJsonDocument(top); } +JSONReader::JSONReader(const QJsonDocument& document) { + QJsonObject top = document.object(); + QJsonArray types = top.value("types").toArray(); + for (int i = types.size() - 1; i >= 0; i--) { + QJsonObject type = types.at(i).toObject(); + QString name = type.value("name").toString(); + QString category = type.value("category").toString(); + if (category == "ENUM") { + + } else if (category == "STREAMABLE") { + + } else if (category == "LIST") { + + } else if (category == "SET") { + + } else if (category == "MAP") { + + } + } + + QJsonArray classes = top.value("classes").toArray(); + for (int i = classes.size() - 1; i >= 0; i--) { + QJsonObject clazz = classes.at(i).toObject(); + QString name = clazz.value("name").toString(); + QJsonArray properties = clazz.value("properties").toArray(); + foreach (const QJsonValue& property, properties) { + QJsonObject object = property.toObject(); + object.value("type"); + object.value("name"); + } + } + + QJsonArray objects = top.value("objects").toArray(); + for (int i = objects.size() - 1; i >= 0; i--) { + QJsonObject object = objects.at(i).toObject(); + int id = object.value("id").toInt(); + int originID = object.value("originID").toInt(); + QJsonObject data = object.value("data").toObject(); + + } + +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index e9599e9e87..afacbf6dac 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -866,14 +866,16 @@ template inline QJsonValue JSONWriter::getData(const QHash _typeStreamers; - QHash _objectStreamers; + QHash _typeStreamers; + QHash _objectStreamers; QHash _sharedObjects; }; From 2b20720f51578f365685845b1f835895b50535de Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 09:57:05 -0700 Subject: [PATCH 12/55] added sendNackPackets() to OctreeInboundPacketProcessor added rollover handling in _missingSequenceNumbers pruning; added EditNack packet types; added getMyEditNackType() to OctreeServer subclasses; added code to randomly skip edit packet sequence numbers for testing in OctreeEditPacketSender --- assignment-client/src/models/ModelServer.h | 1 + .../octree/OctreeInboundPacketProcessor.cpp | 111 ++++++++++++++++-- .../src/octree/OctreeInboundPacketProcessor.h | 11 +- assignment-client/src/octree/OctreeServer.cpp | 6 +- assignment-client/src/octree/OctreeServer.h | 1 + .../src/particles/ParticleServer.h | 1 + assignment-client/src/voxels/VoxelServer.h | 1 + libraries/networking/src/PacketHeaders.h | 5 +- .../networking/src/ReceivedPacketProcessor.h | 6 +- .../octree/src/OctreeEditPacketSender.cpp | 7 ++ libraries/shared/src/GenericThread.h | 2 +- 11 files changed, 133 insertions(+), 19 deletions(-) diff --git a/assignment-client/src/models/ModelServer.h b/assignment-client/src/models/ModelServer.h index 38acc7f1e1..8a26f07f11 100644 --- a/assignment-client/src/models/ModelServer.h +++ b/assignment-client/src/models/ModelServer.h @@ -33,6 +33,7 @@ public: virtual const char* getMyServerName() const { return MODEL_SERVER_NAME; } virtual const char* getMyLoggingServerTargetName() const { return MODEL_SERVER_LOGGING_TARGET_NAME; } virtual const char* getMyDefaultPersistFilename() const { return LOCAL_MODELS_PERSIST_FILE; } + virtual PacketType getMyEditNackType() const { return PacketTypeModelEditNack; } // subclass may implement these method virtual void beforeRun(); diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 931814d4f8..683fe188b4 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -39,7 +39,6 @@ void OctreeInboundPacketProcessor::resetStats() { _singleSenderStats.clear(); } - void OctreeInboundPacketProcessor::processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet) { bool debugProcessPacket = _myServer->wantsVerboseDebug(); @@ -150,6 +149,76 @@ void OctreeInboundPacketProcessor::trackInboundPacket(const QUuid& nodeUUID, uns } } +int OctreeInboundPacketProcessor::sendNackPackets() { + + printf("\t\t sendNackPackets()\n"); + + int packetsSent = 0; + + // if there are packets from _node that are waiting to be processed, + // don't send a NACK since the missing packets may be among those waiting packets. + + NodeToSenderStatsMap::const_iterator begin = _singleSenderStats.begin(), end = _singleSenderStats.end(); + for (NodeToSenderStatsMap::const_iterator i = begin; i != end; i++) { + + QUuid nodeUUID = i.key(); + SingleSenderStats nodeStats = i.value(); + + if (hasPacketsToProcessFrom(nodeUUID)) { + continue; + } + + const SharedNodePointer& destinationNode = NodeList::getInstance()->getNodeHash().value(nodeUUID); + const QSet& missingSequenceNumbers = nodeStats.getMissingSequenceNumbers(); + + // check if there are any sequence numbers that need to be nacked + int numSequenceNumbersAvailable = missingSequenceNumbers.size(); + + // construct nack packet(s) for this node + + QSet::const_iterator missingSequenceNumberIterator = missingSequenceNumbers.begin(); + char packet[MAX_PACKET_SIZE]; + + while (numSequenceNumbersAvailable > 0) { + + char* dataAt = packet; + int bytesRemaining = MAX_PACKET_SIZE; + + // pack header + int numBytesPacketHeader = populatePacketHeader(packet, _myServer->getMyEditNackType()); + dataAt += numBytesPacketHeader; + bytesRemaining -= numBytesPacketHeader; + + // calculate and pack the number of sequence numbers to nack + int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(unsigned short int); + uint16_t numSequenceNumbers = std::min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); + uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; + *numSequenceNumbersAt = numSequenceNumbers; + dataAt += sizeof(uint16_t); + + // pack sequence numbers to nack + printf("\t\t sending NACK with %d seq numbers:\n\t\t", numSequenceNumbers); + for (uint16_t i = 0; i < numSequenceNumbers; i++) { + unsigned short int* sequenceNumberAt = (unsigned short int*)dataAt; + *sequenceNumberAt = *missingSequenceNumberIterator; + dataAt += sizeof(unsigned short int); + printf("%d, ", *missingSequenceNumberIterator); + + missingSequenceNumberIterator++; + } + numSequenceNumbersAvailable -= numSequenceNumbers; + + // send it + qint64 bytesWritten = NodeList::getInstance()->writeDatagram(packet, dataAt - packet, destinationNode); + printf("\t\t wrote %lld bytes\n\n", bytesWritten); + + packetsSent++; + } + } + return packetsSent; +} + + SingleSenderStats::SingleSenderStats() : _totalTransitTime(0), _totalProcessTime(0), @@ -165,17 +234,19 @@ SingleSenderStats::SingleSenderStats() void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { - const int MAX_REASONABLE_SEQUENCE_GAP = 1000; +printf("\t\t tracked seq %d\n", incomingSequence); + + const int UINT16_RANGE = 65536; + + const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work const int MAX_MISSING_SEQUENCE_SIZE = 100; unsigned short int expectedSequence = _totalPackets == 0 ? incomingSequence : _incomingLastSequence + 1; - if (incomingSequence == expectedSequence) { // on time + if (incomingSequence == expectedSequence) { // on time _incomingLastSequence = incomingSequence; } - else { // out of order - - const int UINT16_RANGE = 65536; + else { // out of order int incoming = (int)incomingSequence; int expected = (int)expectedSequence; @@ -202,6 +273,8 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, if (incoming > expected) { // early + printf("\t\t\t packet is early! %d packets were skipped\n", incoming - expected); + // add all sequence numbers that were skipped to the missing sequence numbers list for (int missingSequence = expected; missingSequence < incoming; missingSequence++) { _missingSequenceNumbers.insert(missingSequence < 0 ? missingSequence + UINT16_RANGE : missingSequence); @@ -210,6 +283,8 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, } else { // late + printf("\t\t\t packet is late!\n"); + // remove this from missing sequence number if it's in there _missingSequenceNumbers.remove(incomingSequence); @@ -217,11 +292,27 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, } } - // prune missing sequence list if it gets too big + // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP + // will be removed. if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE_SIZE) { - foreach(unsigned short int missingSequence, _missingSequenceNumbers) { - if (missingSequence <= std::max(0, _incomingLastSequence - MAX_REASONABLE_SEQUENCE_GAP)) { - _missingSequenceNumbers.remove(missingSequence); + + // the acceptable range of older sequence numbers may contain a rollover point; this must be handled. + // some sequence number in this list may be larger than _incomingLastSequence, indicating that they were received + // before the most recent rollover. + int cutoff = (int)_incomingLastSequence - MAX_REASONABLE_SEQUENCE_GAP; + if (cutoff >= 0) { + foreach(unsigned short int missingSequence, _missingSequenceNumbers) { + unsigned short int nonRolloverCutoff = (unsigned short int)cutoff; + if (missingSequence > _incomingLastSequence || missingSequence <= nonRolloverCutoff) { + _missingSequenceNumbers.remove(missingSequence); + } + } + } else { + unsigned short int rolloverCutoff = (unsigned short int)(cutoff + UINT16_RANGE); + foreach(unsigned short int missingSequence, _missingSequenceNumbers) { + if (missingSequence > _incomingLastSequence && missingSequence <= rolloverCutoff) { + _missingSequenceNumbers.remove(missingSequence); + } } } } diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 82471bcddf..9649da2d61 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -32,6 +32,7 @@ public: { return _totalElementsInPacket == 0 ? 0 : _totalProcessTime / _totalElementsInPacket; } quint64 getAverageLockWaitTimePerElement() const { return _totalElementsInPacket == 0 ? 0 : _totalLockWaitTime / _totalElementsInPacket; } + const QSet& getMissingSequenceNumbers() const { return _missingSequenceNumbers; } void trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime); @@ -47,8 +48,9 @@ public: QSet _missingSequenceNumbers; }; -typedef std::map NodeToSenderStatsMap; -typedef std::map::iterator NodeToSenderStatsMapIterator; +typedef QHash NodeToSenderStatsMap; +typedef QHash::iterator NodeToSenderStatsMapIterator; +typedef QHash::const_iterator NodeToSenderStatsMapConstIterator; /// Handles processing of incoming network packets for the voxel-server. As with other ReceivedPacketProcessor classes @@ -75,6 +77,9 @@ public: protected: virtual void processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet); +public slots: + int sendNackPackets(); + private: void trackInboundPacket(const QUuid& nodeUUID, unsigned short int sequence, quint64 transitTime, int voxelsInPacket, quint64 processTime, quint64 lockWaitTime); @@ -87,7 +92,7 @@ private: quint64 _totalLockWaitTime; quint64 _totalElementsInPacket; quint64 _totalPackets; - + NodeToSenderStatsMap _singleSenderStats; }; #endif // hifi_OctreeInboundPacketProcessor_h diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index cb60f0816e..39f51a0ba9 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -648,10 +648,10 @@ bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url int senderNumber = 0; NodeToSenderStatsMap& allSenderStats = _octreeInboundPacketProcessor->getSingleSenderStats(); - for (NodeToSenderStatsMapIterator i = allSenderStats.begin(); i != allSenderStats.end(); i++) { + for (NodeToSenderStatsMapConstIterator i = allSenderStats.begin(); i != allSenderStats.end(); i++) { senderNumber++; - QUuid senderID = i->first; - SingleSenderStats& senderStats = i->second; + QUuid senderID = i.key(); + const SingleSenderStats& senderStats = i.value(); statsString += QString("\r\n Stats for sender %1 uuid: %2\r\n") .arg(senderNumber).arg(senderID.toString()); diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 5595d139be..76b39c5771 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -68,6 +68,7 @@ public: virtual const char* getMyServerName() const = 0; virtual const char* getMyLoggingServerTargetName() const = 0; virtual const char* getMyDefaultPersistFilename() const = 0; + virtual PacketType getMyEditNackType() const = 0; // subclass may implement these method virtual void beforeRun() { }; diff --git a/assignment-client/src/particles/ParticleServer.h b/assignment-client/src/particles/ParticleServer.h index d444368a9d..0b379a903e 100644 --- a/assignment-client/src/particles/ParticleServer.h +++ b/assignment-client/src/particles/ParticleServer.h @@ -33,6 +33,7 @@ public: virtual const char* getMyServerName() const { return PARTICLE_SERVER_NAME; } virtual const char* getMyLoggingServerTargetName() const { return PARTICLE_SERVER_LOGGING_TARGET_NAME; } virtual const char* getMyDefaultPersistFilename() const { return LOCAL_PARTICLES_PERSIST_FILE; } + virtual PacketType getMyEditNackType() const { return PacketTypeParticleEditNack; } // subclass may implement these method virtual void beforeRun(); diff --git a/assignment-client/src/voxels/VoxelServer.h b/assignment-client/src/voxels/VoxelServer.h index b13f83b65f..fadcca2d19 100644 --- a/assignment-client/src/voxels/VoxelServer.h +++ b/assignment-client/src/voxels/VoxelServer.h @@ -42,6 +42,7 @@ public: virtual const char* getMyServerName() const { return VOXEL_SERVER_NAME; } virtual const char* getMyLoggingServerTargetName() const { return VOXEL_SERVER_LOGGING_TARGET_NAME; } virtual const char* getMyDefaultPersistFilename() const { return LOCAL_VOXELS_PERSIST_FILE; } + virtual PacketType getMyEditNackType() const { return PacketTypeVoxelEditNack; } // subclass may implement these method virtual void beforeRun(); diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index 8ac5333d10..5ed4110627 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -66,7 +66,10 @@ enum PacketType { PacketTypeModelAddOrEdit, PacketTypeModelErase, PacketTypeModelAddResponse, - PacketTypeOctreeDataNack + PacketTypeOctreeDataNack, // 45 + PacketTypeVoxelEditNack, + PacketTypeParticleEditNack, + PacketTypeModelEditNack, }; typedef char PacketVersion; diff --git a/libraries/networking/src/ReceivedPacketProcessor.h b/libraries/networking/src/ReceivedPacketProcessor.h index 94ad2d9c41..31cbc3a487 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.h +++ b/libraries/networking/src/ReceivedPacketProcessor.h @@ -35,7 +35,11 @@ public: /// Are there received packets waiting to be processed from a certain node bool hasPacketsToProcessFrom(const SharedNodePointer& sendingNode) const { - return _nodePacketCounts[sendingNode->getUUID()] > 0; + return hasPacketsToProcessFrom(sendingNode->getUUID()); + } + + bool hasPacketsToProcessFrom(const QUuid& nodeUUID) const { + return _nodePacketCounts[nodeUUID] > 0; } /// How many received packets waiting are to be processed diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 898c41de08..da1438136b 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -308,7 +308,14 @@ void OctreeEditPacketSender::initializePacket(EditPacketBuffer& packetBuffer, Pa unsigned short int* sequenceAt = (unsigned short int*)&packetBuffer._currentBuffer[packetBuffer._currentSize]; *sequenceAt = _sequenceNumber; packetBuffer._currentSize += sizeof(unsigned short int); // nudge past sequence + if (randFloat() < 0.6f) _sequenceNumber++; + else + { + int x = randIntInRange(2, 4); + printf("\t\t seq number jumped from %d to %d\n", _sequenceNumber, _sequenceNumber + x); + _sequenceNumber += x; + } // pack in timestamp quint64 now = usecTimestampNow(); diff --git a/libraries/shared/src/GenericThread.h b/libraries/shared/src/GenericThread.h index bbb01894ed..b2c0eb13db 100644 --- a/libraries/shared/src/GenericThread.h +++ b/libraries/shared/src/GenericThread.h @@ -57,7 +57,7 @@ protected: bool isStillRunning() const { return !_stopThread; } -private: +protected: QMutex _mutex; bool _stopThread; From ebfd65dea8cb86310f8c8c103b19c3217fac0e70 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 12:30:16 -0700 Subject: [PATCH 13/55] added OctreeInboundPacketProcessor::process() override to send nack periodically added code to remove dead nodes' stats in sendNackPackets() --- .../octree/OctreeInboundPacketProcessor.cpp | 62 +++++++++++++++++-- .../src/octree/OctreeInboundPacketProcessor.h | 7 ++- assignment-client/src/octree/OctreeServer.cpp | 3 + .../src/ReceivedPacketProcessor.cpp | 1 + .../networking/src/ReceivedPacketProcessor.h | 10 ++- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 683fe188b4..9b849905c5 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -25,7 +25,8 @@ OctreeInboundPacketProcessor::OctreeInboundPacketProcessor(OctreeServer* myServe _totalProcessTime(0), _totalLockWaitTime(0), _totalElementsInPacket(0), - _totalPackets(0) + _totalPackets(0), + _lastNackTime(usecTimestampNow()) { } @@ -35,10 +36,52 @@ void OctreeInboundPacketProcessor::resetStats() { _totalLockWaitTime = 0; _totalElementsInPacket = 0; _totalPackets = 0; + _lastNackTime = usecTimestampNow(); _singleSenderStats.clear(); } +bool OctreeInboundPacketProcessor::process() { + + const quint64 TOO_LONG_SINCE_LAST_NACK = 1 * USECS_PER_SECOND; + quint64 now = usecTimestampNow(); + + if (_packets.size() == 0) { + // calculate time until next sendNackPackets() + quint64 nextNackTime = _lastNackTime + TOO_LONG_SINCE_LAST_NACK; + quint64 now = usecTimestampNow(); + if (now >= nextNackTime) { + // send nacks if we're already past time to send it + _lastNackTime = now; + sendNackPackets(); + } + else { + // otherwise, wait until the next nack time or until a packet arrives + quint64 waitTimeMsecs = (nextNackTime - now) / USECS_PER_MSEC + 1; + _waitingOnPacketsMutex.lock(); + _hasPackets.wait(&_waitingOnPacketsMutex, waitTimeMsecs); + _waitingOnPacketsMutex.unlock(); + } + } + while (_packets.size() > 0) { + lock(); // lock to make sure nothing changes on us + NetworkPacket& packet = _packets.front(); // get the oldest packet + NetworkPacket temporary = packet; // make a copy of the packet in case the vector is resized on us + _packets.erase(_packets.begin()); // remove the oldest packet + _nodePacketCounts[temporary.getNode()->getUUID()]--; + unlock(); // let others add to the packets + processPacket(temporary.getNode(), temporary.getByteArray()); // process our temporary copy + + // if it's time to send nacks, send them. + if (usecTimestampNow() - _lastNackTime >= TOO_LONG_SINCE_LAST_NACK) { + _lastNackTime = now; + sendNackPackets(); + } + } + return isStillRunning(); // keep running till they terminate us +} + + void OctreeInboundPacketProcessor::processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet) { bool debugProcessPacket = _myServer->wantsVerboseDebug(); @@ -155,15 +198,21 @@ int OctreeInboundPacketProcessor::sendNackPackets() { int packetsSent = 0; - // if there are packets from _node that are waiting to be processed, - // don't send a NACK since the missing packets may be among those waiting packets. - - NodeToSenderStatsMap::const_iterator begin = _singleSenderStats.begin(), end = _singleSenderStats.end(); - for (NodeToSenderStatsMap::const_iterator i = begin; i != end; i++) { + NodeToSenderStatsMapIterator i = _singleSenderStats.begin(); + while (i != _singleSenderStats.end()) { QUuid nodeUUID = i.key(); SingleSenderStats nodeStats = i.value(); + // check if this node is still alive. Remove its stats if it's dead. + if (!isAlive(nodeUUID)) { + printf("\t\t removing node %s\n", nodeUUID.toString().toLatin1().data()); + i = _singleSenderStats.erase(i); + continue; + } + + // if there are packets from _node that are waiting to be processed, + // don't send a NACK since the missing packets may be among those waiting packets. if (hasPacketsToProcessFrom(nodeUUID)) { continue; } @@ -214,6 +263,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { packetsSent++; } + i++; } return packetsSent; } diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 9649da2d61..378cc9a891 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -75,9 +75,12 @@ public: NodeToSenderStatsMap& getSingleSenderStats() { return _singleSenderStats; } protected: + virtual void processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet); -public slots: + virtual bool process(); + +public: int sendNackPackets(); private: @@ -94,5 +97,7 @@ private: quint64 _totalPackets; NodeToSenderStatsMap _singleSenderStats; + + quint64 _lastNackTime; }; #endif // hifi_OctreeInboundPacketProcessor_h diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index 39f51a0ba9..c6ef4aa5aa 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -1060,6 +1060,9 @@ void OctreeServer::nodeAdded(SharedNodePointer node) { void OctreeServer::nodeKilled(SharedNodePointer node) { quint64 start = usecTimestampNow(); + // calling this here since nodeKilled slot in ReceivedPacketProcessor can't be triggered by signals yet!! + _octreeInboundPacketProcessor->nodeKilled(node); + qDebug() << qPrintable(_safeServerName) << "server killed node:" << *node; OctreeQueryNode* nodeData = static_cast(node->getLinkedData()); if (nodeData) { diff --git a/libraries/networking/src/ReceivedPacketProcessor.cpp b/libraries/networking/src/ReceivedPacketProcessor.cpp index 3ef518bbc2..d85f09fb0a 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.cpp +++ b/libraries/networking/src/ReceivedPacketProcessor.cpp @@ -54,4 +54,5 @@ void ReceivedPacketProcessor::nodeKilled(SharedNodePointer node) { lock(); _nodePacketCounts.remove(node->getUUID()); unlock(); + printf("\n\t\t nodeKilled()!!!!! --------------------------\n\n"); } diff --git a/libraries/networking/src/ReceivedPacketProcessor.h b/libraries/networking/src/ReceivedPacketProcessor.h index 31cbc3a487..4322c87910 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.h +++ b/libraries/networking/src/ReceivedPacketProcessor.h @@ -33,11 +33,17 @@ public: /// Are there received packets waiting to be processed bool hasPacketsToProcess() const { return _packets.size() > 0; } - /// Are there received packets waiting to be processed from a certain node + /// Is a specified node still alive? + bool isAlive(const QUuid& nodeUUID) const { + return _nodePacketCounts.contains(nodeUUID); + } + + /// Are there received packets waiting to be processed from a specified node bool hasPacketsToProcessFrom(const SharedNodePointer& sendingNode) const { return hasPacketsToProcessFrom(sendingNode->getUUID()); } + /// Are there received packets waiting to be processed from a specified node bool hasPacketsToProcessFrom(const QUuid& nodeUUID) const { return _nodePacketCounts[nodeUUID] > 0; } @@ -59,7 +65,7 @@ protected: virtual void terminating(); -private: +protected: QVector _packets; QHash _nodePacketCounts; From 0f7ce694c00dc2bd9da28eddcaf172c8970ac99d Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 13:52:01 -0700 Subject: [PATCH 14/55] minor changes before moving SentPacketHistory --- libraries/octree/src/OctreeEditPacketSender.cpp | 6 ++++-- libraries/octree/src/OctreeEditPacketSender.h | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index da1438136b..fb0059b96d 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -99,6 +99,8 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned c if (node->getActiveSocket()) { queuePacketForSending(node, QByteArray(reinterpret_cast(buffer), length)); + + // debugging output... bool wantDebugging = false; if (wantDebugging) { @@ -287,8 +289,8 @@ void OctreeEditPacketSender::releaseQueuedMessages() { if (!serversExist()) { _releaseQueuedMessagesPending = true; } else { - for (std::map::iterator i = _pendingEditPackets.begin(); i != _pendingEditPackets.end(); i++) { - releaseQueuedPacket(i->second); + for (QHash::iterator i = _pendingEditPackets.begin(); i != _pendingEditPackets.end(); i++) { + releaseQueuedPacket(i.value()); } } } diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index 0dc628c433..167e43b200 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -101,7 +101,7 @@ protected: void processPreServerExistsPackets(); // These are packets which are destined from know servers but haven't been released because they're still too small - std::map _pendingEditPackets; + QHash _pendingEditPackets; // These are packets that are waiting to be processed because we don't yet know if there are servers int _maxPendingMessages; From 69c2a2d12b4823e30cb26b2ce8902856e3350d9c Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 14:04:21 -0700 Subject: [PATCH 15/55] moved SentPacketHistory to libraries/networking/src --- .../networking/src/SentPacketHistory.cpp | 44 +++++++++++++++++++ libraries/networking/src/SentPacketHistory.h | 35 +++++++++++++++ 2 files changed, 79 insertions(+) create mode 100644 libraries/networking/src/SentPacketHistory.cpp create mode 100644 libraries/networking/src/SentPacketHistory.h diff --git a/libraries/networking/src/SentPacketHistory.cpp b/libraries/networking/src/SentPacketHistory.cpp new file mode 100644 index 0000000000..0ea7fd8b69 --- /dev/null +++ b/libraries/networking/src/SentPacketHistory.cpp @@ -0,0 +1,44 @@ +// +// SentPacketHistory.cpp +// assignement-client/src/octree +// +// Created by Yixin Wang on 6/5/2014 +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#include "SentPacketHistory.h" + +SentPacketHistory::SentPacketHistory(int size) + : _sentPackets(size), + _newestPacketAt(0), + _numExistingPackets(0), + _newestSequenceNumber(0) +{ +} + +void SentPacketHistory::packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const QByteArray& packet) { + _newestSequenceNumber = sequenceNumber; + + // increment _newestPacketAt cyclically, insert new packet there. + // this will overwrite the oldest packet in the buffer + _newestPacketAt = (_newestPacketAt == _sentPackets.size() - 1) ? 0 : _newestPacketAt + 1; + _sentPackets[_newestPacketAt] = packet; + if (_numExistingPackets < _sentPackets.size()) { + _numExistingPackets++; + } +} + + +const QByteArray* SentPacketHistory::getPacket(OCTREE_PACKET_SEQUENCE sequenceNumber) const { + OCTREE_PACKET_SEQUENCE seqDiff = _newestSequenceNumber - sequenceNumber; + if (!(seqDiff >= 0 && seqDiff < _numExistingPackets)) { + return NULL; + } + int packetAt = _newestPacketAt - seqDiff; + if (packetAt < 0) { + packetAt += _sentPackets.size(); + } + return &_sentPackets.at(packetAt); +} diff --git a/libraries/networking/src/SentPacketHistory.h b/libraries/networking/src/SentPacketHistory.h new file mode 100644 index 0000000000..4231400ac1 --- /dev/null +++ b/libraries/networking/src/SentPacketHistory.h @@ -0,0 +1,35 @@ +// +// SentPacketHistory.h +// assignement-client/src/octree +// +// Created by Yixin Wang on 6/5/2014 +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef hifi_SentPacketHistory_h +#define hifi_SentPacketHistory_h + +#include +#include + +#include "OctreePacketData.h" + +class SentPacketHistory { + +public: + SentPacketHistory(int size); + + void packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const QByteArray& packet); + const QByteArray* getPacket(OCTREE_PACKET_SEQUENCE sequenceNumber) const; + +private: + QVector _sentPackets; // circular buffer + int _newestPacketAt; + int _numExistingPackets; + + OCTREE_PACKET_SEQUENCE _newestSequenceNumber; +}; + +#endif From 14f50f4576b65dbe0bd257c5a30c3ced4efb8ceb Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 14:05:27 -0700 Subject: [PATCH 16/55] removed old SentPacketHistory --- .../src/octree/SentPacketHistory.cpp | 44 ------------------- .../src/octree/SentPacketHistory.h | 35 --------------- 2 files changed, 79 deletions(-) delete mode 100644 assignment-client/src/octree/SentPacketHistory.cpp delete mode 100644 assignment-client/src/octree/SentPacketHistory.h diff --git a/assignment-client/src/octree/SentPacketHistory.cpp b/assignment-client/src/octree/SentPacketHistory.cpp deleted file mode 100644 index 0ea7fd8b69..0000000000 --- a/assignment-client/src/octree/SentPacketHistory.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// -// SentPacketHistory.cpp -// assignement-client/src/octree -// -// Created by Yixin Wang on 6/5/2014 -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#include "SentPacketHistory.h" - -SentPacketHistory::SentPacketHistory(int size) - : _sentPackets(size), - _newestPacketAt(0), - _numExistingPackets(0), - _newestSequenceNumber(0) -{ -} - -void SentPacketHistory::packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const QByteArray& packet) { - _newestSequenceNumber = sequenceNumber; - - // increment _newestPacketAt cyclically, insert new packet there. - // this will overwrite the oldest packet in the buffer - _newestPacketAt = (_newestPacketAt == _sentPackets.size() - 1) ? 0 : _newestPacketAt + 1; - _sentPackets[_newestPacketAt] = packet; - if (_numExistingPackets < _sentPackets.size()) { - _numExistingPackets++; - } -} - - -const QByteArray* SentPacketHistory::getPacket(OCTREE_PACKET_SEQUENCE sequenceNumber) const { - OCTREE_PACKET_SEQUENCE seqDiff = _newestSequenceNumber - sequenceNumber; - if (!(seqDiff >= 0 && seqDiff < _numExistingPackets)) { - return NULL; - } - int packetAt = _newestPacketAt - seqDiff; - if (packetAt < 0) { - packetAt += _sentPackets.size(); - } - return &_sentPackets.at(packetAt); -} diff --git a/assignment-client/src/octree/SentPacketHistory.h b/assignment-client/src/octree/SentPacketHistory.h deleted file mode 100644 index 4231400ac1..0000000000 --- a/assignment-client/src/octree/SentPacketHistory.h +++ /dev/null @@ -1,35 +0,0 @@ -// -// SentPacketHistory.h -// assignement-client/src/octree -// -// Created by Yixin Wang on 6/5/2014 -// -// Distributed under the Apache License, Version 2.0. -// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html -// - -#ifndef hifi_SentPacketHistory_h -#define hifi_SentPacketHistory_h - -#include -#include - -#include "OctreePacketData.h" - -class SentPacketHistory { - -public: - SentPacketHistory(int size); - - void packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const QByteArray& packet); - const QByteArray* getPacket(OCTREE_PACKET_SEQUENCE sequenceNumber) const; - -private: - QVector _sentPackets; // circular buffer - int _newestPacketAt; - int _numExistingPackets; - - OCTREE_PACKET_SEQUENCE _newestSequenceNumber; -}; - -#endif From 77b6a209abeafd059af2148b2eac12254841fa09 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 16 Jun 2014 14:51:22 -0700 Subject: [PATCH 17/55] More progress on json reading. --- libraries/metavoxels/src/Bitstream.cpp | 272 ++++++++++++++++++++++++- libraries/metavoxels/src/Bitstream.h | 103 +++++++++- tools/mtc/src/main.cpp | 13 ++ 3 files changed, 379 insertions(+), 9 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 91e2cd8723..8f880a1803 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1825,6 +1825,10 @@ QJsonValue JSONWriter::getData(const QObject* object) { return streamer->getJSONData(*this, object); } +QJsonValue JSONWriter::getData(const GenericValue& value) { + return value.getStreamer()->getJSONData(*this, value.getValue()); +} + void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { if (!_typeStreamerNames.contains(streamer->getName())) { _typeStreamerNames.insert(streamer->getName()); @@ -1877,7 +1881,7 @@ QJsonDocument JSONWriter::getDocument() const { return QJsonDocument(top); } -JSONReader::JSONReader(const QJsonDocument& document) { +JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode) { QJsonObject top = document.object(); QJsonArray types = top.value("types").toArray(); for (int i = types.size() - 1; i >= 0; i--) { @@ -1918,6 +1922,155 @@ JSONReader::JSONReader(const QJsonDocument& document) { } + _contents = top.value("contents").toArray(); + _contentsIterator = _contents.constBegin(); +} + +void JSONReader::putData(const QJsonValue& data, bool& value) { + value = data.toBool(); +} + +void JSONReader::putData(const QJsonValue& data, int& value) { + value = data.toInt(); +} + +void JSONReader::putData(const QJsonValue& data, uint& value) { + value = data.toInt(); +} + +void JSONReader::putData(const QJsonValue& data, float& value) { + value = data.toDouble(); +} + +void JSONReader::putData(const QJsonValue& data, QByteArray& value) { + value = QByteArray::fromPercentEncoding(data.toString().toLatin1()); +} + +void JSONReader::putData(const QJsonValue& data, QColor& value) { + value.setNamedColor(data.toString()); +} + +void JSONReader::putData(const QJsonValue& data, QScriptValue& value) { + QJsonObject object = data.toObject(); + QString type = object.value("type").toString(); + if (type == "UNDEFINED") { + value = QScriptValue(QScriptValue::UndefinedValue); + + } else if (type == "NULL") { + value = QScriptValue(QScriptValue::NullValue); + + } else if (type == "BOOL") { + value = QScriptValue(object.value("value").toBool()); + + } else if (type == "NUMBER") { + value = QScriptValue(object.value("value").toDouble()); + + } else if (type == "STRING") { + value = QScriptValue(object.value("value").toString()); + + } else if (type == "VARIANT") { + QVariant variant; + putData(object.value("value"), variant); + value = ScriptCache::getInstance()->getEngine()->newVariant(variant); + + } else if (type == "QOBJECT") { + QObject* qObject; + putData(object.value("value"), qObject); + value = ScriptCache::getInstance()->getEngine()->newQObject(qObject, QScriptEngine::ScriptOwnership); + + } else if (type == "QMETAOBJECT") { + const QMetaObject* metaObject; + putData(object.value("value"), metaObject); + value = ScriptCache::getInstance()->getEngine()->newQMetaObject(metaObject); + + } else if (type == "DATE") { + QDateTime dateTime; + putData(object.value("value"), dateTime); + value = ScriptCache::getInstance()->getEngine()->newDate(dateTime); + + } else if (type == "REGEXP") { + QRegExp regExp; + putData(object.value("value"), regExp); + value = ScriptCache::getInstance()->getEngine()->newRegExp(regExp); + + } else if (type == "ARRAY") { + QJsonArray array = object.value("value").toArray(); + value = ScriptCache::getInstance()->getEngine()->newArray(array.size()); + for (int i = 0; i < array.size(); i++) { + QScriptValue element; + putData(array.at(i), element); + value.setProperty(i, element); + } + } else if (type == "OBJECT") { + QJsonObject jsonObject = object.value("value").toObject(); + value = ScriptCache::getInstance()->getEngine()->newObject(); + for (QJsonObject::const_iterator it = jsonObject.constBegin(); it != jsonObject.constEnd(); it++) { + QScriptValue element; + putData(it.value(), element); + value.setProperty(it.key(), element); + } + } else { + value = QScriptValue(); + } +} + +void JSONReader::putData(const QJsonValue& data, QString& value) { + value = data.toString(); +} + +void JSONReader::putData(const QJsonValue& data, QUrl& value) { + value = data.toString(); +} + +void JSONReader::putData(const QJsonValue& data, QDateTime& value) { + value.setMSecsSinceEpoch((qint64)data.toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, QRegExp& value) { + QJsonObject object = data.toObject(); + value = QRegExp(object.value("pattern").toString(), (Qt::CaseSensitivity)object.value("caseSensitivity").toInt(), + (QRegExp::PatternSyntax)object.value("patternSyntax").toInt()); + value.setMinimal(object.value("minimal").toBool()); +} + +void JSONReader::putData(const QJsonValue& data, glm::vec3& value) { + QJsonArray array = data.toArray(); + value = glm::vec3(array.at(0).toDouble(), array.at(1).toDouble(), array.at(2).toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, glm::quat& value) { + QJsonArray array = data.toArray(); + value = glm::quat(array.at(0).toDouble(), array.at(1).toDouble(), array.at(2).toDouble(), array.at(3).toDouble()); +} + +void JSONReader::putData(const QJsonValue& data, const QMetaObject*& value) { + ObjectStreamerPointer streamer = _objectStreamers.value(data.toString()); + value = streamer ? streamer->getMetaObject() : NULL; +} + +void JSONReader::putData(const QJsonValue& data, QVariant& value) { + QJsonObject object = data.toObject(); + QString type = object.value("type").toString(); + const TypeStreamer* streamer = _typeStreamers.value(type).data(); + if (!streamer) { + streamer = Bitstream::getTypeStreamers().value(QMetaType::type(type.toLatin1())); + if (!streamer) { + qWarning() << "Unknown type:" << type; + value = QVariant(); + return; + } + } + streamer->putJSONData(*this, object.value("value"), value); +} + +void JSONReader::putData(const QJsonValue& data, SharedObjectPointer& value) { + value = _sharedObjects.value(data.toInt()); +} + +void JSONReader::putData(const QJsonValue& data, QObject*& value) { + QJsonObject object = data.toObject(); + ObjectStreamerPointer streamer = _objectStreamers.value(object.value("class").toString()); + value = streamer ? streamer->putJSONData(*this, object) : NULL; } ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : @@ -1989,6 +2142,23 @@ QJsonObject MappedObjectStreamer::getJSONData(JSONWriter& writer, const QObject* return data; } +QObject* MappedObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const { + if (!_metaObject) { + return NULL; + } + QObject* object = _metaObject->newInstance(); + QJsonArray properties = jsonObject.value("properties").toArray(); + for (int i = 0; i < _properties.size(); i++) { + const StreamerPropertyPair& property = _properties.at(i); + if (property.second.isValid()) { + QVariant value; + property.first->putJSONData(reader, properties.at(i), value); + property.second.write(object, value); + } + } + return object; +} + void MappedObjectStreamer::write(Bitstream& out, const QObject* object) const { foreach (const StreamerPropertyPair& property, _properties) { property.first->write(out, property.second.read(object)); @@ -2093,6 +2263,19 @@ QJsonObject GenericObjectStreamer::getJSONData(JSONWriter& writer, const QObject return data; } +QObject* GenericObjectStreamer::putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const { + GenericSharedObject* object = new GenericSharedObject(_weakSelf); + QJsonArray properties = jsonObject.value("properties").toArray(); + QVariantList values; + for (int i = 0; i < _properties.size(); i++) { + QVariant value; + _properties.at(i).first->putJSONData(reader, properties.at(i), value); + values.append(value); + } + object->setValues(values); + return object; +} + void GenericObjectStreamer::write(Bitstream& out, const QObject* object) const { const QVariantList& values = static_cast(object)->getValues(); for (int i = 0; i < _properties.size(); i++) { @@ -2251,6 +2434,10 @@ QJsonValue TypeStreamer::getJSONVariantData(JSONWriter& writer, const QVariant& return data; } +void TypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(); +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2440,6 +2627,10 @@ QJsonValue EnumTypeStreamer::getJSONData(JSONWriter& writer, const QVariant& val return value.toInt(); } +void EnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = data.toInt(); +} + TypeStreamer::Category EnumTypeStreamer::getCategory() const { return ENUM_CATEGORY; } @@ -2532,6 +2723,11 @@ MappedEnumTypeStreamer::MappedEnumTypeStreamer(const TypeStreamer* baseStreamer, _mappings(mappings) { } +void MappedEnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + _baseStreamer->setEnumValue(value, data.toInt(), _mappings); +} + QVariant MappedEnumTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int value = 0; @@ -2608,6 +2804,10 @@ QJsonValue GenericEnumTypeStreamer::getJSONData(JSONWriter& writer, const QVaria return value.toInt(); } +void GenericEnumTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = data.toInt(); +} + void GenericEnumTypeStreamer::write(Bitstream& out, const QVariant& value) const { int intValue = value.toInt(); out.write(&intValue, _bits); @@ -2629,6 +2829,19 @@ MappedStreamableTypeStreamer::MappedStreamableTypeStreamer(const TypeStreamer* b _fields(fields) { } +void MappedStreamableTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + QJsonArray array = data.toArray(); + for (int i = 0; i < _fields.size(); i++) { + const StreamerIndexPair& pair = _fields.at(i); + if (pair.second != -1) { + QVariant element; + pair.first->putJSONData(reader, array.at(i), element); + _baseStreamer->setField(value, pair.second, element); + } + } +} + QVariant MappedStreamableTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); foreach (const StreamerIndexPair& pair, _fields) { @@ -2709,6 +2922,17 @@ QJsonValue GenericStreamableTypeStreamer::getJSONData(JSONWriter& writer, const return array; } +void GenericStreamableTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantList values; + for (int i = 0; i < _fields.size(); i++) { + QVariant element; + _fields.at(i).first->putJSONData(reader, array.at(i), element); + values.append(element); + } + value = values; +} + void GenericStreamableTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); for (int i = 0; i < _fields.size(); i++) { @@ -2733,6 +2957,15 @@ MappedListTypeStreamer::MappedListTypeStreamer(const TypeStreamer* baseStreamer, _valueStreamer(valueStreamer) { } +void MappedListTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + foreach (const QJsonValue& element, data.toArray()) { + QVariant elementValue; + _valueStreamer->putJSONData(reader, element, elementValue); + _baseStreamer->insert(value, elementValue); + } +} + QVariant MappedListTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int size; @@ -2791,6 +3024,17 @@ QJsonValue GenericListTypeStreamer::getJSONData(JSONWriter& writer, const QVaria return array; } +void GenericListTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantList values; + foreach (const QJsonValue& element, array) { + QVariant elementValue; + _valueStreamer->putJSONData(reader, element, elementValue); + values.append(elementValue); + } + value = values; +} + void GenericListTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantList values = value.toList(); out << values.size(); @@ -2853,6 +3097,18 @@ MappedMapTypeStreamer::MappedMapTypeStreamer(const TypeStreamer* baseStreamer, c _valueStreamer(valueStreamer) { } +void MappedMapTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + value = QVariant(_baseStreamer->getType(), 0); + foreach (const QJsonValue& element, data.toArray()) { + QJsonArray pair = element.toArray(); + QVariant elementKey; + _keyStreamer->putJSONData(reader, pair.at(0), elementKey); + QVariant elementValue; + _valueStreamer->putJSONData(reader, pair.at(1), elementValue); + _baseStreamer->insert(value, elementKey, elementValue); + } +} + QVariant MappedMapTypeStreamer::read(Bitstream& in) const { QVariant object = QVariant(_baseStreamer->getType(), 0); int size; @@ -2926,6 +3182,20 @@ QJsonValue GenericMapTypeStreamer::getJSONData(JSONWriter& writer, const QVarian return array; } +void GenericMapTypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QJsonArray array = data.toArray(); + QVariantPairList values; + foreach (const QJsonValue& element, array) { + QJsonArray pair = element.toArray(); + QVariant elementKey; + _keyStreamer->putJSONData(reader, pair.at(0), elementKey); + QVariant elementValue; + _valueStreamer->putJSONData(reader, pair.at(1), elementValue); + values.append(QVariantPair(elementKey, elementValue)); + } + value = QVariant::fromValue(values); +} + void GenericMapTypeStreamer::write(Bitstream& out, const QVariant& value) const { QVariantPairList values = value.value(); out << values.size(); diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index afacbf6dac..0cad33acea 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -449,6 +449,7 @@ private slots: private: + friend class JSONReader; friend class JSONWriter; ObjectStreamerPointer readGenericObjectStreamer(const QByteArray& name); @@ -477,7 +478,7 @@ private: static QHash& getMetaObjects(); static QMultiHash& getMetaObjectSubClasses(); static QHash& getTypeStreamers(); - + static const QHash& getObjectStreamers(); static QHash createObjectStreamers(); @@ -793,8 +794,9 @@ public: QJsonValue getData(const QVariant& value); QJsonValue getData(const SharedObjectPointer& value); QJsonValue getData(const QObject* value); + QJsonValue getData(const GenericValue& value); - template QJsonValue getData(const T& value); + template QJsonValue getData(const T& value) { return QJsonValue(); } template QJsonValue getData(const QList& list); template QJsonValue getData(const QVector& list); @@ -823,10 +825,6 @@ private: QJsonArray _typeStreamers; }; -template inline QJsonValue JSONWriter::getData(const T& value) { - return QJsonValue(); -} - template inline QJsonValue JSONWriter::getData(const QList& list) { QJsonArray array; foreach (const T& value, list) { @@ -866,19 +864,88 @@ template inline QJsonValue JSONWriter::getData(const QHash void putData(const QJsonValue& data, T& value) { value = T(); } + + template void putData(const QJsonValue& data, QList& list); + template void putData(const QJsonValue& data, QVector& list); + template void putData(const QJsonValue& data, QSet& set); + template void putData(const QJsonValue& data, QHash& hash); + + template JSONReader& operator>>(T& value) { putData(*_contentsIterator++, value); return *this; } + TypeStreamerPointer getTypeStreamer(const QString& name) const { return _typeStreamers.value(name); } ObjectStreamerPointer getObjectStreamer(const QString& name) const { return _objectStreamers.value(name); } SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } private: + QJsonArray _contents; + QJsonArray::const_iterator _contentsIterator; + QHash _typeStreamers; QHash _objectStreamers; QHash _sharedObjects; }; +template inline void JSONReader::putData(const QJsonValue& data, QList& list) { + list.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + list.append(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QVector& list) { + list.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + list.append(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QSet& set) { + set.clear(); + foreach (const QJsonValue& element, data.toArray()) { + T value; + putData(element, value); + set.insert(value); + } +} + +template inline void JSONReader::putData(const QJsonValue& data, QHash& hash) { + hash.clear(); + foreach (const QJsonValue& element, data.toArray()) { + QJsonArray pair = element.toArray(); + K key; + putData(pair.at(0), key); + V value; + putData(pair.at(1), value); + hash.insert(key, value); + } +} + typedef QPair StreamerPropertyPair; /// Contains the information required to stream an object. @@ -895,6 +962,8 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const = 0; virtual QJsonObject getJSONMetadata(JSONWriter& writer) const = 0; virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const = 0; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const = 0; + virtual void write(Bitstream& out, const QObject* object) const = 0; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const = 0; virtual QObject* read(Bitstream& in, QObject* object = NULL) const = 0; @@ -919,6 +988,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -941,6 +1011,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonObject getJSONMetadata(JSONWriter& writer) const; virtual QJsonObject getJSONData(JSONWriter& writer, const QObject* object) const; + virtual QObject* putJSONData(JSONReader& reader, const QJsonObject& jsonObject) const; virtual void write(Bitstream& out, const QObject* object) const; virtual void writeRawDelta(Bitstream& out, const QObject* object, const QObject* reference) const; virtual QObject* read(Bitstream& in, QObject* object = NULL) const; @@ -1035,6 +1106,7 @@ public: virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual bool equal(const QVariant& first, const QVariant& second) const; @@ -1093,6 +1165,8 @@ public: virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const { return writer.getData(value.value()); } + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + T rawValue; reader.putData(data, rawValue); value = QVariant::fromValue(rawValue); } virtual bool equal(const QVariant& first, const QVariant& second) const { return first.value() == second.value(); } virtual void write(Bitstream& out, const QVariant& value) const { out << value.value(); } virtual QVariant read(Bitstream& in) const { T value; in >> value; return QVariant::fromValue(value); } @@ -1117,6 +1191,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual Category getCategory() const; virtual int getBits() const; virtual QMetaEnum getMetaEnum() const; @@ -1144,6 +1219,7 @@ public: MappedEnumTypeStreamer(const TypeStreamer* baseStreamer, int bits, const QHash& mappings); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1180,6 +1256,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1212,6 +1289,7 @@ public: MappedStreamableTypeStreamer(const TypeStreamer* baseStreamer, const QVector& fields); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1230,6 +1308,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1284,6 +1363,7 @@ public: MappedListTypeStreamer(const TypeStreamer* baseStreamer, const TypeStreamerPointer& valueStreamer); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1302,6 +1382,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1366,6 +1447,7 @@ public: MappedMapTypeStreamer(const TypeStreamer* baseStreamer, const TypeStreamerPointer& keyStreamer, const TypeStreamerPointer& valueStreamer); + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual void readRawDelta(Bitstream& in, QVariant& object, const QVariant& reference) const; @@ -1386,6 +1468,7 @@ public: virtual void writeMetadata(Bitstream& out, bool full) const; virtual QJsonValue getJSONMetadata(JSONWriter& writer) const; virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; + virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual void write(Bitstream& out, const QVariant& value) const; virtual QVariant read(Bitstream& in) const; virtual Category getCategory() const; @@ -1421,6 +1504,7 @@ public: template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; @@ -1431,6 +1515,7 @@ public: template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ __attribute__((unused)) static const int* _TypePtr##X = &X::Type; @@ -1442,6 +1527,7 @@ public: template<> void Bitstream::writeRawDelta(const X& value, const X& reference); \ template<> void Bitstream::readRawDelta(X& value, const X& reference); \ template<> QJsonValue JSONWriter::getData(const X& value); \ + template<> void JSONReader::putData(const QJsonValue& data, X& value); \ bool operator==(const X& first, const X& second); \ bool operator!=(const X& first, const X& second); \ static const int* _TypePtr##X = &X::Type; \ @@ -1453,7 +1539,8 @@ public: Bitstream& operator>>(Bitstream& in, S::N& obj); \ template<> inline void Bitstream::writeRawDelta(const S::N& value, const S::N& reference) { *this << value; } \ template<> inline void Bitstream::readRawDelta(S::N& value, const S::N& reference) { *this >> value; } \ - template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } + template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } \ + template<> inline void JSONReader::putData(const QJsonValue& data, S::N& value) { value = (S::N)data.toInt(); } #define IMPLEMENT_ENUM_METATYPE(S, N) \ static int S##N##MetaTypeId = registerEnumMetaType(&S::staticMetaObject, #N); \ diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index eadccdb310..267e45a5a5 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -230,6 +230,19 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << " return array;\n"; out << "}\n"; + out << "template<> void JSONReader::putData(const QJsonValue& data, " << name << "& value) {\n"; + out << " QJsonArray array = data.toArray();\n"; + out << " QJsonArray::const_iterator it = array.constBegin();\n"; + /* foreach (const QString& base, str.clazz.bases) { + out << " foreach (const QJsonValue& element, getData(static_cast(value)).toArray()) {\n"; + out << " array.append(element);\n"; + out << " }\n"; + } */ + foreach (const Field& field, str.fields) { + out << " putData(*it++, value." << field.name << ");\n"; + } + out << "}\n"; + out << "bool operator==(const " << name << "& first, const " << name << "& second) {\n"; if (str.clazz.bases.isEmpty() && str.fields.isEmpty()) { out << " return true"; From ddfe98ad43a1e4a545807f5faeac65a4d1d6fd0b Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 15:32:39 -0700 Subject: [PATCH 18/55] added code to parse nack packets in OctreeEditPacketSender --- .../octree/src/OctreeEditPacketSender.cpp | 49 +++++++++++++++++-- libraries/octree/src/OctreeEditPacketSender.h | 21 ++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index fb0059b96d..69b58aaa41 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -17,6 +17,43 @@ #include #include "OctreeEditPacketSender.h" +void NackedPacketHistory::packetSent(const QByteArray& packet) { + // extract sequence number for the sent packet history + int numBytesPacketHeader = numBytesForPacketHeader(packet); + const char* dataAt = reinterpret_cast(packet.data()); + unsigned short int sequence = (*((unsigned short int*)(dataAt + numBytesPacketHeader))); + + // add packet to history + _sentPacketHistory.packetSent(sequence, packet); +} + +bool NackedPacketHistory::hasNextNackedPacket() const { + return !_nackedSequenceNumbers.isEmpty(); +} + +const QByteArray* NackedPacketHistory::getNextNackedPacket() { + if (!_nackedSequenceNumbers.isEmpty()) { + // could return null if packet is not in the history + return _sentPacketHistory.getPacket(_nackedSequenceNumbers.dequeue()); + } + return NULL; +} + +void NackedPacketHistory::parseNackPacket(const QByteArray& packet) { + int numBytesPacketHeader = numBytesForPacketHeader(packet); + const unsigned char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; + + // read number of sequence numbers + uint16_t numSequenceNumbers = (*(uint16_t*)dataAt); + dataAt += sizeof(uint16_t); + + // read sequence numbers + for (int i = 0; i < numSequenceNumbers; i++) { + unsigned short int sequenceNumber = (*(unsigned short int*)dataAt); + _nackedSequenceNumbers.enqueue(sequenceNumber); + dataAt += sizeof(unsigned short int); + } +} EditPacketBuffer::EditPacketBuffer(PacketType type, unsigned char* buffer, ssize_t length, QUuid nodeUUID) : _nodeUUID(nodeUUID), @@ -97,9 +134,9 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned c if (node->getType() == getMyNodeType() && ((node->getUUID() == nodeUUID) || (nodeUUID.isNull()))) { if (node->getActiveSocket()) { - queuePacketForSending(node, QByteArray(reinterpret_cast(buffer), length)); - - + QByteArray packet(reinterpret_cast(buffer), length); + queuePacketForSending(node, packet); + _nackedPacketHistories[nodeUUID].packetSent(packet); // debugging output... bool wantDebugging = false; @@ -338,3 +375,9 @@ bool OctreeEditPacketSender::process() { // base class does most of the work. return PacketSender::process(); } + +void OctreeEditPacketSender::parseNackPacket(const QByteArray& packet) { + // parse sending node from packet + QUuid sendingNodeUUID = uuidFromPacketHeader(packet); + _nackedPacketHistories[sendingNodeUUID].parseNackPacket(packet); +} diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index 167e43b200..2586ca034f 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -12,9 +12,24 @@ #ifndef hifi_OctreeEditPacketSender_h #define hifi_OctreeEditPacketSender_h +#include #include #include #include "JurisdictionMap.h" +#include "SentPacketHistory.h" + +class NackedPacketHistory { +public: + NackedPacketHistory() : _sentPacketHistory(1000), _nackedSequenceNumbers() { } +public: + void packetSent(const QByteArray& packet); + bool hasNextNackedPacket() const; + const QByteArray* getNextNackedPacket(); + void parseNackPacket(const QByteArray& packet); +private: + SentPacketHistory _sentPacketHistory; + QQueue _nackedSequenceNumbers; +}; /// Used for construction of edit packets class EditPacketBuffer { @@ -90,6 +105,9 @@ public: virtual char getMyNodeType() const = 0; virtual void adjustEditPacketForClockSkew(unsigned char* codeColorBuffer, ssize_t length, int clockSkew) { }; +public: + void parseNackPacket(const QByteArray& packet); + protected: bool _shouldSend; void queuePacketToNode(const QUuid& nodeID, unsigned char* buffer, ssize_t length); @@ -114,5 +132,8 @@ protected: unsigned short int _sequenceNumber; int _maxPacketSize; + + // TODO: garbage-collect this + QHash _nackedPacketHistories; }; #endif // hifi_OctreeEditPacketSender_h From 5e37704772405a9862fe5aec3329e938e9c5c055 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 15:59:38 -0700 Subject: [PATCH 19/55] SentPacketHistory now handles rollover updated SentPacketHistory path in comments --- .../networking/src/SentPacketHistory.cpp | 19 ++++++++++++++----- libraries/networking/src/SentPacketHistory.h | 11 +++++------ libraries/octree/src/OctreeEditPacketSender.h | 2 +- 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/libraries/networking/src/SentPacketHistory.cpp b/libraries/networking/src/SentPacketHistory.cpp index 0ea7fd8b69..37c8953861 100644 --- a/libraries/networking/src/SentPacketHistory.cpp +++ b/libraries/networking/src/SentPacketHistory.cpp @@ -1,6 +1,6 @@ // // SentPacketHistory.cpp -// assignement-client/src/octree +// libraries/networking/src // // Created by Yixin Wang on 6/5/2014 // @@ -18,7 +18,7 @@ SentPacketHistory::SentPacketHistory(int size) { } -void SentPacketHistory::packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const QByteArray& packet) { +void SentPacketHistory::packetSent(uint16_t sequenceNumber, const QByteArray& packet) { _newestSequenceNumber = sequenceNumber; // increment _newestPacketAt cyclically, insert new packet there. @@ -31,9 +31,18 @@ void SentPacketHistory::packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const } -const QByteArray* SentPacketHistory::getPacket(OCTREE_PACKET_SEQUENCE sequenceNumber) const { - OCTREE_PACKET_SEQUENCE seqDiff = _newestSequenceNumber - sequenceNumber; - if (!(seqDiff >= 0 && seqDiff < _numExistingPackets)) { +const QByteArray* SentPacketHistory::getPacket(uint16_t sequenceNumber) const { + + const int UINT16_RANGE = UINT16_MAX + 1; + + // if sequenceNumber > _newestSequenceNumber, assume sequenceNumber is from before the most recent rollover + // correct the diff so that it correctly represents how far back in the history sequenceNumber is + int seqDiff = (int)_newestSequenceNumber - (int)sequenceNumber; + if (seqDiff < 0) { + seqDiff += UINT16_RANGE; + } + // if desired sequence number is too old to be found in the history, return null + if (seqDiff >= _numExistingPackets) { return NULL; } int packetAt = _newestPacketAt - seqDiff; diff --git a/libraries/networking/src/SentPacketHistory.h b/libraries/networking/src/SentPacketHistory.h index 4231400ac1..e3c7736b72 100644 --- a/libraries/networking/src/SentPacketHistory.h +++ b/libraries/networking/src/SentPacketHistory.h @@ -1,6 +1,6 @@ // // SentPacketHistory.h -// assignement-client/src/octree +// libraries/networking/src // // Created by Yixin Wang on 6/5/2014 // @@ -11,25 +11,24 @@ #ifndef hifi_SentPacketHistory_h #define hifi_SentPacketHistory_h +#include #include #include -#include "OctreePacketData.h" - class SentPacketHistory { public: SentPacketHistory(int size); - void packetSent(OCTREE_PACKET_SEQUENCE sequenceNumber, const QByteArray& packet); - const QByteArray* getPacket(OCTREE_PACKET_SEQUENCE sequenceNumber) const; + void packetSent(uint16_t sequenceNumber, const QByteArray& packet); + const QByteArray* getPacket(uint16_t sequenceNumber) const; private: QVector _sentPackets; // circular buffer int _newestPacketAt; int _numExistingPackets; - OCTREE_PACKET_SEQUENCE _newestSequenceNumber; + uint16_t _newestSequenceNumber; }; #endif diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index 2586ca034f..a737131dff 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -133,7 +133,7 @@ protected: unsigned short int _sequenceNumber; int _maxPacketSize; - // TODO: garbage-collect this + // TODO: garbage-collect this and _pendingEditPackets QHash _nackedPacketHistories; }; #endif // hifi_OctreeEditPacketSender_h From 95b25247849aa63149eac4704e98ef3692a12fff Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 16:08:48 -0700 Subject: [PATCH 20/55] removed magic number 65536 from OctreeInboundPacketProcessor --- assignment-client/src/octree/OctreeInboundPacketProcessor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 9b849905c5..1902ba28c3 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -286,7 +286,7 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, printf("\t\t tracked seq %d\n", incomingSequence); - const int UINT16_RANGE = 65536; + const int UINT16_RANGE = UINT16_MAX + 1; const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work const int MAX_MISSING_SEQUENCE_SIZE = 100; @@ -305,7 +305,7 @@ printf("\t\t tracked seq %d\n", incomingSequence); int absGap = std::abs(incoming - expected); if (absGap >= UINT16_RANGE - MAX_REASONABLE_SEQUENCE_GAP) { // rollover likely occurred between incoming and expected. - // correct the larger of the two so that it's within [-65536, -1] while the other remains within [0, 65535] + // correct the larger of the two so that it's within [-UINT16_RANGE, -1] while the other remains within [0, UINT16_RANGE-1] if (incoming > expected) { incoming -= UINT16_RANGE; } else { From 18a9d74b88cb48841a22a8e881e59df85bc0e3b3 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 16:35:30 -0700 Subject: [PATCH 21/55] changed OctreeEditPacketSender to queue packets for resend as nack is parsed --- libraries/networking/src/SentPacketHistory.h | 2 +- .../octree/src/OctreeEditPacketSender.cpp | 79 ++++++++----------- libraries/octree/src/OctreeEditPacketSender.h | 17 +--- 3 files changed, 38 insertions(+), 60 deletions(-) diff --git a/libraries/networking/src/SentPacketHistory.h b/libraries/networking/src/SentPacketHistory.h index e3c7736b72..53a6919c42 100644 --- a/libraries/networking/src/SentPacketHistory.h +++ b/libraries/networking/src/SentPacketHistory.h @@ -18,7 +18,7 @@ class SentPacketHistory { public: - SentPacketHistory(int size); + SentPacketHistory(int size = 1000); void packetSent(uint16_t sequenceNumber, const QByteArray& packet); const QByteArray* getPacket(uint16_t sequenceNumber) const; diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 69b58aaa41..439148a9a5 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -17,44 +17,6 @@ #include #include "OctreeEditPacketSender.h" -void NackedPacketHistory::packetSent(const QByteArray& packet) { - // extract sequence number for the sent packet history - int numBytesPacketHeader = numBytesForPacketHeader(packet); - const char* dataAt = reinterpret_cast(packet.data()); - unsigned short int sequence = (*((unsigned short int*)(dataAt + numBytesPacketHeader))); - - // add packet to history - _sentPacketHistory.packetSent(sequence, packet); -} - -bool NackedPacketHistory::hasNextNackedPacket() const { - return !_nackedSequenceNumbers.isEmpty(); -} - -const QByteArray* NackedPacketHistory::getNextNackedPacket() { - if (!_nackedSequenceNumbers.isEmpty()) { - // could return null if packet is not in the history - return _sentPacketHistory.getPacket(_nackedSequenceNumbers.dequeue()); - } - return NULL; -} - -void NackedPacketHistory::parseNackPacket(const QByteArray& packet) { - int numBytesPacketHeader = numBytesForPacketHeader(packet); - const unsigned char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; - - // read number of sequence numbers - uint16_t numSequenceNumbers = (*(uint16_t*)dataAt); - dataAt += sizeof(uint16_t); - - // read sequence numbers - for (int i = 0; i < numSequenceNumbers; i++) { - unsigned short int sequenceNumber = (*(unsigned short int*)dataAt); - _nackedSequenceNumbers.enqueue(sequenceNumber); - dataAt += sizeof(unsigned short int); - } -} - EditPacketBuffer::EditPacketBuffer(PacketType type, unsigned char* buffer, ssize_t length, QUuid nodeUUID) : _nodeUUID(nodeUUID), _currentType(type), @@ -126,7 +88,7 @@ bool OctreeEditPacketSender::serversExist() const { // This method is called when the edit packet layer has determined that it has a fully formed packet destined for // a known nodeID. -void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned char* buffer, ssize_t length) { +void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsigned char* buffer, ssize_t length) { NodeList* nodeList = NodeList::getInstance(); foreach (const SharedNodePointer& node, nodeList->getNodeHash()) { @@ -134,14 +96,19 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, unsigned c if (node->getType() == getMyNodeType() && ((node->getUUID() == nodeUUID) || (nodeUUID.isNull()))) { if (node->getActiveSocket()) { - QByteArray packet(reinterpret_cast(buffer), length); + QByteArray packet(reinterpret_cast(buffer), length); queuePacketForSending(node, packet); - _nackedPacketHistories[nodeUUID].packetSent(packet); + + // extract sequence number and add packet to history + int numBytesPacketHeader = numBytesForPacketHeader(packet); + const char* dataAt = reinterpret_cast(packet.data()); + unsigned short int sequence = (*((unsigned short int*)(dataAt + numBytesPacketHeader))); + _sentPacketHistories[nodeUUID].packetSent(sequence, packet); // debugging output... bool wantDebugging = false; if (wantDebugging) { - int numBytesPacketHeader = numBytesForPacketHeader(reinterpret_cast(buffer)); + int numBytesPacketHeader = numBytesForPacketHeader(reinterpret_cast(buffer)); unsigned short int sequence = (*((unsigned short int*)(buffer + numBytesPacketHeader))); quint64 createdAt = (*((quint64*)(buffer + numBytesPacketHeader + sizeof(sequence)))); quint64 queuedAt = usecTimestampNow(); @@ -377,7 +344,31 @@ bool OctreeEditPacketSender::process() { } void OctreeEditPacketSender::parseNackPacket(const QByteArray& packet) { - // parse sending node from packet + // parse sending node from packet, retrieve packet history for that node QUuid sendingNodeUUID = uuidFromPacketHeader(packet); - _nackedPacketHistories[sendingNodeUUID].parseNackPacket(packet); + + // if packet history doesn't exist for the sender node (somehow), bail + if (!_sentPacketHistories.contains(sendingNodeUUID)) { + return; + } + const SentPacketHistory& sentPacketHistory = _sentPacketHistories.value(sendingNodeUUID); + + int numBytesPacketHeader = numBytesForPacketHeader(packet); + const unsigned char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; + + // read number of sequence numbers + uint16_t numSequenceNumbers = (*(uint16_t*)dataAt); + dataAt += sizeof(uint16_t); + + // read sequence numbers and queue packets for resend + for (int i = 0; i < numSequenceNumbers; i++) { + unsigned short int sequenceNumber = (*(unsigned short int*)dataAt); + dataAt += sizeof(unsigned short int); + + // retrieve packet from history + const QByteArray* packet = sentPacketHistory.getPacket(sequenceNumber); + if (packet) { + queuePacketToNode(sendingNodeUUID, (const unsigned char*)packet->constData(), packet->length()); + } + } } diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index a737131dff..45913b8db2 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -18,19 +18,6 @@ #include "JurisdictionMap.h" #include "SentPacketHistory.h" -class NackedPacketHistory { -public: - NackedPacketHistory() : _sentPacketHistory(1000), _nackedSequenceNumbers() { } -public: - void packetSent(const QByteArray& packet); - bool hasNextNackedPacket() const; - const QByteArray* getNextNackedPacket(); - void parseNackPacket(const QByteArray& packet); -private: - SentPacketHistory _sentPacketHistory; - QQueue _nackedSequenceNumbers; -}; - /// Used for construction of edit packets class EditPacketBuffer { public: @@ -110,7 +97,7 @@ public: protected: bool _shouldSend; - void queuePacketToNode(const QUuid& nodeID, unsigned char* buffer, ssize_t length); + void queuePacketToNode(const QUuid& nodeID, const unsigned char* buffer, ssize_t length); void queuePendingPacketToNodes(PacketType type, unsigned char* buffer, ssize_t length); void queuePacketToNodes(unsigned char* buffer, ssize_t length); void initializePacket(EditPacketBuffer& packetBuffer, PacketType type); @@ -134,6 +121,6 @@ protected: int _maxPacketSize; // TODO: garbage-collect this and _pendingEditPackets - QHash _nackedPacketHistories; + QHash _sentPacketHistories; }; #endif // hifi_OctreeEditPacketSender_h From 4d84e1fff1c2269797fcd4b761cc016323b3bb6b Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 16:41:00 -0700 Subject: [PATCH 22/55] added processNackPacket calls to DatagramProcessor --- interface/src/DatagramProcessor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index cdd7e7ef0f..e032056fc8 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -145,6 +145,12 @@ void DatagramProcessor::processDatagrams() { } break; } + case PacketTypeVoxelEditNack: + application->_voxelEditSender.processNackPacket(incomingPacket); + case PacketTypeParticleEditNack: + application->_particleEditSender.processNackPacket(incomingPacket); + case PacketTypeModelEditNack: + application->_modelEditSender.processNackPacket(incomingPacket); default: nodeList->processNodeData(senderSockAddr, incomingPacket); break; From e3db60d1efa4f2dd750668aad6fcfca04158ca37 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 16:42:04 -0700 Subject: [PATCH 23/55] forgot to add "break;"s --- interface/src/DatagramProcessor.cpp | 3 +++ libraries/octree/src/OctreeEditPacketSender.cpp | 2 +- libraries/octree/src/OctreeEditPacketSender.h | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/interface/src/DatagramProcessor.cpp b/interface/src/DatagramProcessor.cpp index e032056fc8..29528da126 100644 --- a/interface/src/DatagramProcessor.cpp +++ b/interface/src/DatagramProcessor.cpp @@ -147,10 +147,13 @@ void DatagramProcessor::processDatagrams() { } case PacketTypeVoxelEditNack: application->_voxelEditSender.processNackPacket(incomingPacket); + break; case PacketTypeParticleEditNack: application->_particleEditSender.processNackPacket(incomingPacket); + break; case PacketTypeModelEditNack: application->_modelEditSender.processNackPacket(incomingPacket); + break; default: nodeList->processNodeData(senderSockAddr, incomingPacket); break; diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 439148a9a5..16bb22987d 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -343,7 +343,7 @@ bool OctreeEditPacketSender::process() { return PacketSender::process(); } -void OctreeEditPacketSender::parseNackPacket(const QByteArray& packet) { +void OctreeEditPacketSender::processNackPacket(const QByteArray& packet) { // parse sending node from packet, retrieve packet history for that node QUuid sendingNodeUUID = uuidFromPacketHeader(packet); diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index 45913b8db2..419e850ff8 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -93,7 +93,7 @@ public: virtual void adjustEditPacketForClockSkew(unsigned char* codeColorBuffer, ssize_t length, int clockSkew) { }; public: - void parseNackPacket(const QByteArray& packet); + void processNackPacket(const QByteArray& packet); protected: bool _shouldSend; From 06f8464ec9ac54683cfbe0392ab790526a58ac04 Mon Sep 17 00:00:00 2001 From: wangyix Date: Mon, 16 Jun 2014 17:17:48 -0700 Subject: [PATCH 24/55] edit nacks ready for test; seq numbers sometimes repeat?? --- .../src/octree/OctreeInboundPacketProcessor.cpp | 6 +++--- libraries/networking/src/SentPacketHistory.cpp | 6 ++++++ libraries/octree/src/OctreeEditPacketSender.cpp | 17 +++++++++-------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 1902ba28c3..b795139ed5 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -194,8 +194,6 @@ void OctreeInboundPacketProcessor::trackInboundPacket(const QUuid& nodeUUID, uns int OctreeInboundPacketProcessor::sendNackPackets() { - printf("\t\t sendNackPackets()\n"); - int packetsSent = 0; NodeToSenderStatsMapIterator i = _singleSenderStats.begin(); @@ -336,7 +334,9 @@ printf("\t\t tracked seq %d\n", incomingSequence); printf("\t\t\t packet is late!\n"); // remove this from missing sequence number if it's in there - _missingSequenceNumbers.remove(incomingSequence); + if (_missingSequenceNumbers.remove(incomingSequence)) { + printf("\t\t\t\t packet %d recovered!!!\n", incomingSequence); + } // do not update _incomingLastSequence } diff --git a/libraries/networking/src/SentPacketHistory.cpp b/libraries/networking/src/SentPacketHistory.cpp index 37c8953861..eb55d2e1b4 100644 --- a/libraries/networking/src/SentPacketHistory.cpp +++ b/libraries/networking/src/SentPacketHistory.cpp @@ -19,6 +19,12 @@ SentPacketHistory::SentPacketHistory(int size) } void SentPacketHistory::packetSent(uint16_t sequenceNumber, const QByteArray& packet) { + + if (sequenceNumber != 0 && sequenceNumber != _newestSequenceNumber + 1) { + printf("\t packet history received unexpected seq number! prev: %d received: %d\n", _newestSequenceNumber, sequenceNumber); + } + + _newestSequenceNumber = sequenceNumber; // increment _newestPacketAt cyclically, insert new packet there. diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 16bb22987d..2e0999c4f8 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -97,6 +97,9 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsi ((node->getUUID() == nodeUUID) || (nodeUUID.isNull()))) { if (node->getActiveSocket()) { QByteArray packet(reinterpret_cast(buffer), length); + + bool send = randFloat() < 0.7f; + if (send) queuePacketForSending(node, packet); // extract sequence number and add packet to history @@ -105,6 +108,10 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsi unsigned short int sequence = (*((unsigned short int*)(dataAt + numBytesPacketHeader))); _sentPacketHistories[nodeUUID].packetSent(sequence, packet); + if (!send) { + printf("\t\t dropped packet %d !!!\n", sequence); + } + // debugging output... bool wantDebugging = false; if (wantDebugging) { @@ -314,14 +321,7 @@ void OctreeEditPacketSender::initializePacket(EditPacketBuffer& packetBuffer, Pa unsigned short int* sequenceAt = (unsigned short int*)&packetBuffer._currentBuffer[packetBuffer._currentSize]; *sequenceAt = _sequenceNumber; packetBuffer._currentSize += sizeof(unsigned short int); // nudge past sequence - if (randFloat() < 0.6f) _sequenceNumber++; - else - { - int x = randIntInRange(2, 4); - printf("\t\t seq number jumped from %d to %d\n", _sequenceNumber, _sequenceNumber + x); - _sequenceNumber += x; - } // pack in timestamp quint64 now = usecTimestampNow(); @@ -368,7 +368,8 @@ void OctreeEditPacketSender::processNackPacket(const QByteArray& packet) { // retrieve packet from history const QByteArray* packet = sentPacketHistory.getPacket(sequenceNumber); if (packet) { - queuePacketToNode(sendingNodeUUID, (const unsigned char*)packet->constData(), packet->length()); + const SharedNodePointer& node = NodeList::getInstance()->getNodeHash().value(sendingNodeUUID); + queuePacketForSending(node, *packet); } } } From 37c977af024966ea2cde01d843f8ec6f60c54308 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Mon, 16 Jun 2014 19:12:21 -0700 Subject: [PATCH 25/55] Tests working for reading JSON. --- libraries/metavoxels/src/Bitstream.cpp | 208 +++++++++++++++++++++--- libraries/metavoxels/src/Bitstream.h | 6 +- tests/metavoxels/src/MetavoxelTests.cpp | 60 ++++++- tools/mtc/src/main.cpp | 27 ++- 4 files changed, 260 insertions(+), 41 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 8f880a1803..86f508ca11 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1178,7 +1178,9 @@ Bitstream& Bitstream::operator>(ObjectStreamerPointer& streamer) { } for (int i = 0; i < localProperties.size(); i++) { const StreamerPropertyPair& property = properties.at(i); - if (localProperties.at(i).first != property.first || property.second.propertyIndex() != i) { + const StreamerPropertyPair& localProperty = localProperties.at(i); + if (property.first != localProperty.first || + property.second.propertyIndex() != localProperty.second.propertyIndex()) { streamer = ObjectStreamerPointer(new MappedObjectStreamer(metaObject, properties)); return *this; } @@ -1866,7 +1868,6 @@ void JSONWriter::addSharedObject(const SharedObjectPointer& object) { QJsonObject sharedObject; sharedObject.insert("id", object->getID()); - sharedObject.insert("originID", object->getOriginID()); sharedObject.insert("data", getData(static_cast(object.data()))); _sharedObjects.replace(index, sharedObject); } @@ -1887,17 +1888,121 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge for (int i = types.size() - 1; i >= 0; i--) { QJsonObject type = types.at(i).toObject(); QString name = type.value("name").toString(); + QByteArray latinName = name.toLatin1(); + const TypeStreamer* baseStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(latinName)); + if (!baseStreamer) { + baseStreamer = Bitstream::getEnumStreamersByName().value(latinName); + } + if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { + continue; + } QString category = type.value("category").toString(); + if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + TypeStreamerPointer streamer; + if (category == "ENUM") { + QVector values; + int highestValue = 0; + foreach (const QJsonValue& value, type.value("values").toArray()) { + QJsonObject object = value.toObject(); + int value = object.value("value").toInt(); + highestValue = qMax(value, highestValue); + values.append(NameIntPair(object.value("key").toString().toLatin1(), value)); + } + streamer = TypeStreamerPointer(new GenericEnumTypeStreamer(latinName, + values, getBitsForHighestValue(highestValue), QByteArray())); + + } else if (category == "STREAMABLE") { + QVector fields; + foreach (const QJsonValue& field, type.value("fields").toArray()) { + QJsonObject object = field.toObject(); + fields.append(StreamerNamePair(getTypeStreamer(object.value("type").toString()), + object.value("name").toString().toLatin1())); + } + streamer = TypeStreamerPointer(new GenericStreamableTypeStreamer(latinName, + fields, QByteArray())); + + } else if (category == "LIST") { + streamer = TypeStreamerPointer(new GenericListTypeStreamer(latinName, + getTypeStreamer(type.value("valueType").toString()))); + + } else if (category == "SET") { + streamer = TypeStreamerPointer(new GenericSetTypeStreamer(latinName, + getTypeStreamer(type.value("valueType").toString()))); + + } else if (category == "MAP") { + streamer = TypeStreamerPointer(new GenericMapTypeStreamer(latinName, + getTypeStreamer(type.value("keyType").toString()), + getTypeStreamer(type.value("valueType").toString()))); + } + _typeStreamers.insert(name, streamer); + static_cast(streamer.data())->_weakSelf = streamer; + continue; + } if (category == "ENUM") { - + QHash mappings; + int highestValue = 0; + QMetaEnum metaEnum = baseStreamer->getMetaEnum(); + QJsonArray array = type.value("values").toArray(); + bool matches = (array.size() == metaEnum.keyCount()); + foreach (const QJsonValue& value, array) { + QJsonObject object = value.toObject(); + int value = object.value("value").toInt(); + highestValue = qMax(value, highestValue); + int mapping = metaEnum.keyToValue(object.value("key").toString().toLatin1()); + if (mapping != -1) { + mappings.insert(value, mapping); + } + matches &= (value == mapping); + } + if (matches) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedEnumTypeStreamer(baseStreamer, + getBitsForHighestValue(highestValue), mappings))); + } } else if (category == "STREAMABLE") { - + QVector fields; + QJsonArray array = type.value("fields").toArray(); + const QVector& metaFields = baseStreamer->getMetaFields(); + bool matches = (array.size() == metaFields.size()); + for (int j = 0; j < array.size(); j++) { + QJsonObject object = array.at(j).toObject(); + TypeStreamerPointer streamer = getTypeStreamer(object.value("type").toString()); + int index = baseStreamer->getFieldIndex(object.value("name").toString().toLatin1()); + fields.append(StreamerIndexPair(streamer, index)); + matches &= (index == j && streamer == metaFields.at(j).getStreamer()); + } + if (matches) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedStreamableTypeStreamer(baseStreamer, fields))); + } } else if (category == "LIST") { - + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedListTypeStreamer(baseStreamer, valueStreamer))); + } } else if (category == "SET") { - + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedSetTypeStreamer(baseStreamer, valueStreamer))); + } } else if (category == "MAP") { - + TypeStreamerPointer keyStreamer = getTypeStreamer(type.value("keyType").toString()); + TypeStreamerPointer valueStreamer = getTypeStreamer(type.value("valueType").toString()); + if (keyStreamer == baseStreamer->getKeyStreamer() && valueStreamer == baseStreamer->getValueStreamer()) { + _typeStreamers.insert(name, baseStreamer->getSelf()); + + } else { + _typeStreamers.insert(name, TypeStreamerPointer(new MappedMapTypeStreamer( + baseStreamer, keyStreamer, valueStreamer))); + } } } @@ -1905,11 +2010,45 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge for (int i = classes.size() - 1; i >= 0; i--) { QJsonObject clazz = classes.at(i).toObject(); QString name = clazz.value("name").toString(); - QJsonArray properties = clazz.value("properties").toArray(); - foreach (const QJsonValue& property, properties) { - QJsonObject object = property.toObject(); - object.value("type"); - object.value("name"); + QByteArray latinName = name.toLatin1(); + const ObjectStreamer* baseStreamer = Bitstream::getObjectStreamers().value( + Bitstream::getMetaObjects().value(latinName)); + if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { + continue; + } + if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + QVector properties; + foreach (const QJsonValue& property, clazz.value("properties").toArray()) { + QJsonObject object = property.toObject(); + properties.append(StreamerNamePair(getTypeStreamer(object.value("type").toString()), + object.value("name").toString().toLatin1())); + } + ObjectStreamerPointer streamer = ObjectStreamerPointer(new GenericObjectStreamer( + latinName, properties, QByteArray())); + _objectStreamers.insert(name, streamer); + static_cast(streamer.data())->_weakSelf = streamer; + continue; + } + const QMetaObject* metaObject = baseStreamer->getMetaObject(); + const QVector& baseProperties = baseStreamer->getProperties(); + QVector properties; + QJsonArray propertyArray = clazz.value("properties").toArray(); + bool matches = (baseProperties.size() == propertyArray.size()); + for (int j = 0; j < propertyArray.size(); j++) { + QJsonObject object = propertyArray.at(j).toObject(); + TypeStreamerPointer typeStreamer = getTypeStreamer(object.value("type").toString()); + QMetaProperty metaProperty = metaObject->property(metaObject->indexOfProperty( + object.value("name").toString().toLatin1())); + properties.append(StreamerPropertyPair(typeStreamer, metaProperty)); + + const StreamerPropertyPair& baseProperty = baseProperties.at(i); + matches &= (typeStreamer == baseProperty.first && + metaProperty.propertyIndex() == baseProperty.second.propertyIndex()); + } + if (matches) { + _objectStreamers.insert(name, baseStreamer->getSelf()); + } else { + _objectStreamers.insert(name, ObjectStreamerPointer(new MappedObjectStreamer(metaObject, properties))); } } @@ -1917,9 +2056,11 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge for (int i = objects.size() - 1; i >= 0; i--) { QJsonObject object = objects.at(i).toObject(); int id = object.value("id").toInt(); - int originID = object.value("originID").toInt(); - QJsonObject data = object.value("data").toObject(); - + QObject* qObject; + putData(object.value("data"), qObject); + if (qObject) { + _sharedObjects.insert(id, static_cast(qObject)); + } } _contents = top.value("contents").toArray(); @@ -2051,16 +2192,12 @@ void JSONReader::putData(const QJsonValue& data, const QMetaObject*& value) { void JSONReader::putData(const QJsonValue& data, QVariant& value) { QJsonObject object = data.toObject(); QString type = object.value("type").toString(); - const TypeStreamer* streamer = _typeStreamers.value(type).data(); - if (!streamer) { - streamer = Bitstream::getTypeStreamers().value(QMetaType::type(type.toLatin1())); - if (!streamer) { - qWarning() << "Unknown type:" << type; - value = QVariant(); - return; - } + TypeStreamerPointer streamer = getTypeStreamer(type); + if (streamer) { + streamer->putJSONVariantData(*this, object.value("value"), value); + } else { + value = QVariant(); } - streamer->putJSONData(*this, object.value("value"), value); } void JSONReader::putData(const QJsonValue& data, SharedObjectPointer& value) { @@ -2073,6 +2210,19 @@ void JSONReader::putData(const QJsonValue& data, QObject*& value) { value = streamer ? streamer->putJSONData(*this, object) : NULL; } +TypeStreamerPointer JSONReader::getTypeStreamer(const QString& name) const { + TypeStreamerPointer streamer = _typeStreamers.value(name); + if (!streamer) { + const TypeStreamer* defaultStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(name.toLatin1())); + if (defaultStreamer) { + streamer = defaultStreamer->getSelf(); + } else { + qWarning() << "Unknown type:" << name; + } + } + return streamer; +} + ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } @@ -2438,6 +2588,10 @@ void TypeStreamer::putJSONData(JSONReader& reader, const QJsonValue& data, QVari value = QVariant(); } +void TypeStreamer::putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + putJSONData(reader, data, value); +} + bool TypeStreamer::equal(const QVariant& first, const QVariant& second) const { return first == second; } @@ -2750,6 +2904,12 @@ const char* GenericTypeStreamer::getName() const { return _name.constData(); } +void GenericTypeStreamer::putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const { + QVariant containedValue; + putJSONData(reader, data, containedValue); + value = QVariant::fromValue(GenericValue(_weakSelf, containedValue)); +} + QVariant GenericTypeStreamer::readVariant(Bitstream& in) const { return QVariant::fromValue(GenericValue(_weakSelf, read(in))); } diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 0cad33acea..776a362717 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -893,7 +893,7 @@ public: template JSONReader& operator>>(T& value) { putData(*_contentsIterator++, value); return *this; } - TypeStreamerPointer getTypeStreamer(const QString& name) const { return _typeStreamers.value(name); } + TypeStreamerPointer getTypeStreamer(const QString& name) const; ObjectStreamerPointer getObjectStreamer(const QString& name) const { return _objectStreamers.value(name); } SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } @@ -1020,6 +1020,7 @@ public: private: friend class Bitstream; + friend class JSONReader; QByteArray _name; WeakObjectStreamerPointer _weakSelf; @@ -1107,6 +1108,7 @@ public: virtual QJsonValue getJSONData(JSONWriter& writer, const QVariant& value) const; virtual QJsonValue getJSONVariantData(JSONWriter& writer, const QVariant& value) const; virtual void putJSONData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; + virtual void putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual bool equal(const QVariant& first, const QVariant& second) const; @@ -1237,11 +1239,13 @@ public: GenericTypeStreamer(const QByteArray& name); virtual const char* getName() const; + virtual void putJSONVariantData(JSONReader& reader, const QJsonValue& data, QVariant& value) const; virtual QVariant readVariant(Bitstream& in) const; protected: friend class Bitstream; + friend class JSONReader; QByteArray _name; WeakTypeStreamerPointer _weakSelf; diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 618178383f..4a3010caf4 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -252,8 +252,64 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { jsonWriter << testObjectReadA; jsonWriter << testObjectReadB; jsonWriter << messageRead; - qDebug() << jsonWriter.getDocument().toJson(); - qDebug(); + jsonWriter << endRead; + QByteArray encodedJson = jsonWriter.getDocument().toJson(); + + // and read from JSON + JSONReader jsonReader(QJsonDocument::fromJson(encodedJson), Bitstream::ALL_GENERICS); + jsonReader >> testObjectReadA; + jsonReader >> testObjectReadB; + jsonReader >> messageRead; + jsonReader >> endRead; + + // reassign the ids + testObjectReadA->setID(testObjectWrittenA->getID()); + testObjectReadA->setOriginID(testObjectWrittenA->getOriginID()); + testObjectReadB->setID(testObjectWrittenB->getID()); + testObjectReadB->setOriginID(testObjectWrittenB->getOriginID()); + + // and back to binary + QByteArray secondCompareArray; + QDataStream secondCompareOutStream(&secondCompareArray, QIODevice::WriteOnly); + Bitstream secondCompareOut(secondCompareOutStream, Bitstream::FULL_METADATA); + secondCompareOut << testObjectReadA; + secondCompareOut << testObjectReadB; + secondCompareOut << messageRead; + secondCompareOut << endRead; + secondCompareOut.flush(); + + if (compareArray != secondCompareArray) { + qDebug() << "Mismatch between written/JSON streams (generics)."; + return true; + } + + // once more, with mapping! + JSONReader secondJSONReader(QJsonDocument::fromJson(encodedJson)); + secondJSONReader >> testObjectReadA; + secondJSONReader >> testObjectReadB; + secondJSONReader >> messageRead; + secondJSONReader >> endRead; + + // reassign the ids + testObjectReadA->setID(testObjectWrittenA->getID()); + testObjectReadA->setOriginID(testObjectWrittenA->getOriginID()); + testObjectReadB->setID(testObjectWrittenB->getID()); + testObjectReadB->setOriginID(testObjectWrittenB->getOriginID()); + + // and back to binary + QByteArray thirdCompareArray; + QDataStream thirdCompareOutStream(&thirdCompareArray, QIODevice::WriteOnly); + Bitstream thirdCompareOut(thirdCompareOutStream, Bitstream::FULL_METADATA); + thirdCompareOut << testObjectReadA; + thirdCompareOut << testObjectReadB; + thirdCompareOut << messageRead; + thirdCompareOut << endRead; + thirdCompareOut.flush(); + + if (compareArray != thirdCompareArray) { + qDebug() << "Mismatch between written/JSON streams (mapped)."; + return true; + } return false; } diff --git a/tools/mtc/src/main.cpp b/tools/mtc/src/main.cpp index 267e45a5a5..dd0ab837a5 100644 --- a/tools/mtc/src/main.cpp +++ b/tools/mtc/src/main.cpp @@ -123,11 +123,6 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << "QHash " << name << "::createFieldIndices() {\n"; out << " QHash indices;\n"; out << " int index = 1;\n"; - foreach (const QString& base, str.clazz.bases) { - out << " foreach (const MetaField& field, " << base << "::getMetaFields()) {\n"; - out << " indices.insert(field.getName(), index++);\n"; - out << " }\n"; - } out << " foreach (const MetaField& field, getMetaFields()) {\n"; out << " indices.insert(field.getName(), index++);\n"; out << " }\n"; @@ -231,15 +226,19 @@ void generateOutput (QTextStream& out, const QList& streamables) { out << "}\n"; out << "template<> void JSONReader::putData(const QJsonValue& data, " << name << "& value) {\n"; - out << " QJsonArray array = data.toArray();\n"; - out << " QJsonArray::const_iterator it = array.constBegin();\n"; - /* foreach (const QString& base, str.clazz.bases) { - out << " foreach (const QJsonValue& element, getData(static_cast(value)).toArray()) {\n"; - out << " array.append(element);\n"; - out << " }\n"; - } */ - foreach (const Field& field, str.fields) { - out << " putData(*it++, value." << field.name << ");\n"; + if (!(str.clazz.bases.isEmpty() && str.fields.isEmpty())) { + out << " QJsonArray array = data.toArray(), subarray;\n"; + out << " QJsonArray::const_iterator it = array.constBegin();\n"; + foreach (const QString& base, str.clazz.bases) { + out << " subarray = QJsonArray();\n"; + out << " for (int i = 0; i < " << base << "::getMetaFields().size(); i++) {\n"; + out << " subarray.append(*it++);\n"; + out << " }\n"; + out << " putData(subarray, static_cast<" << base << "&>(value));\n"; + } + foreach (const Field& field, str.fields) { + out << " putData(*it++, value." << field.name << ");\n"; + } } out << "}\n"; From dc71f87ea4f5ac2dc2270a4e48478b4bf4217929 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 10:42:56 -0700 Subject: [PATCH 26/55] edit nacks seem to be working; added mutex for releaseQueuedPacket() to prevent duplicate packets being queued up due to the steps of queueing the packet and clearing it not being atomic. --- .../octree/OctreeInboundPacketProcessor.cpp | 2 +- .../networking/src/SentPacketHistory.cpp | 2 +- .../octree/src/OctreeEditPacketSender.cpp | 30 +++++++++++++++---- libraries/octree/src/OctreeEditPacketSender.h | 2 ++ 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index b795139ed5..b0c4d905bc 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -282,7 +282,7 @@ SingleSenderStats::SingleSenderStats() void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { -printf("\t\t tracked seq %d\n", incomingSequence); +printf("\t\t tracked seq %hu\n", incomingSequence); const int UINT16_RANGE = UINT16_MAX + 1; diff --git a/libraries/networking/src/SentPacketHistory.cpp b/libraries/networking/src/SentPacketHistory.cpp index eb55d2e1b4..a34dada397 100644 --- a/libraries/networking/src/SentPacketHistory.cpp +++ b/libraries/networking/src/SentPacketHistory.cpp @@ -21,7 +21,7 @@ SentPacketHistory::SentPacketHistory(int size) void SentPacketHistory::packetSent(uint16_t sequenceNumber, const QByteArray& packet) { if (sequenceNumber != 0 && sequenceNumber != _newestSequenceNumber + 1) { - printf("\t packet history received unexpected seq number! prev: %d received: %d\n", _newestSequenceNumber, sequenceNumber); + printf("\t\tpacket history received unexpected seq number! prev: %hu received: %hu **************** \n", _newestSequenceNumber, sequenceNumber); } diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 2e0999c4f8..251886f70c 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -104,12 +104,28 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsi // extract sequence number and add packet to history int numBytesPacketHeader = numBytesForPacketHeader(packet); - const char* dataAt = reinterpret_cast(packet.data()); - unsigned short int sequence = (*((unsigned short int*)(dataAt + numBytesPacketHeader))); + const char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; + unsigned short int sequence = *((unsigned short int*)dataAt); +/* +// debug +dataAt += sizeof(unsigned short int); + +// extract time stamp +quint64 sentTime = *((quint64*)dataAt); +dataAt += sizeof(quint64); + +PacketType type = packetTypeForPacket(packet); + + +printf("adding packet to history. size: %d\n", packet.length()); +printf("type: %d, seq: %hu, time: %llu\n", (unsigned char)type, sequence, sentTime); +printf("destination node: %s\n", nodeUUID.toString().toLatin1().data()); +fflush(stdout); +*/ _sentPacketHistories[nodeUUID].packetSent(sequence, packet); if (!send) { - printf("\t\t dropped packet %d !!!\n", sequence); + printf("\t dropped packet %d !!! ---------------------------\n", sequence); } // debugging output... @@ -307,11 +323,15 @@ void OctreeEditPacketSender::releaseQueuedMessages() { } void OctreeEditPacketSender::releaseQueuedPacket(EditPacketBuffer& packetBuffer) { + _releaseQueuedPacketMutex.lock(); + if (packetBuffer._currentSize > 0 && packetBuffer._currentType != PacketTypeUnknown) { queuePacketToNode(packetBuffer._nodeUUID, &packetBuffer._currentBuffer[0], packetBuffer._currentSize); + packetBuffer._currentSize = 0; + packetBuffer._currentType = PacketTypeUnknown; } - packetBuffer._currentSize = 0; - packetBuffer._currentType = PacketTypeUnknown; + + _releaseQueuedPacketMutex.unlock(); } void OctreeEditPacketSender::initializePacket(EditPacketBuffer& packetBuffer, PacketType type) { diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index 419e850ff8..e9c616085f 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -122,5 +122,7 @@ protected: // TODO: garbage-collect this and _pendingEditPackets QHash _sentPacketHistories; + + QMutex _releaseQueuedPacketMutex; }; #endif // hifi_OctreeEditPacketSender_h From 6e71523346108b6cc2b5d59343413e409f6e6b44 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 10:58:44 -0700 Subject: [PATCH 27/55] added OctreeEditPacketSender::nodeKilled(), no locks yet also added nodeKilled() calls to the 3 editsenders in Application::nodeKilled() --- interface/src/Application.cpp | 10 ++++++++-- libraries/octree/src/OctreeEditPacketSender.cpp | 7 +++++++ libraries/octree/src/OctreeEditPacketSender.h | 11 +++++++---- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 7a7d7c10f8..02cf59ad87 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -3249,10 +3249,16 @@ void Application::nodeAdded(SharedNodePointer node) { void Application::nodeKilled(SharedNodePointer node) { - // this is here because connecting NodeList::nodeKilled to OctreePacketProcessor::nodeKilled doesn't work: - // OctreePacketProcessor::nodeKilled is not called when NodeList::nodeKilled is emitted for some reason. + // These are here because connecting NodeList::nodeKilled to OctreePacketProcessor::nodeKilled doesn't work: + // OctreePacketProcessor::nodeKilled is not being called when NodeList::nodeKilled is emitted. + // This may have to do with GenericThread::threadRoutine() blocking the QThread event loop + _octreeProcessor.nodeKilled(node); + _voxelEditSender.nodeKilled(node); + _particleEditSender.nodeKilled(node); + _modelEditSender.nodeKilled(node); + if (node->getType() == NodeType::VoxelServer) { QUuid nodeUUID = node->getUUID(); // see if this is the first we've heard of this node... diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 251886f70c..f82e7e3101 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -393,3 +393,10 @@ void OctreeEditPacketSender::processNackPacket(const QByteArray& packet) { } } } + +void OctreeEditPacketSender::nodeKilled(SharedNodePointer node) { + // TODO: add locks + QUuid nodeUUID = node->getUUID(); + _pendingEditPackets.remove(nodeUUID); + _sentPacketHistories.remove(nodeUUID); +} diff --git a/libraries/octree/src/OctreeEditPacketSender.h b/libraries/octree/src/OctreeEditPacketSender.h index e9c616085f..c16c0a2d4b 100644 --- a/libraries/octree/src/OctreeEditPacketSender.h +++ b/libraries/octree/src/OctreeEditPacketSender.h @@ -91,7 +91,10 @@ public: // you must override these... virtual char getMyNodeType() const = 0; virtual void adjustEditPacketForClockSkew(unsigned char* codeColorBuffer, ssize_t length, int clockSkew) { }; - + +public slots: + void nodeKilled(SharedNodePointer node); + public: void processNackPacket(const QByteArray& packet); @@ -120,9 +123,9 @@ protected: unsigned short int _sequenceNumber; int _maxPacketSize; - // TODO: garbage-collect this and _pendingEditPackets - QHash _sentPacketHistories; - QMutex _releaseQueuedPacketMutex; + + // TODO: add locks for this and _pendingEditPackets + QHash _sentPacketHistories; }; #endif // hifi_OctreeEditPacketSender_h From 81879123844e1aade1362c4fb26006251800f3b6 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 11:34:35 -0700 Subject: [PATCH 28/55] added qDebug() check in SentPacketHistory for seq numbers used (unsigned short int)1 instead of 1 when calculating expectedSequence --- .../src/octree/OctreeInboundPacketProcessor.cpp | 6 +++--- libraries/networking/src/SentPacketHistory.cpp | 13 ++++++++----- libraries/octree/src/OctreeEditPacketSender.cpp | 2 +- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index b0c4d905bc..44c9576943 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -289,7 +289,7 @@ printf("\t\t tracked seq %hu\n", incomingSequence); const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work const int MAX_MISSING_SEQUENCE_SIZE = 100; - unsigned short int expectedSequence = _totalPackets == 0 ? incomingSequence : _incomingLastSequence + 1; + unsigned short int expectedSequence = _totalPackets == 0 ? incomingSequence : _incomingLastSequence + (unsigned short int)1; if (incomingSequence == expectedSequence) { // on time _incomingLastSequence = incomingSequence; @@ -346,8 +346,8 @@ printf("\t\t tracked seq %hu\n", incomingSequence); // will be removed. if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE_SIZE) { - // the acceptable range of older sequence numbers may contain a rollover point; this must be handled. - // some sequence number in this list may be larger than _incomingLastSequence, indicating that they were received + // some older sequence numbers may be from before a rollover point; this must be handled. + // some sequence numbers in this list may be larger than _incomingLastSequence, indicating that they were received // before the most recent rollover. int cutoff = (int)_incomingLastSequence - MAX_REASONABLE_SEQUENCE_GAP; if (cutoff >= 0) { diff --git a/libraries/networking/src/SentPacketHistory.cpp b/libraries/networking/src/SentPacketHistory.cpp index a34dada397..1e1157ba71 100644 --- a/libraries/networking/src/SentPacketHistory.cpp +++ b/libraries/networking/src/SentPacketHistory.cpp @@ -9,22 +9,26 @@ // #include "SentPacketHistory.h" +#include SentPacketHistory::SentPacketHistory(int size) : _sentPackets(size), _newestPacketAt(0), _numExistingPackets(0), - _newestSequenceNumber(0) + _newestSequenceNumber(UINT16_MAX) { } void SentPacketHistory::packetSent(uint16_t sequenceNumber, const QByteArray& packet) { - if (sequenceNumber != 0 && sequenceNumber != _newestSequenceNumber + 1) { - printf("\t\tpacket history received unexpected seq number! prev: %hu received: %hu **************** \n", _newestSequenceNumber, sequenceNumber); + // check if given seq number has the expected value. if not, something's wrong with + // the code calling this function + uint16_t expectedSequenceNumber = _newestSequenceNumber + (uint16_t)1; + if (sequenceNumber != expectedSequenceNumber) { + qDebug() << "Unexpected sequence number passed to SentPacketHistory::packetSent()!" + << "Expected:" << expectedSequenceNumber << "Actual:" << sequenceNumber; } - _newestSequenceNumber = sequenceNumber; // increment _newestPacketAt cyclically, insert new packet there. @@ -36,7 +40,6 @@ void SentPacketHistory::packetSent(uint16_t sequenceNumber, const QByteArray& pa } } - const QByteArray* SentPacketHistory::getPacket(uint16_t sequenceNumber) const { const int UINT16_RANGE = UINT16_MAX + 1; diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index f82e7e3101..01d2ecf464 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -34,7 +34,7 @@ OctreeEditPacketSender::OctreeEditPacketSender() : _maxPendingMessages(DEFAULT_MAX_PENDING_MESSAGES), _releaseQueuedMessagesPending(false), _serverJurisdictions(NULL), - _sequenceNumber(0), + _sequenceNumber(65500), _maxPacketSize(MAX_PACKET_SIZE) { } From 011e7c2de2f7ccfbfd2b50e7a47b3d36e7e913e0 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 11:53:09 -0700 Subject: [PATCH 29/55] removed debug code --- .../octree/OctreeInboundPacketProcessor.cpp | 18 +++---------- .../src/octree/OctreeInboundPacketProcessor.h | 3 +-- .../src/ReceivedPacketProcessor.cpp | 1 - .../octree/src/OctreeEditPacketSender.cpp | 25 ++----------------- 4 files changed, 6 insertions(+), 41 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 44c9576943..9a4c6d7cbb 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -204,7 +204,6 @@ int OctreeInboundPacketProcessor::sendNackPackets() { // check if this node is still alive. Remove its stats if it's dead. if (!isAlive(nodeUUID)) { - printf("\t\t removing node %s\n", nodeUUID.toString().toLatin1().data()); i = _singleSenderStats.erase(i); continue; } @@ -244,12 +243,10 @@ int OctreeInboundPacketProcessor::sendNackPackets() { dataAt += sizeof(uint16_t); // pack sequence numbers to nack - printf("\t\t sending NACK with %d seq numbers:\n\t\t", numSequenceNumbers); for (uint16_t i = 0; i < numSequenceNumbers; i++) { unsigned short int* sequenceNumberAt = (unsigned short int*)dataAt; *sequenceNumberAt = *missingSequenceNumberIterator; dataAt += sizeof(unsigned short int); - printf("%d, ", *missingSequenceNumberIterator); missingSequenceNumberIterator++; } @@ -257,7 +254,6 @@ int OctreeInboundPacketProcessor::sendNackPackets() { // send it qint64 bytesWritten = NodeList::getInstance()->writeDatagram(packet, dataAt - packet, destinationNode); - printf("\t\t wrote %lld bytes\n\n", bytesWritten); packetsSent++; } @@ -282,8 +278,6 @@ SingleSenderStats::SingleSenderStats() void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { -printf("\t\t tracked seq %hu\n", incomingSequence); - const int UINT16_RANGE = UINT16_MAX + 1; const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work @@ -321,8 +315,6 @@ printf("\t\t tracked seq %hu\n", incomingSequence); if (incoming > expected) { // early - printf("\t\t\t packet is early! %d packets were skipped\n", incoming - expected); - // add all sequence numbers that were skipped to the missing sequence numbers list for (int missingSequence = expected; missingSequence < incoming; missingSequence++) { _missingSequenceNumbers.insert(missingSequence < 0 ? missingSequence + UINT16_RANGE : missingSequence); @@ -331,14 +323,10 @@ printf("\t\t tracked seq %hu\n", incomingSequence); } else { // late - printf("\t\t\t packet is late!\n"); - // remove this from missing sequence number if it's in there - if (_missingSequenceNumbers.remove(incomingSequence)) { - printf("\t\t\t\t packet %d recovered!!!\n", incomingSequence); - } + _missingSequenceNumbers.remove(incomingSequence); - // do not update _incomingLastSequence + // do not update _incomingLastSequence; it shouldn't become smaller } } @@ -373,4 +361,4 @@ printf("\t\t tracked seq %hu\n", incomingSequence); _totalLockWaitTime += lockWaitTime; _totalElementsInPacket += editsInPacket; _totalPackets++; -} \ No newline at end of file +} diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index 378cc9a891..d3b3b80208 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -37,7 +37,6 @@ public: void trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime); - quint64 _totalTransitTime; quint64 _totalProcessTime; quint64 _totalLockWaitTime; @@ -95,7 +94,7 @@ private: quint64 _totalLockWaitTime; quint64 _totalElementsInPacket; quint64 _totalPackets; - + NodeToSenderStatsMap _singleSenderStats; quint64 _lastNackTime; diff --git a/libraries/networking/src/ReceivedPacketProcessor.cpp b/libraries/networking/src/ReceivedPacketProcessor.cpp index d85f09fb0a..3ef518bbc2 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.cpp +++ b/libraries/networking/src/ReceivedPacketProcessor.cpp @@ -54,5 +54,4 @@ void ReceivedPacketProcessor::nodeKilled(SharedNodePointer node) { lock(); _nodePacketCounts.remove(node->getUUID()); unlock(); - printf("\n\t\t nodeKilled()!!!!! --------------------------\n\n"); } diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index 01d2ecf464..f49fe9f22f 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -34,7 +34,7 @@ OctreeEditPacketSender::OctreeEditPacketSender() : _maxPendingMessages(DEFAULT_MAX_PENDING_MESSAGES), _releaseQueuedMessagesPending(false), _serverJurisdictions(NULL), - _sequenceNumber(65500), + _sequenceNumber(0), _maxPacketSize(MAX_PACKET_SIZE) { } @@ -98,36 +98,15 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsi if (node->getActiveSocket()) { QByteArray packet(reinterpret_cast(buffer), length); - bool send = randFloat() < 0.7f; - if (send) queuePacketForSending(node, packet); // extract sequence number and add packet to history int numBytesPacketHeader = numBytesForPacketHeader(packet); const char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; unsigned short int sequence = *((unsigned short int*)dataAt); -/* -// debug -dataAt += sizeof(unsigned short int); - -// extract time stamp -quint64 sentTime = *((quint64*)dataAt); -dataAt += sizeof(quint64); - -PacketType type = packetTypeForPacket(packet); - - -printf("adding packet to history. size: %d\n", packet.length()); -printf("type: %d, seq: %hu, time: %llu\n", (unsigned char)type, sequence, sentTime); -printf("destination node: %s\n", nodeUUID.toString().toLatin1().data()); -fflush(stdout); -*/ + _sentPacketHistories[nodeUUID].packetSent(sequence, packet); - if (!send) { - printf("\t dropped packet %d !!! ---------------------------\n", sequence); - } - // debugging output... bool wantDebugging = false; if (wantDebugging) { From 5f62b43ba6e0ebd1aafd5874c00d1377242c0649 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 12:14:44 -0700 Subject: [PATCH 30/55] added edit nack types to non-verified list --- libraries/networking/src/PacketHeaders.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/PacketHeaders.h b/libraries/networking/src/PacketHeaders.h index 5ed4110627..0f87b0e607 100644 --- a/libraries/networking/src/PacketHeaders.h +++ b/libraries/networking/src/PacketHeaders.h @@ -79,7 +79,7 @@ const QSet NON_VERIFIED_PACKETS = QSet() << PacketTypeDomainList << PacketTypeDomainListRequest << PacketTypeDomainOAuthRequest << PacketTypeCreateAssignment << PacketTypeRequestAssignment << PacketTypeStunResponse << PacketTypeNodeJsonStats << PacketTypeVoxelQuery << PacketTypeParticleQuery << PacketTypeModelQuery - << PacketTypeOctreeDataNack; + << PacketTypeOctreeDataNack << PacketTypeVoxelEditNack << PacketTypeParticleEditNack << PacketTypeModelEditNack; const int NUM_BYTES_MD5_HASH = 16; const int NUM_STATIC_HEADER_BYTES = sizeof(PacketVersion) + NUM_BYTES_RFC4122_UUID; From 793117367155fd9e92ebb6e086d65e6b03afc5e6 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 12:33:31 -0700 Subject: [PATCH 31/55] Working on tools to convert bitstreams to/from json. --- CMakeLists.txt | 1 + cmake/macros/AutoMTC.cmake | 6 +- libraries/metavoxels/src/Bitstream.cpp | 1 + libraries/metavoxels/src/Bitstream.h | 4 ++ tests/metavoxels/src/MetavoxelTests.cpp | 2 + tools/CMakeLists.txt | 6 ++ tools/bitstream2json/CMakeLists.txt | 20 +++++++ tools/bitstream2json/src/main.cpp | 70 +++++++++++++++++++++++ tools/json2bitstream/CMakeLists.txt | 20 +++++++ tools/json2bitstream/src/main.cpp | 76 +++++++++++++++++++++++++ 10 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 tools/CMakeLists.txt create mode 100644 tools/bitstream2json/CMakeLists.txt create mode 100644 tools/bitstream2json/src/main.cpp create mode 100644 tools/json2bitstream/CMakeLists.txt create mode 100644 tools/json2bitstream/src/main.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index cb1e4224cf..a399e11168 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -49,4 +49,5 @@ add_subdirectory(assignment-client) add_subdirectory(domain-server) add_subdirectory(interface) add_subdirectory(tests) +add_subdirectory(tools) add_subdirectory(voxel-edit) diff --git a/cmake/macros/AutoMTC.cmake b/cmake/macros/AutoMTC.cmake index 1682b9cd56..6f0216de7f 100644 --- a/cmake/macros/AutoMTC.cmake +++ b/cmake/macros/AutoMTC.cmake @@ -9,13 +9,9 @@ # macro(AUTO_MTC TARGET ROOT_DIR) - if (NOT TARGET mtc) - add_subdirectory("${ROOT_DIR}/tools/mtc" "${ROOT_DIR}/tools/mtc") - endif () - set(AUTOMTC_SRC ${TARGET}_automtc.cpp) file(GLOB INCLUDE_FILES src/*.h) add_custom_command(OUTPUT ${AUTOMTC_SRC} COMMAND mtc -o ${AUTOMTC_SRC} ${INCLUDE_FILES} DEPENDS mtc ${INCLUDE_FILES}) -endmacro() \ No newline at end of file +endmacro() diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 86f508ca11..0667e71fb1 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -33,6 +33,7 @@ REGISTER_SIMPLE_TYPE_STREAMER(QByteArray) REGISTER_SIMPLE_TYPE_STREAMER(QColor) REGISTER_SIMPLE_TYPE_STREAMER(QScriptValue) REGISTER_SIMPLE_TYPE_STREAMER(QString) +REGISTER_SIMPLE_TYPE_STREAMER(QVariant) REGISTER_SIMPLE_TYPE_STREAMER(QUrl) REGISTER_SIMPLE_TYPE_STREAMER(QVariantList) REGISTER_SIMPLE_TYPE_STREAMER(QVariantHash) diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 776a362717..66a42e6991 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -805,6 +805,8 @@ public: template JSONWriter& operator<<(const T& value) { _contents.append(getData(value)); return *this; } + void appendToContents(const QJsonValue& value) { _contents.append(value); } + void addSharedObject(const SharedObjectPointer& object); void addObjectStreamer(const ObjectStreamer* streamer); void addTypeStreamer(const TypeStreamer* streamer); @@ -893,6 +895,8 @@ public: template JSONReader& operator>>(T& value) { putData(*_contentsIterator++, value); return *this; } + QJsonValue retrieveNextFromContents() { return *_contentsIterator++; } + TypeStreamerPointer getTypeStreamer(const QString& name) const; ObjectStreamerPointer getObjectStreamer(const QString& name) const { return _objectStreamers.value(name); } SharedObjectPointer getSharedObject(int id) const { return _sharedObjects.value(id); } diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 4a3010caf4..85be9a5c8b 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -255,6 +255,8 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { jsonWriter << endRead; QByteArray encodedJson = jsonWriter.getDocument().toJson(); + qDebug() << encodedJson; + // and read from JSON JSONReader jsonReader(QJsonDocument::fromJson(encodedJson), Bitstream::ALL_GENERICS); jsonReader >> testObjectReadA; diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt new file mode 100644 index 0000000000..79db82e90f --- /dev/null +++ b/tools/CMakeLists.txt @@ -0,0 +1,6 @@ +cmake_minimum_required(VERSION 2.8) + +# add the tool directories +add_subdirectory(bitstream2json) +add_subdirectory(json2bitstream) +add_subdirectory(mtc) diff --git a/tools/bitstream2json/CMakeLists.txt b/tools/bitstream2json/CMakeLists.txt new file mode 100644 index 0000000000..6483c024a7 --- /dev/null +++ b/tools/bitstream2json/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME bitstream2json) + +set(ROOT_DIR ../..) +set(MACRO_DIR "${ROOT_DIR}/cmake/macros") + +find_package(Qt5 COMPONENTS Network Script Widgets) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") + +target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/bitstream2json/src/main.cpp b/tools/bitstream2json/src/main.cpp new file mode 100644 index 0000000000..0f299527b0 --- /dev/null +++ b/tools/bitstream2json/src/main.cpp @@ -0,0 +1,70 @@ +// +// main.cpp +// tools/bitstream2json/src +// +// Created by Andrzej Kapolka on 6/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include +#include +#include + +#include + +using namespace std; + +int main (int argc, char** argv) { + // need the core application for the script engine + QCoreApplication app(argc, argv); + + if (argc < 3) { + cerr << "Usage: bitstream2json inputfile outputfile [types...]" << endl; + return 0; + } + QFile inputFile(argv[1]); + if (!inputFile.open(QIODevice::ReadOnly)) { + cerr << "Failed to open input file: " << inputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QDataStream inputData(&inputFile); + Bitstream input(inputData, Bitstream::FULL_METADATA, Bitstream::ALL_GENERICS); + + QFile outputFile(argv[2]); + if (!outputFile.open(QIODevice::WriteOnly)) { + cerr << "Failed to open output file: " << outputFile.errorString().toLatin1().constData() << endl; + return 1; + } + JSONWriter output; + + if (argc < 4) { + // default type is a single QVariant + QVariant value; + input >> value; + output << value; + + } else { + for (int i = 3; i < argc; i++) { + int type = QMetaType::type(argv[i]); + if (type == QMetaType::UnknownType) { + cerr << "Unknown type: " << argv[i] << endl; + return 1; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamer(type); + if (!streamer) { + cerr << "Non-streamable type: " << argv[i] << endl; + return 1; + } + QVariant value = streamer->read(input); + output.appendToContents(streamer->getJSONData(output, value)); + } + } + + outputFile.write(output.getDocument().toJson()); + + return 0; +} diff --git a/tools/json2bitstream/CMakeLists.txt b/tools/json2bitstream/CMakeLists.txt new file mode 100644 index 0000000000..799f3bcc4c --- /dev/null +++ b/tools/json2bitstream/CMakeLists.txt @@ -0,0 +1,20 @@ +cmake_minimum_required(VERSION 2.8) + +if (WIN32) + cmake_policy (SET CMP0020 NEW) +endif (WIN32) + +set(TARGET_NAME json2bitstream) + +set(ROOT_DIR ../..) +set(MACRO_DIR "${ROOT_DIR}/cmake/macros") + +find_package(Qt5 COMPONENTS Network Script Widgets) + +include(${MACRO_DIR}/SetupHifiProject.cmake) +setup_hifi_project(${TARGET_NAME} TRUE) + +link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") +link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") + +target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/json2bitstream/src/main.cpp b/tools/json2bitstream/src/main.cpp new file mode 100644 index 0000000000..ff09bcfdc6 --- /dev/null +++ b/tools/json2bitstream/src/main.cpp @@ -0,0 +1,76 @@ +// +// main.cpp +// tools/json2bitstream/src +// +// Created by Andrzej Kapolka on 6/17/14. +// Copyright 2014 High Fidelity, Inc. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html + +#include + +#include +#include +#include + +#include + +using namespace std; + +int main (int argc, char** argv) { + // need the core application for the script engine + QCoreApplication app(argc, argv); + + if (argc < 3) { + cerr << "Usage: bitstream2json inputfile outputfile [types...]" << endl; + return 0; + } + QFile inputFile(argv[1]); + if (!inputFile.open(QIODevice::ReadOnly)) { + cerr << "Failed to open input file: " << inputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QJsonParseError error; + QJsonDocument document = QJsonDocument::fromJson(inputFile.readAll(), &error); + if (error.error != QJsonParseError::NoError) { + cerr << "Failed to read input file: " << error.errorString().toLatin1().constData() << endl; + return 1; + } + JSONReader input(document, Bitstream::ALL_GENERICS); + + QFile outputFile(argv[2]); + if (!outputFile.open(QIODevice::WriteOnly)) { + cerr << "Failed to open output file: " << outputFile.errorString().toLatin1().constData() << endl; + return 1; + } + QDataStream outputData(&outputFile); + Bitstream output(outputData, Bitstream::FULL_METADATA); + + if (argc < 4) { + // default type is a single QVariant + QVariant value; + input >> value; + output << value; + + } else { + for (int i = 3; i < argc; i++) { + int type = QMetaType::type(argv[i]); + if (type == QMetaType::UnknownType) { + cerr << "Unknown type: " << argv[i] << endl; + return 1; + } + const TypeStreamer* streamer = Bitstream::getTypeStreamer(type); + if (!streamer) { + cerr << "Non-streamable type: " << argv[i] << endl; + return 1; + } + QVariant value; + streamer->putJSONData(input, input.retrieveNextFromContents(), value); + streamer->write(output, value); + } + } + output.flush(); + + return 0; +} From 604b17185beee992101113a95205f1419b582590 Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 14:43:13 -0700 Subject: [PATCH 32/55] call writeUnverifiedDatagram for sending edit nacks --- assignment-client/src/octree/OctreeInboundPacketProcessor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 9a4c6d7cbb..91a5c62752 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -253,7 +253,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { numSequenceNumbersAvailable -= numSequenceNumbers; // send it - qint64 bytesWritten = NodeList::getInstance()->writeDatagram(packet, dataAt - packet, destinationNode); + qint64 bytesWritten = NodeList::getInstance()->writeUnverifiedDatagram(packet, dataAt - packet, destinationNode); packetsSent++; } From c3090dd223e87dc0b0bdc191b86592d7afa11ef5 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 16:12:06 -0700 Subject: [PATCH 33/55] Tweak to the way streamers are written to JSON. --- libraries/metavoxels/src/Bitstream.cpp | 48 +++++++++----------------- 1 file changed, 16 insertions(+), 32 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 0667e71fb1..9f6ceb9b2e 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1836,14 +1836,9 @@ void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { if (!_typeStreamerNames.contains(streamer->getName())) { _typeStreamerNames.insert(streamer->getName()); - // start with a placeholder; then remove/replace with actual metadata - int index = _typeStreamers.size(); - _typeStreamers.append(QJsonValue()); QJsonValue metadata = streamer->getJSONMetadata(*this); - if (metadata.isNull()) { - _typeStreamers.removeAt(index); - } else { - _typeStreamers.replace(index, metadata); + if (!metadata.isNull()) { + _typeStreamers.append(metadata); } } } @@ -1851,11 +1846,7 @@ void JSONWriter::addTypeStreamer(const TypeStreamer* streamer) { void JSONWriter::addObjectStreamer(const ObjectStreamer* streamer) { if (!_objectStreamerNames.contains(streamer->getName())) { _objectStreamerNames.insert(streamer->getName()); - - // start with a placeholder; then replace with actual metadata - int index = _objectStreamers.size(); - _objectStreamers.append(QJsonValue()); - _objectStreamers.replace(index, streamer->getJSONMetadata(*this)); + _objectStreamers.append(streamer->getJSONMetadata(*this)); } } @@ -1863,14 +1854,10 @@ void JSONWriter::addSharedObject(const SharedObjectPointer& object) { if (!_sharedObjectIDs.contains(object->getID())) { _sharedObjectIDs.insert(object->getID()); - // start with a placeholder; then replace with actual object - int index = _sharedObjects.size(); - _sharedObjects.append(QJsonValue()); - QJsonObject sharedObject; sharedObject.insert("id", object->getID()); sharedObject.insert("data", getData(static_cast(object.data()))); - _sharedObjects.replace(index, sharedObject); + _sharedObjects.append(sharedObject); } } @@ -1885,9 +1872,8 @@ QJsonDocument JSONWriter::getDocument() const { JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode) { QJsonObject top = document.object(); - QJsonArray types = top.value("types").toArray(); - for (int i = types.size() - 1; i >= 0; i--) { - QJsonObject type = types.at(i).toObject(); + foreach (const QJsonValue& element, top.value("types").toArray()) { + QJsonObject type = element.toObject(); QString name = type.value("name").toString(); QByteArray latinName = name.toLatin1(); const TypeStreamer* baseStreamer = Bitstream::getTypeStreamers().value(QMetaType::type(latinName)); @@ -1966,12 +1952,12 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge QJsonArray array = type.value("fields").toArray(); const QVector& metaFields = baseStreamer->getMetaFields(); bool matches = (array.size() == metaFields.size()); - for (int j = 0; j < array.size(); j++) { - QJsonObject object = array.at(j).toObject(); + for (int i = 0; i < array.size(); i++) { + QJsonObject object = array.at(i).toObject(); TypeStreamerPointer streamer = getTypeStreamer(object.value("type").toString()); int index = baseStreamer->getFieldIndex(object.value("name").toString().toLatin1()); fields.append(StreamerIndexPair(streamer, index)); - matches &= (index == j && streamer == metaFields.at(j).getStreamer()); + matches &= (index == i && streamer == metaFields.at(i).getStreamer()); } if (matches) { _typeStreamers.insert(name, baseStreamer->getSelf()); @@ -2006,10 +1992,9 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } } - - QJsonArray classes = top.value("classes").toArray(); - for (int i = classes.size() - 1; i >= 0; i--) { - QJsonObject clazz = classes.at(i).toObject(); + + foreach (const QJsonValue& element, top.value("classes").toArray()) { + QJsonObject clazz = element.toObject(); QString name = clazz.value("name").toString(); QByteArray latinName = name.toLatin1(); const ObjectStreamer* baseStreamer = Bitstream::getObjectStreamers().value( @@ -2035,8 +2020,8 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge QVector properties; QJsonArray propertyArray = clazz.value("properties").toArray(); bool matches = (baseProperties.size() == propertyArray.size()); - for (int j = 0; j < propertyArray.size(); j++) { - QJsonObject object = propertyArray.at(j).toObject(); + for (int i = 0; i < propertyArray.size(); i++) { + QJsonObject object = propertyArray.at(i).toObject(); TypeStreamerPointer typeStreamer = getTypeStreamer(object.value("type").toString()); QMetaProperty metaProperty = metaObject->property(metaObject->indexOfProperty( object.value("name").toString().toLatin1())); @@ -2053,9 +2038,8 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } - QJsonArray objects = top.value("objects").toArray(); - for (int i = objects.size() - 1; i >= 0; i--) { - QJsonObject object = objects.at(i).toObject(); + foreach (const QJsonValue& element, top.value("objects").toArray()) { + QJsonObject object = element.toObject(); int id = object.value("id").toInt(); QObject* qObject; putData(object.value("data"), qObject); From 7f4cf3719e709078f07a61d62fe89441bf22a7ee Mon Sep 17 00:00:00 2001 From: wangyix Date: Tue, 17 Jun 2014 16:40:08 -0700 Subject: [PATCH 34/55] added rollover handling to OctreeSceneStats --- libraries/octree/src/OctreeSceneStats.cpp | 131 +++++++++++++++------- 1 file changed, 89 insertions(+), 42 deletions(-) diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 868ef29886..e99096dd5c 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -842,8 +842,8 @@ const char* OctreeSceneStats::getItemValue(Item item) { } void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, - bool wasStatsPacket, int nodeClockSkewUsec) { - const bool wantExtraDebugging = false; + bool wasStatsPacket, int nodeClockSkewUsec) { + const bool wantExtraDebugging = true; int numBytesPacketHeader = numBytesForPacketHeader(packet); const unsigned char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; @@ -852,10 +852,10 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, dataAt += sizeof(OCTREE_PACKET_FLAGS); OCTREE_PACKET_SEQUENCE sequence = (*(OCTREE_PACKET_SEQUENCE*)dataAt); dataAt += sizeof(OCTREE_PACKET_SEQUENCE); - + OCTREE_PACKET_SENT_TIME sentAt = (*(OCTREE_PACKET_SENT_TIME*)dataAt); dataAt += sizeof(OCTREE_PACKET_SENT_TIME); - + //bool packetIsColored = oneAtBit(flags, PACKET_IS_COLOR_BIT); //bool packetIsCompressed = oneAtBit(flags, PACKET_IS_COMPRESSED_BIT); @@ -877,49 +877,67 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, return; // ignore any packets that are unreasonable } + const int UINT16_RANGE = UINT16_MAX + 1; + // determine our expected sequence number... handle rollover appropriately - OCTREE_PACKET_SEQUENCE expected = _incomingPacket > 0 ? _incomingLastSequence + 1 : sequence; - - // Guard against possible corrupted packets... with bad sequence numbers - const int MAX_RESONABLE_SEQUENCE_OFFSET = 2000; - const int MIN_RESONABLE_SEQUENCE_OFFSET = -2000; - int sequenceOffset = (sequence - expected); - if (sequenceOffset > MAX_RESONABLE_SEQUENCE_OFFSET || sequenceOffset < MIN_RESONABLE_SEQUENCE_OFFSET) { - qDebug() << "ignoring unreasonable packet... sequence:" << sequence << "_incomingLastSequence:" << _incomingLastSequence; - return; // ignore any packets that are unreasonable - } - - // track packets here... - _incomingPacket++; - _incomingBytes += packet.size(); - if (!wasStatsPacket) { - _incomingWastedBytes += (MAX_PACKET_SIZE - packet.size()); - } + OCTREE_PACKET_SEQUENCE expected = _incomingPacket > 0 ? _incomingLastSequence + (quint16)1 : sequence; const int USECS_PER_MSEC = 1000; float flightTimeMsecs = flightTime / USECS_PER_MSEC; _incomingFlightTimeAverage.updateAverage(flightTimeMsecs); - + // track out of order and possibly lost packets... if (sequence == _incomingLastSequence) { if (wantExtraDebugging) { qDebug() << "last packet duplicate got:" << sequence << "_incomingLastSequence:" << _incomingLastSequence; } - } else { + } + else { if (sequence != expected) { if (wantExtraDebugging) { qDebug() << "out of order... got:" << sequence << "expected:" << expected; } + int sequenceInt = (int)sequence; + int expectedInt = (int)expected; + + // if distance between sequence and expected are more than half of the total range of possible seq numbers, + // assume that a rollover occurred between the two. + // correct the larger one so it's in the range [-UINT16_RANGE, -1] while the other remains in [0, UINT16_RANGE-1] + // after doing so, sequenceInt and expectedInt can be correctly compared to each other, though one may be negative + if (std::abs(sequenceInt - expectedInt) > UINT16_RANGE / 2) { + if (sequenceInt > expectedInt) { + sequenceInt -= UINT16_RANGE; + } + else { + expectedInt -= UINT16_RANGE; + } + } + + // Guard against possible corrupted packets... with bad sequence numbers + const int MAX_RESONABLE_SEQUENCE_OFFSET = 2000; + const int MIN_RESONABLE_SEQUENCE_OFFSET = -2000; + + int sequenceOffset = (sequenceInt - expectedInt); + if (sequenceOffset > MAX_RESONABLE_SEQUENCE_OFFSET || sequenceOffset < MIN_RESONABLE_SEQUENCE_OFFSET) { + qDebug() << "ignoring unreasonable packet... sequence:" << sequence << "_incomingLastSequence:" << _incomingLastSequence; + return; // ignore any packets that are unreasonable + } + // if the sequence is less than our expected, then this might be a packet // that was delayed and so we should find it in our lostSequence list - if (sequence < expected) { + if (sequenceInt < expectedInt) { + + // if no rollover between them: sequenceInt, expectedInt are both in range [0, UINT16_RANGE-1] + // if rollover between them: sequenceInt in [-UINT16_RANGE, -1], expectedInt in [0, UINT16_RANGE-1] + if (wantExtraDebugging) { qDebug() << "this packet is later than expected..."; } - if (sequence < std::max(0, (expected - MAX_MISSING_SEQUENCE_OLD_AGE))) { + if (sequenceInt < expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE) { _incomingReallyLate++; - } else { + } + else { _incomingLate++; } @@ -931,57 +949,80 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, _sequenceNumbersToNack.remove(sequence); _incomingLikelyLost--; _incomingRecovered++; - } else { + } + else { // if we're still in our pruning window, and we didn't find it in our missing list, // than this is really unexpected and can probably only happen if the packet was a // duplicate - if (sequence >= std::max(0, (expected - MAX_MISSING_SEQUENCE_OLD_AGE))) { + if (sequenceInt >= expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE) { if (wantExtraDebugging) { - qDebug() << "sequence:" << sequence << "WAS NOT found in _missingSequenceNumbers, and not that old... (expected - MAX_MISSING_SEQUENCE_OLD_AGE):" << (expected - MAX_MISSING_SEQUENCE_OLD_AGE); + qDebug() << "sequence:" << sequence << "WAS NOT found in _missingSequenceNumbers, and not that old... (expected - MAX_MISSING_SEQUENCE_OLD_AGE):" + << (uint16_t)(expectedInt - MAX_MISSING_SEQUENCE_OLD_AGE); } _incomingPossibleDuplicate++; } } + + // don't update _incomingLastSequence in this case. + // only bump the last sequence if it was greater than our expected sequence, this will keep us from + // accidentally going backwards when an out of order (recovered) packet comes in + } - - if (sequence > expected) { + else { // sequenceInt > expectedInt + + // if no rollover between them: sequenceInt, expectedInt are both in range [0, UINT16_RANGE-1] + // if rollover between them: sequenceInt in [0, UINT16_RANGE-1], expectedInt in [-UINT16_RANGE, -1] + if (wantExtraDebugging) { qDebug() << "this packet is earlier than expected..."; } _incomingEarly++; // hmm... so, we either didn't get some packets, or this guy came early... - unsigned int missing = sequence - expected; + int missing = sequenceInt - expectedInt; if (wantExtraDebugging) { qDebug() << ">>>>>>>> missing gap=" << missing; } _incomingLikelyLost += missing; - for(unsigned int missingSequence = expected; missingSequence < sequence; missingSequence++) { + for (int missingSequenceInt = expectedInt; missingSequenceInt < sequenceInt; missingSequenceInt++) { + OCTREE_PACKET_SEQUENCE missingSequence = missingSequenceInt >= 0 ? missingSequenceInt : missingSequenceInt + UINT16_RANGE; _missingSequenceNumbers << missingSequence; _sequenceNumbersToNack << missingSequence; } + + _incomingLastSequence = sequence; } } + else { // sequence = expected + + _incomingLastSequence = sequence; + } } - // only bump the last sequence if it was greater than our expected sequence, this will keep us from - // accidentally going backwards when an out of order (recovered) packet comes in - if (sequence >= expected) { - _incomingLastSequence = sequence; - } - // do some garbage collecting on our _missingSequenceNumbers if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE) { if (wantExtraDebugging) { qDebug() << "too many _missingSequenceNumbers:" << _missingSequenceNumbers.size(); } + + int oldAgeCutoff = (int)_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE; + foreach(uint16_t missingItem, _missingSequenceNumbers) { if (wantExtraDebugging) { qDebug() << "checking item:" << missingItem << "is it in need of pruning?"; - qDebug() << "(_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE):" - << (_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE); + qDebug() << "(_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE):" + << (uint16_t)((int)_incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE); } - if (missingItem <= std::max(0, _incomingLastSequence - MAX_MISSING_SEQUENCE_OLD_AGE)) { + + bool prune; + if (oldAgeCutoff >= 0) { + prune = (missingItem <= oldAgeCutoff || missingItem > _incomingLastSequence); + } + else { + prune = (missingItem <= oldAgeCutoff + UINT16_RANGE && missingItem > _incomingLastSequence); + } + + if (prune) { if (wantExtraDebugging) { qDebug() << "pruning really old missing sequence:" << missingItem; } @@ -991,6 +1032,12 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, } } + // track packets here... + _incomingPacket++; + _incomingBytes += packet.size(); + if (!wasStatsPacket) { + _incomingWastedBytes += (MAX_PACKET_SIZE - packet.size()); + } } int OctreeSceneStats::getNumSequenceNumbersToNack() const { From 758beff3d9b15391b656af283e1ab2626ed6d815 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 17:06:28 -0700 Subject: [PATCH 35/55] Make sure to use same enum streamer for the ones registered by type and the ones registered by scope/name. --- libraries/metavoxels/src/Bitstream.cpp | 14 +++++++++++--- tests/metavoxels/src/MetavoxelTests.cpp | 2 -- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 9f6ceb9b2e..d78b16aab4 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1621,6 +1621,10 @@ const QHash& Bitstream::getEnumStreamers() { return enumStreamers; } +static QByteArray getEnumName(const char* scope, const char* name) { + return QByteArray(scope) + "::" + name; +} + QHash Bitstream::createEnumStreamers() { QHash enumStreamers; foreach (const QMetaObject* metaObject, getMetaObjects()) { @@ -1628,7 +1632,11 @@ QHash Bitstream::createEnumStreamers() { QMetaEnum metaEnum = metaObject->enumerator(i); const TypeStreamer*& streamer = enumStreamers[ScopeNamePair(metaEnum.scope(), metaEnum.name())]; if (!streamer) { - streamer = new EnumTypeStreamer(metaEnum); + // look for a streamer registered by name + streamer = getTypeStreamers().value(QMetaType::type(getEnumName(metaEnum.scope(), metaEnum.name()))); + if (!streamer) { + streamer = new EnumTypeStreamer(metaEnum); + } } } } @@ -2705,7 +2713,7 @@ QDebug& operator<<(QDebug& debug, const QMetaObject* metaObject) { EnumTypeStreamer::EnumTypeStreamer(const QMetaObject* metaObject, const char* name) : _metaObject(metaObject), _enumName(name), - _name(QByteArray(metaObject->className()) + "::" + name), + _name(getEnumName(metaObject->className(), name)), _bits(-1) { _type = QMetaType::Int; @@ -2713,7 +2721,7 @@ EnumTypeStreamer::EnumTypeStreamer(const QMetaObject* metaObject, const char* na } EnumTypeStreamer::EnumTypeStreamer(const QMetaEnum& metaEnum) : - _name(QByteArray(metaEnum.scope()) + "::" + metaEnum.name()), + _name(getEnumName(metaEnum.scope(), metaEnum.name())), _metaEnum(metaEnum), _bits(-1) { diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 85be9a5c8b..4a3010caf4 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -255,8 +255,6 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { jsonWriter << endRead; QByteArray encodedJson = jsonWriter.getDocument().toJson(); - qDebug() << encodedJson; - // and read from JSON JSONReader jsonReader(QJsonDocument::fromJson(encodedJson), Bitstream::ALL_GENERICS); jsonReader >> testObjectReadA; From 5e0cab59ea6a874a91a192a67bfeccef7c3e4d11 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 18:12:06 -0700 Subject: [PATCH 36/55] Some comments. --- libraries/metavoxels/src/Bitstream.h | 83 +++++++++++++++++++++------- 1 file changed, 63 insertions(+), 20 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 66a42e6991..8af35c86fa 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -199,12 +199,20 @@ template inline RepeatedValueStreamer& return *this; } -/// A stream for bit-aligned data. +/// A stream for bit-aligned data. Through a combination of code generation, reflection, macros, and templates, provides a +/// serialization mechanism that may be used for both networking and persistent storage. For unreliable networking, the +/// class provides a mapping system that resends mappings for ids until they are acknowledged (and thus persisted). For +/// version-resilient persistence, the class provides a metadata system that maps stored types to local types (or to +/// generic containers), allowing one to add and remove fields to classes without breaking compatibility with previously +/// stored data. class Bitstream : public QObject { Q_OBJECT public: + /// Stores a set of mappings from values to ids written. Typically, one would store these mappings along with the send + /// record of the packet that contained them, persisting the mappings if/when the packet is acknowledged by the remote + /// party. class WriteMappings { public: QHash objectStreamerOffsets; @@ -214,6 +222,9 @@ public: QHash sharedObjectOffsets; }; + /// Stores a set of mappings from ids to values read. Typically, one would store these mappings along with the receive + /// record of the packet that contained them, persisting the mappings if/when the remote party indicates that it + /// has received the local party's acknowledgement of the packet. class ReadMappings { public: QHash objectStreamerValues; @@ -223,11 +234,14 @@ public: QHash sharedObjectValues; }; - /// Registers a metaobject under its name so that instances of it can be streamed. + /// Registers a metaobject under its name so that instances of it can be streamed. Consider using the REGISTER_META_OBJECT + /// at the top level of the source file associated with the class rather than calling this function directly. /// \return zero; the function only returns a value so that it can be used in static initialization static int registerMetaObject(const char* className, const QMetaObject* metaObject); - /// Registers a streamer for the specified Qt-registered type. + /// Registers a streamer for the specified Qt-registered type. Consider using one of the registration macros (such as + /// REGISTER_SIMPLE_TYPE_STREAMER) at the top level of the associated source file rather than calling this function + /// directly. /// \return zero; the function only returns a value so that it can be used in static initialization static int registerTypeStreamer(int type, TypeStreamer* streamer); @@ -237,7 +251,9 @@ public: /// Returns the meta-object registered under the supplied class name, if any. static const QMetaObject* getMetaObject(const QByteArray& className); - /// Returns the list of registered subclasses for the supplied meta-object. + /// Returns the list of registered subclasses for the supplied meta-object. When registered, metaobjects register + /// themselves as subclasses of all of their parents, mostly in order to allow editors to provide lists of available + /// subclasses. static QList getMetaObjectSubClasses(const QMetaObject* metaObject); enum MetadataType { NO_METADATA, HASH_METADATA, FULL_METADATA }; @@ -254,7 +270,8 @@ public: Bitstream(QDataStream& underlying, MetadataType metadataType = NO_METADATA, GenericsMode = NO_GENERICS, QObject* parent = NULL); - /// Substitutes the supplied metaobject for the given class name's default mapping. + /// Substitutes the supplied metaobject for the given class name's default mapping. This is mostly useful for testing the + /// process of mapping between different types, but may in the future be used for permanently renaming classes. void addMetaObjectSubstitution(const QByteArray& className, const QMetaObject* metaObject); /// Substitutes the supplied type for the given type name's default mapping. @@ -773,7 +790,9 @@ template inline Bitstream& Bitstream::operator>>(QHash& return *this; } -/// Tracks state when writing to JSON. +/// Provides a means of writing Bitstream-able data to JSON rather than the usual binary format in a manner that allows it to +/// be manipulated and re-read, converted to binary, etc. To use, create a JSONWriter, stream values in using the << operator, +/// and call getDocument to obtain the JSON data. class JSONWriter { public: @@ -862,10 +881,14 @@ template inline QJsonValue JSONWriter::getData(const QHash> operator. class JSONReader { public: + /// Creates a reader to read from the supplied document. + /// \param genericsMode the generics mode to use: NO_GENERICS to map all types in the document to built-in types, + /// FALLBACK_GENERICS to use generic containers where no matching built-in type is found, or ALL_GENERICS to + /// read all types to generic containers JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode = Bitstream::NO_GENERICS); void putData(const QJsonValue& data, bool& value); @@ -1049,10 +1072,12 @@ private: Q_DECLARE_METATYPE(const QMetaObject*) -/// Macro for registering streamable meta-objects. +/// Macro for registering streamable meta-objects. Typically, one would use this macro at the top level of the source file +/// associated with the class. #define REGISTER_META_OBJECT(x) static int x##Registration = Bitstream::registerMetaObject(#x, &x::staticMetaObject); -/// Contains a value along with a pointer to its streamer. +/// Contains a value along with a pointer to its streamer. This is stored in QVariants when using fallback generics and +/// no mapping to a built-in type can be found, or when using all generics and the value is any non-simple type. class GenericValue { public: @@ -1071,7 +1096,8 @@ private: Q_DECLARE_METATYPE(GenericValue) -/// Contains a list of property values along with a pointer to their metadata. +/// Contains a list of property values along with a pointer to their metadata. This is stored when using fallback generics +/// and no mapping to a built-in class can be found, or for all QObjects when using all generics. class GenericSharedObject : public SharedObject { Q_OBJECT @@ -1495,16 +1521,19 @@ public: virtual void writeVariant(Bitstream& out, const QVariant& value) const; }; -/// Macro for registering simple type streamers. +/// Macro for registering simple type streamers. Typically, one would use this at the top level of the source file +/// associated with the type. #define REGISTER_SIMPLE_TYPE_STREAMER(X) static int X##Streamer = \ Bitstream::registerTypeStreamer(qMetaTypeId(), new SimpleTypeStreamer()); -/// Macro for registering collection type streamers. +/// Macro for registering collection type (QList, QVector, QSet, QMap) streamers. Typically, one would use this at the top +/// level of the source file associated with the type. #define REGISTER_COLLECTION_TYPE_STREAMER(X) static int x##Streamer = \ Bitstream::registerTypeStreamer(qMetaTypeId(), new CollectionTypeStreamer()); -/// Declares the metatype and the streaming operators. The last lines -/// ensure that the generated file will be included in the link phase. +/// Declares the metatype and the streaming operators. Typically, one would use this immediately after the definition of a +/// type flagged as STREAMABLE in its header file. The last lines ensure that the generated file will be included in the link +/// phase. #ifdef _WIN32 #define DECLARE_STREAMABLE_METATYPE(X) Q_DECLARE_METATYPE(X) \ Bitstream& operator<<(Bitstream& out, const X& obj); \ @@ -1542,6 +1571,9 @@ public: _Pragma(STRINGIFY(unused(_TypePtr##X))) #endif +/// Declares an enum metatype. This is used when one desires to use an enum defined in a QObject outside of that class's +/// direct properties. Typically, one would use this immediately after the definition of the QObject containing the enum +/// in its header file. #define DECLARE_ENUM_METATYPE(S, N) Q_DECLARE_METATYPE(S::N) \ Bitstream& operator<<(Bitstream& out, const S::N& obj); \ Bitstream& operator>>(Bitstream& in, S::N& obj); \ @@ -1550,6 +1582,8 @@ public: template<> inline QJsonValue JSONWriter::getData(const S::N& value) { return (int)value; } \ template<> inline void JSONReader::putData(const QJsonValue& data, S::N& value) { value = (S::N)data.toInt(); } +/// Implements an enum metatype. This performs the implementation of the previous macro, and would normally be used at the +/// top level of the source file associated with the QObject containing the enum. #define IMPLEMENT_ENUM_METATYPE(S, N) \ static int S##N##MetaTypeId = registerEnumMetaType(&S::staticMetaObject, #N); \ Bitstream& operator<<(Bitstream& out, const S::N& obj) { \ @@ -1562,7 +1596,9 @@ public: return in.read(&obj, bits); \ } -/// Registers a simple type and its streamer. +/// Registers a simple type and its streamer. This would typically be used at the top level of the source file associated with +/// the type, and combines the registration of the type with the Qt metatype system with the registration of a simple type +/// streamer. /// \return the metatype id template int registerSimpleMetaType() { int type = qRegisterMetaType(); @@ -1570,7 +1606,8 @@ template int registerSimpleMetaType() { return type; } -/// Registers an enum type and its streamer. +/// Registers an enum type and its streamer. Rather than using this directly, consider using the IMPLEMENT_ENUM_METATYPE +/// macro. /// \return the metatype id template int registerEnumMetaType(const QMetaObject* metaObject, const char* name) { int type = qRegisterMetaType(); @@ -1578,7 +1615,8 @@ template int registerEnumMetaType(const QMetaObject* metaObject, const return type; } -/// Registers a streamable type and its streamer. +/// Registers a streamable type and its streamer. Rather than using this directly, consider using the +/// DECLARE_STREAMABLE_METATYPE macro and the mtc code generation tool. /// \return the metatype id template int registerStreamableMetaType() { int type = qRegisterMetaType(); @@ -1586,7 +1624,9 @@ template int registerStreamableMetaType() { return type; } -/// Registers a collection type and its streamer. +/// Registers a collection type and its streamer. This would typically be used at the top level of the source file associated +/// with the type, and combines the registration of the type with the Qt metatype system with the registration of a collection +/// type streamer. /// \return the metatype id template int registerCollectionMetaType() { int type = qRegisterMetaType(); @@ -1594,7 +1634,9 @@ template int registerCollectionMetaType() { return type; } -/// Flags a class as streamable (use as you would Q_OBJECT). +/// Flags a class as streamable. Use as you would Q_OBJECT: as the first line of a class definition, before any members or +/// access qualifiers. Typically, one would follow the definition with DECLARE_STREAMABLE_METATYPE. The mtc tool looks for +/// this macro in order to generate streaming (etc.) code for types. #define STREAMABLE public: \ static const int Type; \ static const QVector& getMetaFields(); \ @@ -1604,7 +1646,8 @@ template int registerCollectionMetaType() { private: \ static QHash createFieldIndices(); -/// Flags a field or base class as streaming. +/// Flags a (public) field within or base class of a streamable type as streaming. Use before all base classes or fields that +/// you want to stream. #define STREAM #endif // hifi_Bitstream_h From b8e6660e1948f2e80482a8b641b9781ebb68195f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Tue, 17 Jun 2014 19:15:29 -0700 Subject: [PATCH 37/55] More commentation. --- libraries/metavoxels/src/Bitstream.cpp | 15 ++++++++-- libraries/metavoxels/src/Bitstream.h | 38 +++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index d78b16aab4..ef5b9b7906 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1879,6 +1879,7 @@ QJsonDocument JSONWriter::getDocument() const { } JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode genericsMode) { + // create and map the type streamers in order QJsonObject top = document.object(); foreach (const QJsonValue& element, top.value("types").toArray()) { QJsonObject type = element.toObject(); @@ -1889,10 +1890,11 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge baseStreamer = Bitstream::getEnumStreamersByName().value(latinName); } if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { - continue; + continue; // no built-in type and no generics allowed; we give up } QString category = type.value("category").toString(); if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + // create a generic streamer TypeStreamerPointer streamer; if (category == "ENUM") { QVector values; @@ -1933,6 +1935,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge static_cast(streamer.data())->_weakSelf = streamer; continue; } + // create a mapped streamer, determining along the way whether it matches our base if (category == "ENUM") { QHash mappings; int highestValue = 0; @@ -1949,6 +1952,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } matches &= (value == mapping); } + // if everything matches our built-in enum, we can use that, which will be faster if (matches) { _typeStreamers.insert(name, baseStreamer->getSelf()); } else { @@ -1967,6 +1971,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge fields.append(StreamerIndexPair(streamer, index)); matches &= (index == i && streamer == metaFields.at(i).getStreamer()); } + // if everything matches our built-in streamable, we can use that, which will be faster if (matches) { _typeStreamers.insert(name, baseStreamer->getSelf()); } else { @@ -2001,6 +2006,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } + // create and map the object streamers in order foreach (const QJsonValue& element, top.value("classes").toArray()) { QJsonObject clazz = element.toObject(); QString name = clazz.value("name").toString(); @@ -2008,9 +2014,10 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge const ObjectStreamer* baseStreamer = Bitstream::getObjectStreamers().value( Bitstream::getMetaObjects().value(latinName)); if (!baseStreamer && genericsMode == Bitstream::NO_GENERICS) { - continue; + continue; // no built-in class and no generics allowed; we give up } if (!baseStreamer || genericsMode == Bitstream::ALL_GENERICS) { + // create a generic streamer QVector properties; foreach (const QJsonValue& property, clazz.value("properties").toArray()) { QJsonObject object = property.toObject(); @@ -2023,6 +2030,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge static_cast(streamer.data())->_weakSelf = streamer; continue; } + // create a mapped streamer, determining along the way whether it matches our base const QMetaObject* metaObject = baseStreamer->getMetaObject(); const QVector& baseProperties = baseStreamer->getProperties(); QVector properties; @@ -2039,6 +2047,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge matches &= (typeStreamer == baseProperty.first && metaProperty.propertyIndex() == baseProperty.second.propertyIndex()); } + // if everything matches our built-in type, we can use that directly, which will be faster if (matches) { _objectStreamers.insert(name, baseStreamer->getSelf()); } else { @@ -2046,6 +2055,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } + // create and map the objects in order foreach (const QJsonValue& element, top.value("objects").toArray()) { QJsonObject object = element.toObject(); int id = object.value("id").toInt(); @@ -2056,6 +2066,7 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge } } + // prepare the contents for extraction _contents = top.value("contents").toArray(); _contentsIterator = _contents.constBegin(); } diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 8af35c86fa..64765833b9 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -205,6 +205,36 @@ template inline RepeatedValueStreamer& /// version-resilient persistence, the class provides a metadata system that maps stored types to local types (or to /// generic containers), allowing one to add and remove fields to classes without breaking compatibility with previously /// stored data. +/// +/// The basic usage requires one to create a Bitstream that wraps an underlying QDataStream, specifying the metadata type +/// desired and (for readers) the generics mode. Then, one uses the << or >> operators to write or read values to/from +/// the stream (a stream instance may be used for reading or writing, but not both). For write streams, the flush +/// function should be called on completion to write any partial data. +/// +/// Polymorphic types are supported via the QVariant and QObject*/SharedObjectPointer types. When you write a QVariant or +/// QObject, the type or class name (at minimum) is written to the stream. When you read a QVariant or QObject, the default +/// behavior is to look for a corresponding registered type with the written name. With hash metadata, Bitstream can verify +/// that the local type matches the stream type, throwing the streamed version away if not. With full metadata, Bitstream can +/// create a mapping from the streamed type to the local type that applies the intersection of the two types' fields. +/// +/// To register types for streaming, select from the provided templates and macros. To register a QObject (or SharedObject) +/// subclass, use REGISTER_META_OBJECT in the class's source file. To register a streamable class for use in QVariant, use +/// the STREAMABLE/STREAM/DECLARE_STREAMABLE_METATYPE macros and use the mtc tool to generate the associated implementation +/// code. To register a QObject enum for use outside the QObject's direct properties, use the +/// DECLARE_ENUM_METATYPE/IMPLEMENT_ENUM_METATYPE macro pair. To register a collection type (QList, QVector, QSet, QMap) of +/// streamable types, use the registerCollectionMetaType template function. +/// +/// Delta-streaming is supported through the writeDelta/readDelta functions (analogous to <>), which accept a reference +/// value and stream only the information necessary to turn the reference into the target value. This assumes that the +/// reference value provided when reading is identical to the one used when writing. +/// +/// Special delta handling is provided for objects tracked by SharedObjectPointers. SharedObjects have IDs (representing their +/// unique identity on one system) as well as origin IDs (representing their identity as preserved over the course of +/// mutation). Once a shared object with a given ID is written (and its mapping persisted), that object's state will not be +/// written again; only its ID will be sent. However, if an object with a new ID but the same origin ID is written, that +/// object's state will be encoded as a delta between the previous object and the new one. So, to transmit delta-encoded +/// objects, one should treat each local SharedObject instance as immutable, replacing it when mutated with a cloned instance +/// generated by calling SharedObject::clone(true) and applying the desired changes. class Bitstream : public QObject { Q_OBJECT @@ -1001,7 +1031,7 @@ protected: friend class Bitstream; const QMetaObject* _metaObject; - ObjectStreamerPointer _self; + ObjectStreamerPointer _self; ///< set/used for built-in classes (never deleted), to obtain shared pointers }; /// A streamer that maps to a local class. @@ -1050,7 +1080,7 @@ private: friend class JSONReader; QByteArray _name; - WeakObjectStreamerPointer _weakSelf; + WeakObjectStreamerPointer _weakSelf; ///< promoted to strong references in GenericSharedObject when reading QVector _properties; QByteArray _hash; }; @@ -1184,7 +1214,7 @@ protected: friend class Bitstream; int _type; - TypeStreamerPointer _self; + TypeStreamerPointer _self; ///< set/used for built-in types (never deleted), to obtain shared pointers }; QDebug& operator<<(QDebug& debug, const TypeStreamer* typeStreamer); @@ -1278,7 +1308,7 @@ protected: friend class JSONReader; QByteArray _name; - WeakTypeStreamerPointer _weakSelf; + WeakTypeStreamerPointer _weakSelf; ///< promoted to strong references in GenericValue when reading }; /// A streamer for generic enums. From 98b4d6f697b100152f3aedf4be905af8e55444a7 Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Wed, 18 Jun 2014 10:27:57 +0300 Subject: [PATCH 38/55] Allow the project to compile using Visual Studio 2013 --- libraries/audio/src/AudioRingBuffer.cpp | 1 + libraries/voxels/src/Tags.cpp | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/audio/src/AudioRingBuffer.cpp b/libraries/audio/src/AudioRingBuffer.cpp index 9b50ed0bcb..2101fcb9cd 100644 --- a/libraries/audio/src/AudioRingBuffer.cpp +++ b/libraries/audio/src/AudioRingBuffer.cpp @@ -10,6 +10,7 @@ // #include +#include #include #include diff --git a/libraries/voxels/src/Tags.cpp b/libraries/voxels/src/Tags.cpp index d3c534735e..003e9ae80d 100644 --- a/libraries/voxels/src/Tags.cpp +++ b/libraries/voxels/src/Tags.cpp @@ -177,7 +177,9 @@ int retrieveData(std::string filename, std::stringstream &ss) { int type = file.peek(); if (type == 0x0A) { ss.flush(); - ss << file; + std::copy(std::istreambuf_iterator(file), + std::istreambuf_iterator(), + std::ostreambuf_iterator(ss)); return 0; } From 32c97bb6a2679a4a4961d08c74f067a9dc9ffddc Mon Sep 17 00:00:00 2001 From: Oren Hurvitz Date: Wed, 18 Jun 2014 12:46:49 +0300 Subject: [PATCH 39/55] Update Windows build instructions --- BUILD.md | 158 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 125 insertions(+), 33 deletions(-) diff --git a/BUILD.md b/BUILD.md index 487bcb0eb1..b8bc1cd14c 100644 --- a/BUILD.md +++ b/BUILD.md @@ -4,7 +4,7 @@ Dependencies * [Qt](http://qt-project.org/downloads) ~> 5.2.0 * [zLib](http://www.zlib.net/) ~> 1.2.8 * [glm](http://glm.g-truc.net/0.9.5/index.html) ~> 0.9.5.2 -* [qxmpp](https://code.google.com/p/qxmpp/) ~> 0.7.6 +* [qxmpp](https://github.com/qxmpp-project/qxmpp/) ~> 0.7.6 * [GnuTLS](http://gnutls.org/download.html) ~> 3.2.12 * IMPORTANT: GnuTLS 3.2.12 is critical to avoid a security vulnerability. @@ -83,17 +83,41 @@ If the build completes successfully, you will have built targets for all compone Windows === -Currently building on Windows has only been tested on Windows SDK 7.1 with Visual Studio C++ 2010 Express. +####Visual Studio + +Currently building on Windows has been tested using the following compilers: +* Visual Studio C++ 2010 Express +* Visual Studio 2013 + +(If anyone can test using Visual Studio 2013 Express then please update this document) + +#####Windows SDK 7.1 + +Whichever version of Visual Studio you use, first install [Microsoft Windows SDK for Windows 7 and .NET Framework 4](http://www.microsoft.com/en-us/download/details.aspx?id=8279). + +#####Visual Studio C++ 2010 Express Visual Studio C++ 2010 Express can be downloaded [here](http://www.visualstudio.com/en-us/downloads#d-2010-express). The following patches/service packs are also required: -* [Windows SDK 7.1/.NET 4 Framework](http://www.microsoft.com/en-us/download/details.aspx?id=8279) * [VS2010 SP1](http://www.microsoft.com/en-us/download/details.aspx?id=23691) * [VS2010 SP1 Compiler Update](http://www.microsoft.com/en-us/download/details.aspx?id=4422) +Some of the build instructions will ask you to start a Visual Studio Command Prompt. You should have a shortcut in your Start menu called "Open Visual Studio Command Prompt (2010)" which will do so. + +#####Visual Studio 2013 + +This product must be purchased separately. + +Visual Studio 2013 doesn't have a shortcut to start a Visual Studio Command Prompt. Instead, start a regular command prompt and then run: + + "%VS120COMNTOOLS%\vsvars32.bat" + ####Qt -You can use the online installer, or the offline installer. If you use the offline installer, be sure to select the "OpenGL" version. +You can use the online installer or the offline installer. If you use the offline installer, be sure to select the "OpenGL" version. + +NOTE: Qt does not support 64-bit builds on Windows 7, so you must use the 32-bit version of libraries for interface.exe to run. The 32-bit version of the static library is the one linked by our CMake find modules. + * Download the online installer [here](http://qt-project.org/downloads) * When it asks you to select components, ONLY select the following: * Qt > Qt 5.2.0 > **msvc2010 32-bit OpenGL** @@ -101,67 +125,135 @@ You can use the online installer, or the offline installer. If you use the offli * Download the offline installer [here](http://download.qt-project.org/official_releases/qt/5.2/5.2.0/qt-windows-opensource-5.2.0-msvc2010_opengl-x86-offline.exe) Once Qt is installed, you need to manually configure the following: -* Make sure the Qt runtime DLLs are loadable (You could add the Qt\5.2.0\msvc2010_opengl\bin\ directory to your path.) - You must do this before you attempt to build because some tools for the build depend on Qt. -* Set the QT_CMAKE_PREFIX_PATH environment variable to your Qt\5.2.0\msvc2010_opengl directory - -####zLib -NOTE: zLib should configure itself correctly on install. However, sometimes zLib doesn't properly detect system components and fails to configure itself correctly. When it fails, it will not correctly set the #if HAVE_UNISTD_H at line 287 of zconf.h to #if 0... if it fails, you're build will have errors in the voxels target. You can correct this by setting the #if to 0 instead of 1, since Windows does not have unistd.h. +* Make sure the Qt runtime DLLs are loadable. You must do this before you attempt to build because some tools for the build depend on Qt. E.g., add to the PATH: `Qt\5.2.0\msvc2010_opengl\bin\`. +* Set the QT_CMAKE_PREFIX_PATH environment variable to your `Qt\5.2.0\msvc2010_opengl` directory. ####External Libraries CMake will need to know where the headers and libraries for required external dependencies are. -If you installed zLib using the installer, the Cmake find module for zLib should locate it on your system. - -The recommended route for CMake to find the other external dependencies is to place all of the dependencies in one folder and set one ENV variable - HIFI_LIB_DIR. That ENV variable should point to a directory with the following structure: +The recommended route for CMake to find the external dependencies is to place all of the dependencies in one folder and set one ENV variable - HIFI_LIB_DIR. That ENV variable should point to a directory with the following structure: root_lib_dir - -> glm - -> glm - -> glm.hpp + -> freeglut + -> bin + -> include + -> lib -> glew -> bin -> include -> lib - -> freeglut + -> glm + -> glm + -> glm.hpp + -> gnutls -> bin -> include -> lib -> qxmpp -> include -> lib - -> gnutls - -> bin - -> include - -> lib + -> zlib + -> include + -> lib + -> test -For many of the external libraries where precompiled binaries are readily available you should be able to simply copy the extracted folder that you get from the download links provided at the top of the guide. Otherwise you may need to build from source and install the built product to this directory. The `root_lib_dir` in the above example can be wherever you choose on your system - as long as the environment variable HIFI_LIB_DIR is set to it. +For many of the external libraries where precompiled binaries are readily available you should be able to simply copy the extracted folder that you get from the download links provided at the top of the guide. Otherwise you may need to build from source and install the built product to this directory. The `root_lib_dir` in the above example can be wherever you choose on your system - as long as the environment variable HIFI_LIB_DIR is set to it. From here on, whenever you see %HIFI_LIB_DIR% you should substitute the directory that you chose. -*NOTE: Be careful with glm. For the folder other libraries would normally call 'include', the folder containing the headers, glm opts to use 'glm'. You will have a glm folder nested inside the top-level glm folder.* +As with the Qt libraries, you will need to make sure that directories containing DLL'S are in your path. Where possible, you can use static builds of the external dependencies to avoid this requirement. -*NOTE: Qt does not support 64-bit builds on Windows 7, so you must use the 32-bit version of libraries for interface.exe to run. The 32-bit version of the static library is the one linked by our CMake find modules* +#### OpenSSL -##### DLLs -As with the Qt libraries, you will need to make sure the directories containing dynamically-linked libraries is in your path. +QT will use OpenSSL if it's available, but it doesn't install it, so you must install it separately. -For example, for a dynamically linked build of freeglut, the directory to add to your path in which the DLL is found is `FREEGLUT_DIR/bin`. Where possible, you can use static builds of the external dependencies to avoid this requirement. +Your system may already have several versions of the OpenSSL DLL's (ssleay32.dll, libeay32.dll) lying around, but they may be the wrong version. If these DLL's are in the PATH then QT will try to use them, and if they're the wrong version then you will see the following errors in the console: + + QSslSocket: cannot resolve TLSv1_1_client_method + QSslSocket: cannot resolve TLSv1_2_client_method + QSslSocket: cannot resolve TLSv1_1_server_method + QSslSocket: cannot resolve TLSv1_2_server_method + QSslSocket: cannot resolve SSL_select_next_proto + QSslSocket: cannot resolve SSL_CTX_set_next_proto_select_cb + QSslSocket: cannot resolve SSL_get0_next_proto_negotiated + +To prevent these problems, install OpenSSL yourself. Download the following binary packages [from this website](http://slproweb.com/products/Win32OpenSSL.html): +* Visual C++ 2008 Redistributables +* Win32 OpenSSL v1.0.1h + +Install OpenSSL into the Windows system directory, to make sure that QT uses the version that you've just installed, and not some other version. + +#### Zlib + +Download the compiled DLL from the [zlib website](http://www.zlib.net/). Extract to %HIFI_LIB_DIR%\zlib. + +Add the following environment variables (remember to substitute your own directory for %HIFI_LIB_DIR%): + + ZLIB_LIBRARY=%HIFI_LIB_DIR%\zlib\lib\zdll.lib + ZLIB_INCLUDE_DIR=%HIFI_LIB_DIR%\zlib\include + +Add to the PATH: `%HIFI_LIB_DIR%\zlib` + +Important! This should be added at the beginning of the path, not the end. That's because your +system likely has many copies of zlib1.dll, and you want High Fidelity to use the correct version. If High Fidelity picks up the wrong zlib1.dll then it might be unable to use it, and that would cause it to fail to start, showing only the cryptic error "The application was unable to start correctly: 0xc0000022". + +#### freeglut + +Download the binary package: `freeglut-MSVC-2.8.1-1.mp.zip`. Extract to %HIFI_LIB_DIR%\freeglut. + +Add to the PATH: `%HIFI_LIB_DIR%\freeglut\bin` + +#### GLEW + +Download the binary package: `glew-1.10.0-win32.zip`. Extract to %HIFI_LIB_DIR%\glew (you'll need to rename the default directory name). + +Add to the PATH: `%HIFI_LIB_DIR%\glew\bin\Release\Win32` + +#### GLM + +This package contains only headers, so there's nothing to add to the PATH. + +Be careful with glm. For the folder other libraries would normally call 'include', the folder containing the headers, glm opts to use 'glm'. You will have a glm folder nested inside the top-level glm folder. + +#### GnuTLS -#####GnuTLS You can get a precompiled version of GnuTLS for Windows [here](http://gnutls.org/download.html). -To use GnuTLS with Visual Studio, you will need to create `libgnutls-28.lib`, the import library for Visual Studio projects. this is done using the `lib` command in the `bin` folder of your GnuTLS download. Run the following in a Visual Studio Command Prompt (found in the tools menu of Visual Studio). +To use GnuTLS with Visual Studio, you will need to create `libgnutls-28.lib`, the import library for Visual Studio projects. This is done using the `lib` command in the `bin` folder of your GnuTLS download. Start a Visual Studio Command Prompt, and then run: - $GNUTLS_DIR\bin> lib /def:libgnutls-28.def + cd %HIFI_LIB_DIR%\gnutls\bin + lib /def:libgnutls-28.def + copy libgnutls-28.lib ..\lib -This will create `libgnutls-28.lib` in the `bin` folder. Copy that file to the `lib` sub-folder of your GnuTLS folder, and the Cmake FindGnuTLS module in this repo will find it during the Cmake run. +The Cmake FindGnuTLS module will now find libgnutls-28.lib during the Cmake run. -####Building in Visual Studio +Add to the PATH: `%HIFI_LIB_DIR%\gnutls\bin` + +#### qxmpp + +Download a source-code release from the [qxmpp GitHub page](https://github.com/qxmpp-project/qxmpp/releases). + +Start a Visual Studio Command Prompt. + + mkdir %HIFI_LIB_DIR%\build + tar xfz qxmpp-0.7.6.tar.gz -C %HIFI_LIB_DIR%\build + cd %HIFI_LIB_DIR%\build\qxmpp-0.7.6 + qmake PREFIX=%HIFI_LIB_DIR%\qxmpp # This creates "Makefile" + nmake + nmake install + +Add to the PATH: `%HIFI_LIB_DIR%\qxmpp\lib` + +#### Build High Fidelity using Visual Studio Follow the same build steps from the CMake section, but pass a different generator to CMake. - cmake .. -G "Visual Studio 10" + cmake .. -DZLIB_LIBRARY=%ZLIB_LIBRARY% -DZLIB_INCLUDE_DIR=%ZLIB_INCLUDE_DIR% -G "Visual Studio 10" + +If you're using Visual Studio 2013 then pass "Visual Studio 12" instead of "Visual Studio 10" (yes, 12, not 13). + +Open %HIFI_DIR%\build\hifi.sln and compile. ####Running Interface -If you need to debug Interface, you can run interface from within Visual Studio (see the section below). You can also run Interface by launching it from command line or File Explorer from $YOUR_HIFI_PATH\build\interface\Debug\interface.exe +If you need to debug Interface, you can run interface from within Visual Studio (see the section below). You can also run Interface by launching it from command line or File Explorer from %HIFI_DIR%\build\interface\Debug\interface.exe ####Debugging Interface * In the Solution Explorer, right click interface and click Set as StartUp Project From 8c4e365958654580132ed955ae22bb1cdbc72aed Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 18 Jun 2014 09:28:42 -0700 Subject: [PATCH 40/55] added forgotten i++ in sendNackPackets() plus minor style fixes --- .../octree/OctreeInboundPacketProcessor.cpp | 71 +++++++------------ .../src/octree/OctreeInboundPacketProcessor.h | 6 +- .../src/ReceivedPacketProcessor.cpp | 5 +- .../networking/src/ReceivedPacketProcessor.h | 12 ++++ libraries/octree/src/OctreeSceneStats.cpp | 11 ++- 5 files changed, 50 insertions(+), 55 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 91a5c62752..6bc085df4b 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -17,6 +17,7 @@ #include "OctreeInboundPacketProcessor.h" static QUuid DEFAULT_NODE_ID_REF; +const quint64 TOO_LONG_SINCE_LAST_NACK = 1 * USECS_PER_SECOND; OctreeInboundPacketProcessor::OctreeInboundPacketProcessor(OctreeServer* myServer) : _myServer(myServer), @@ -41,46 +42,33 @@ void OctreeInboundPacketProcessor::resetStats() { _singleSenderStats.clear(); } -bool OctreeInboundPacketProcessor::process() { - - const quint64 TOO_LONG_SINCE_LAST_NACK = 1 * USECS_PER_SECOND; +unsigned long OctreeInboundPacketProcessor::getMaxWait() const { + // calculate time until next sendNackPackets() + quint64 nextNackTime = _lastNackTime + TOO_LONG_SINCE_LAST_NACK; quint64 now = usecTimestampNow(); - - if (_packets.size() == 0) { - // calculate time until next sendNackPackets() - quint64 nextNackTime = _lastNackTime + TOO_LONG_SINCE_LAST_NACK; - quint64 now = usecTimestampNow(); - if (now >= nextNackTime) { - // send nacks if we're already past time to send it - _lastNackTime = now; - sendNackPackets(); - } - else { - // otherwise, wait until the next nack time or until a packet arrives - quint64 waitTimeMsecs = (nextNackTime - now) / USECS_PER_MSEC + 1; - _waitingOnPacketsMutex.lock(); - _hasPackets.wait(&_waitingOnPacketsMutex, waitTimeMsecs); - _waitingOnPacketsMutex.unlock(); - } + if (now >= nextNackTime) { + return 0; } - while (_packets.size() > 0) { - lock(); // lock to make sure nothing changes on us - NetworkPacket& packet = _packets.front(); // get the oldest packet - NetworkPacket temporary = packet; // make a copy of the packet in case the vector is resized on us - _packets.erase(_packets.begin()); // remove the oldest packet - _nodePacketCounts[temporary.getNode()->getUUID()]--; - unlock(); // let others add to the packets - processPacket(temporary.getNode(), temporary.getByteArray()); // process our temporary copy - - // if it's time to send nacks, send them. - if (usecTimestampNow() - _lastNackTime >= TOO_LONG_SINCE_LAST_NACK) { - _lastNackTime = now; - sendNackPackets(); - } - } - return isStillRunning(); // keep running till they terminate us + return (nextNackTime - now) / USECS_PER_MSEC + 1; } +void OctreeInboundPacketProcessor::preProcess() { + // check if it's time to send a nack. If yes, do so + quint64 now = usecTimestampNow(); + if (now - _lastNackTime >= TOO_LONG_SINCE_LAST_NACK) { + _lastNackTime = now; + sendNackPackets(); + } +} + +void OctreeInboundPacketProcessor::midProcess() { + // check if it's time to send a nack. If yes, do so + quint64 now = usecTimestampNow(); + if (now - _lastNackTime >= TOO_LONG_SINCE_LAST_NACK) { + _lastNackTime = now; + sendNackPackets(); + } +} void OctreeInboundPacketProcessor::processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet) { @@ -211,6 +199,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { // if there are packets from _node that are waiting to be processed, // don't send a NACK since the missing packets may be among those waiting packets. if (hasPacketsToProcessFrom(nodeUUID)) { + i++; continue; } @@ -279,7 +268,6 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { const int UINT16_RANGE = UINT16_MAX + 1; - const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work const int MAX_MISSING_SEQUENCE_SIZE = 100; @@ -287,9 +275,7 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, if (incomingSequence == expectedSequence) { // on time _incomingLastSequence = incomingSequence; - } - else { // out of order - + } else { // out of order int incoming = (int)incomingSequence; int expected = (int)expectedSequence; @@ -312,17 +298,13 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, // now that rollover has been corrected for (if it occurred), incoming and expected can be // compared to each other directly, though one of them might be negative - if (incoming > expected) { // early - // add all sequence numbers that were skipped to the missing sequence numbers list for (int missingSequence = expected; missingSequence < incoming; missingSequence++) { _missingSequenceNumbers.insert(missingSequence < 0 ? missingSequence + UINT16_RANGE : missingSequence); } _incomingLastSequence = incomingSequence; - } else { // late - // remove this from missing sequence number if it's in there _missingSequenceNumbers.remove(incomingSequence); @@ -333,7 +315,6 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, // prune missing sequence list if it gets too big; sequence numbers that are older than MAX_REASONABLE_SEQUENCE_GAP // will be removed. if (_missingSequenceNumbers.size() > MAX_MISSING_SEQUENCE_SIZE) { - // some older sequence numbers may be from before a rollover point; this must be handled. // some sequence numbers in this list may be larger than _incomingLastSequence, indicating that they were received // before the most recent rollover. diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.h b/assignment-client/src/octree/OctreeInboundPacketProcessor.h index d3b3b80208..46a57205cb 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.h +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.h @@ -77,9 +77,11 @@ protected: virtual void processPacket(const SharedNodePointer& sendingNode, const QByteArray& packet); - virtual bool process(); + virtual unsigned long getMaxWait() const; + virtual void preProcess(); + virtual void midProcess(); -public: +private: int sendNackPackets(); private: diff --git a/libraries/networking/src/ReceivedPacketProcessor.cpp b/libraries/networking/src/ReceivedPacketProcessor.cpp index 3ef518bbc2..59e1ecd456 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.cpp +++ b/libraries/networking/src/ReceivedPacketProcessor.cpp @@ -35,9 +35,10 @@ bool ReceivedPacketProcessor::process() { if (_packets.size() == 0) { _waitingOnPacketsMutex.lock(); - _hasPackets.wait(&_waitingOnPacketsMutex); + _hasPackets.wait(&_waitingOnPacketsMutex, getMaxWait()); _waitingOnPacketsMutex.unlock(); } + preProcess(); while (_packets.size() > 0) { lock(); // lock to make sure nothing changes on us NetworkPacket& packet = _packets.front(); // get the oldest packet @@ -46,7 +47,9 @@ bool ReceivedPacketProcessor::process() { _nodePacketCounts[temporary.getNode()->getUUID()]--; unlock(); // let others add to the packets processPacket(temporary.getNode(), temporary.getByteArray()); // process our temporary copy + midProcess(); } + postProcess(); return isStillRunning(); // keep running till they terminate us } diff --git a/libraries/networking/src/ReceivedPacketProcessor.h b/libraries/networking/src/ReceivedPacketProcessor.h index 4322c87910..607f9e54c2 100644 --- a/libraries/networking/src/ReceivedPacketProcessor.h +++ b/libraries/networking/src/ReceivedPacketProcessor.h @@ -63,6 +63,18 @@ protected: /// Implements generic processing behavior for this thread. virtual bool process(); + /// Determines the timeout of the wait when there are no packets to process. Default value means no timeout + virtual unsigned long getMaxWait() const { return ULONG_MAX; } + + /// Override to do work before the packets processing loop. Default does nothing. + virtual void preProcess() { } + + /// Override to do work inside the packet processing loop after a packet is processed. Default does nothing. + virtual void midProcess() { } + + /// Override to do work after the packets processing loop. Default does nothing. + virtual void postProcess() { } + virtual void terminating(); protected: diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index e99096dd5c..1ed078ed14 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -843,7 +843,7 @@ const char* OctreeSceneStats::getItemValue(Item item) { void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, bool wasStatsPacket, int nodeClockSkewUsec) { - const bool wantExtraDebugging = true; + const bool wantExtraDebugging = false; int numBytesPacketHeader = numBytesForPacketHeader(packet); const unsigned char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; @@ -891,8 +891,7 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, if (wantExtraDebugging) { qDebug() << "last packet duplicate got:" << sequence << "_incomingLastSequence:" << _incomingLastSequence; } - } - else { + } else { if (sequence != expected) { if (wantExtraDebugging) { qDebug() << "out of order... got:" << sequence << "expected:" << expected; @@ -967,8 +966,7 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, // only bump the last sequence if it was greater than our expected sequence, this will keep us from // accidentally going backwards when an out of order (recovered) packet comes in - } - else { // sequenceInt > expectedInt + } else { // sequenceInt > expectedInt // if no rollover between them: sequenceInt, expectedInt are both in range [0, UINT16_RANGE-1] // if rollover between them: sequenceInt in [0, UINT16_RANGE-1], expectedInt in [-UINT16_RANGE, -1] @@ -992,8 +990,7 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, _incomingLastSequence = sequence; } - } - else { // sequence = expected + } else { // sequence = expected _incomingLastSequence = sequence; } From b63c88f42c811adabe6c3148ee8a5238ec6347b2 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 18 Jun 2014 09:38:39 -0700 Subject: [PATCH 41/55] removed more spaces --- libraries/octree/src/OctreeEditPacketSender.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/libraries/octree/src/OctreeEditPacketSender.cpp b/libraries/octree/src/OctreeEditPacketSender.cpp index f49fe9f22f..43e253b2da 100644 --- a/libraries/octree/src/OctreeEditPacketSender.cpp +++ b/libraries/octree/src/OctreeEditPacketSender.cpp @@ -97,14 +97,12 @@ void OctreeEditPacketSender::queuePacketToNode(const QUuid& nodeUUID, const unsi ((node->getUUID() == nodeUUID) || (nodeUUID.isNull()))) { if (node->getActiveSocket()) { QByteArray packet(reinterpret_cast(buffer), length); - queuePacketForSending(node, packet); // extract sequence number and add packet to history int numBytesPacketHeader = numBytesForPacketHeader(packet); const char* dataAt = reinterpret_cast(packet.data()) + numBytesPacketHeader; unsigned short int sequence = *((unsigned short int*)dataAt); - _sentPacketHistories[nodeUUID].packetSent(sequence, packet); // debugging output... @@ -303,13 +301,11 @@ void OctreeEditPacketSender::releaseQueuedMessages() { void OctreeEditPacketSender::releaseQueuedPacket(EditPacketBuffer& packetBuffer) { _releaseQueuedPacketMutex.lock(); - if (packetBuffer._currentSize > 0 && packetBuffer._currentType != PacketTypeUnknown) { queuePacketToNode(packetBuffer._nodeUUID, &packetBuffer._currentBuffer[0], packetBuffer._currentSize); packetBuffer._currentSize = 0; packetBuffer._currentType = PacketTypeUnknown; } - _releaseQueuedPacketMutex.unlock(); } From 54f32d331e258a801d041a1e436ff42c044c4f96 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 18 Jun 2014 10:20:19 -0700 Subject: [PATCH 42/55] replaced UINT16_MAX with std::numeric_limits::max --- libraries/networking/src/SentPacketHistory.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/SentPacketHistory.cpp b/libraries/networking/src/SentPacketHistory.cpp index 1e1157ba71..841b5e909c 100644 --- a/libraries/networking/src/SentPacketHistory.cpp +++ b/libraries/networking/src/SentPacketHistory.cpp @@ -8,6 +8,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include #include "SentPacketHistory.h" #include @@ -15,7 +16,7 @@ SentPacketHistory::SentPacketHistory(int size) : _sentPackets(size), _newestPacketAt(0), _numExistingPackets(0), - _newestSequenceNumber(UINT16_MAX) + _newestSequenceNumber(std::numeric_limits::max()) { } @@ -42,7 +43,7 @@ void SentPacketHistory::packetSent(uint16_t sequenceNumber, const QByteArray& pa const QByteArray* SentPacketHistory::getPacket(uint16_t sequenceNumber) const { - const int UINT16_RANGE = UINT16_MAX + 1; + const int UINT16_RANGE = std::numeric_limits::max() + 1; // if sequenceNumber > _newestSequenceNumber, assume sequenceNumber is from before the most recent rollover // correct the diff so that it correctly represents how far back in the history sequenceNumber is From 7cef5eeeec0cf31d05bd3905451d7251aea20ba5 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 18 Jun 2014 10:27:02 -0700 Subject: [PATCH 43/55] replaced UINT16_MAX at 2 other places --- assignment-client/src/octree/OctreeInboundPacketProcessor.cpp | 3 ++- libraries/octree/src/OctreeSceneStats.cpp | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 6bc085df4b..b13f3d7096 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -9,6 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include #include #include @@ -267,7 +268,7 @@ SingleSenderStats::SingleSenderStats() void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, quint64 transitTime, int editsInPacket, quint64 processTime, quint64 lockWaitTime) { - const int UINT16_RANGE = UINT16_MAX + 1; + const int UINT16_RANGE = std::numeric_limits::max() + 1; const int MAX_REASONABLE_SEQUENCE_GAP = 1000; // this must be less than UINT16_RANGE / 2 for rollover handling to work const int MAX_MISSING_SEQUENCE_SIZE = 100; diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 1ed078ed14..d037ec79ad 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -9,6 +9,7 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +#include #include #include @@ -877,7 +878,7 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, return; // ignore any packets that are unreasonable } - const int UINT16_RANGE = UINT16_MAX + 1; + const int UINT16_RANGE = std::numeric_limits::max() + 1; // determine our expected sequence number... handle rollover appropriately OCTREE_PACKET_SEQUENCE expected = _incomingPacket > 0 ? _incomingLastSequence + (quint16)1 : sequence; From 26075d9d26f084d3158a246ef04e18c6db8b784f Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Wed, 18 Jun 2014 11:29:56 -0700 Subject: [PATCH 44/55] add support for changing the model URL of a model --- examples/editModels.js | 54 +++++++++++++++++++--- interface/src/models/ModelTreeRenderer.cpp | 34 ++++++++++++-- interface/src/models/ModelTreeRenderer.h | 4 ++ libraries/octree/src/OctreeRenderer.h | 2 +- 4 files changed, 83 insertions(+), 11 deletions(-) diff --git a/examples/editModels.js b/examples/editModels.js index 3cd727570c..66062d091d 100644 --- a/examples/editModels.js +++ b/examples/editModels.js @@ -706,8 +706,8 @@ function rayPlaneIntersection(pickRay, point, normal) { function Tooltip() { this.x = 285; this.y = 115; - this.width = 110; - this.height = 115 ; + this.width = 500; + this.height = 145 ; this.margin = 5; this.decimals = 3; @@ -735,6 +735,9 @@ function Tooltip() { text += "yaw: " + angles.y.toFixed(this.decimals) + "\n" text += "roll: " + angles.z.toFixed(this.decimals) + "\n" text += "Scale: " + 2 * properties.radius.toFixed(this.decimals) + "\n" + text += "ID: " + properties.id + "\n" + text += "model url: " + properties.modelURL + "\n" + text += "animation url: " + properties.animationURL + "\n" Overlays.editOverlay(this.textOverlay, { text: text }); } @@ -1004,9 +1007,11 @@ var modelMenuAddedDelete = false; function setupModelMenus() { print("setupModelMenus()"); // add our menuitems + Menu.addMenuItem({ menuName: "Edit", menuItemName: "Models", isSeparator: true, beforeItem: "Physics" }); + Menu.addMenuItem({ menuName: "Edit", menuItemName: "Edit Properties...", + shortcutKeyEvent: { text: "`" }, afterItem: "Models" }); if (!Menu.menuItemExists("Edit","Delete")) { print("no delete... adding ours"); - Menu.addMenuItem({ menuName: "Edit", menuItemName: "Models", isSeparator: true, beforeItem: "Physics" }); Menu.addMenuItem({ menuName: "Edit", menuItemName: "Delete", shortcutKeyEvent: { text: "backspace" }, afterItem: "Models" }); modelMenuAddedDelete = true; @@ -1016,9 +1021,10 @@ function setupModelMenus() { } function cleanupModelMenus() { + Menu.removeSeparator("Edit", "Models"); + Menu.removeMenuItem("Edit", "Edit Properties..."); if (modelMenuAddedDelete) { // delete our menuitems - Menu.removeSeparator("Edit", "Models"); Menu.removeMenuItem("Edit", "Delete"); } } @@ -1039,7 +1045,8 @@ Controller.mouseMoveEvent.connect(mouseMoveEvent); Controller.mouseReleaseEvent.connect(mouseReleaseEvent); setupModelMenus(); -Menu.menuItemEvent.connect(function(menuItem){ + +function handeMenuEvent(menuItem){ print("menuItemEvent() in JS... menuItem=" + menuItem); if (menuItem == "Delete") { if (leftController.grabbing) { @@ -1057,8 +1064,36 @@ Menu.menuItemEvent.connect(function(menuItem){ } else { print(" Delete Model.... not holding..."); } + } else if (menuItem == "Edit Properties...") { + var editModelID = -1; + if (leftController.grabbing) { + print(" Edit Properties.... leftController.modelID="+ leftController.modelID); + editModelID = leftController.modelID; + } else if (rightController.grabbing) { + print(" Edit Properties.... rightController.modelID="+ rightController.modelID); + editModelID = rightController.modelID; + } else if (modelSelected) { + print(" Edit Properties.... selectedModelID="+ selectedModelID); + editModelID = selectedModelID; + } else { + print(" Edit Properties.... not holding..."); + } + if (editModelID != -1) { + print(" Edit Properties.... about to edit properties..."); + var propertyName = Window.prompt("Which property would you like to change?", "modelURL"); + var properties = Models.getModelProperties(editModelID); + var oldValue = properties[propertyName]; + var newValue = Window.prompt("New value for: " + propertyName, oldValue); + if (newValue != NULL) { + properties[propertyName] = newValue; + Models.editModel(editModelID, properties); + } + } + tooltip.show(false); } -}); +} +Menu.menuItemEvent.connect(handeMenuEvent); + // handling of inspect.js concurrence @@ -1092,4 +1127,11 @@ Controller.keyReleaseEvent.connect(function(event) { xIsPressed = false; somethingChanged = true; } + // since sometimes our menu shortcut keys don't work, trap our menu items here also and fire the appropriate menu items + if (event.text == "`") { + handeMenuEvent("Edit Properties..."); + } + if (event.text == "BACKSPACE") { + handeMenuEvent("Delete"); + } }); \ No newline at end of file diff --git a/interface/src/models/ModelTreeRenderer.cpp b/interface/src/models/ModelTreeRenderer.cpp index b20bdce8f5..9c4b08fd99 100644 --- a/interface/src/models/ModelTreeRenderer.cpp +++ b/interface/src/models/ModelTreeRenderer.cpp @@ -22,6 +22,17 @@ ModelTreeRenderer::ModelTreeRenderer() : } ModelTreeRenderer::~ModelTreeRenderer() { + clearModelsCache(); +} + +void ModelTreeRenderer::clear() { + OctreeRenderer::clear(); + clearModelsCache(); +} + +void ModelTreeRenderer::clearModelsCache() { + qDebug() << "ModelTreeRenderer::clearModelsCache()..."; + // delete the models in _knownModelsItemModels foreach(Model* model, _knownModelsItemModels) { delete model; @@ -71,24 +82,39 @@ Model* ModelTreeRenderer::getModel(const ModelItem& modelItem) { if (modelItem.isKnownID()) { if (_knownModelsItemModels.find(modelItem.getID()) != _knownModelsItemModels.end()) { model = _knownModelsItemModels[modelItem.getID()]; - } else { - + if (QUrl(modelItem.getModelURL()) != model->getURL()) { + delete model; // delete the old model... + model = NULL; + _knownModelsItemModels.remove(modelItem.getID()); + } + } + + // if we don't have a model... + if (!model) { // Make sure we only create new models on the thread that owns the ModelTreeRenderer if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "getModel", Qt::BlockingQueuedConnection, Q_RETURN_ARG(Model*, model), Q_ARG(const ModelItem&, modelItem)); return model; } - + model = new Model(); model->init(); model->setURL(QUrl(modelItem.getModelURL())); _knownModelsItemModels[modelItem.getID()] = model; } + } else { if (_unknownModelsItemModels.find(modelItem.getCreatorTokenID()) != _unknownModelsItemModels.end()) { model = _unknownModelsItemModels[modelItem.getCreatorTokenID()]; - } else { + if (QUrl(modelItem.getModelURL()) != model->getURL()) { + delete model; // delete the old model... + model = NULL; + _unknownModelsItemModels.remove(modelItem.getID()); + } + } + + if (!model) { // Make sure we only create new models on the thread that owns the ModelTreeRenderer if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "getModel", Qt::BlockingQueuedConnection, diff --git a/interface/src/models/ModelTreeRenderer.h b/interface/src/models/ModelTreeRenderer.h index 2d418b1a25..b6df71565d 100644 --- a/interface/src/models/ModelTreeRenderer.h +++ b/interface/src/models/ModelTreeRenderer.h @@ -51,7 +51,11 @@ public: virtual const FBXGeometry* getGeometryForModel(const ModelItem& modelItem); + /// clears the tree + virtual void clear(); + protected: + void clearModelsCache(); Model* getModel(const ModelItem& modelItem); QMap _knownModelsItemModels; QMap _unknownModelsItemModels; diff --git a/libraries/octree/src/OctreeRenderer.h b/libraries/octree/src/OctreeRenderer.h index 18e68e26aa..d61ed3afce 100644 --- a/libraries/octree/src/OctreeRenderer.h +++ b/libraries/octree/src/OctreeRenderer.h @@ -62,7 +62,7 @@ public: static bool renderOperation(OctreeElement* element, void* extraData); /// clears the tree - void clear(); + virtual void clear(); protected: Octree* _tree; bool _managedTree; From 52de729d60429d99c2ff9c5ea97ae710bb4bcb7c Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 11:31:58 -0700 Subject: [PATCH 45/55] Add virtual destructor to get rid of Xcode warning. --- libraries/metavoxels/src/Bitstream.cpp | 3 +++ libraries/metavoxels/src/Bitstream.h | 1 + 2 files changed, 4 insertions(+) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 80aa07b026..e5cf34054d 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1661,6 +1661,9 @@ ObjectStreamer::ObjectStreamer(const QMetaObject* metaObject) : _metaObject(metaObject) { } +ObjectStreamer::~ObjectStreamer() { +} + const QVector& ObjectStreamer::getProperties() const { static QVector emptyProperties; return emptyProperties; diff --git a/libraries/metavoxels/src/Bitstream.h b/libraries/metavoxels/src/Bitstream.h index 1589473b0e..5dc2fcc35d 100644 --- a/libraries/metavoxels/src/Bitstream.h +++ b/libraries/metavoxels/src/Bitstream.h @@ -773,6 +773,7 @@ class ObjectStreamer { public: ObjectStreamer(const QMetaObject* metaObject); + virtual ~ObjectStreamer(); const QMetaObject* getMetaObject() const { return _metaObject; } const ObjectStreamerPointer& getSelf() const { return _self; } From d78ed66616fc6836a0efa935d145f369c6e0191d Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 18 Jun 2014 11:37:54 -0700 Subject: [PATCH 46/55] removed spaces before comments in OctreeInboundPacketProcessor --- .../src/octree/OctreeInboundPacketProcessor.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index b13f3d7096..78ab9259fd 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -274,9 +274,9 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, unsigned short int expectedSequence = _totalPackets == 0 ? incomingSequence : _incomingLastSequence + (unsigned short int)1; - if (incomingSequence == expectedSequence) { // on time + if (incomingSequence == expectedSequence) { // on time _incomingLastSequence = incomingSequence; - } else { // out of order + } else { // out of order int incoming = (int)incomingSequence; int expected = (int)expectedSequence; @@ -299,13 +299,13 @@ void SingleSenderStats::trackInboundPacket(unsigned short int incomingSequence, // now that rollover has been corrected for (if it occurred), incoming and expected can be // compared to each other directly, though one of them might be negative - if (incoming > expected) { // early + if (incoming > expected) { // early // add all sequence numbers that were skipped to the missing sequence numbers list for (int missingSequence = expected; missingSequence < incoming; missingSequence++) { _missingSequenceNumbers.insert(missingSequence < 0 ? missingSequence + UINT16_RANGE : missingSequence); } _incomingLastSequence = incomingSequence; - } else { // late + } else { // late // remove this from missing sequence number if it's in there _missingSequenceNumbers.remove(incomingSequence); From e0689b14840a2917b88eed3a3e26c347de1c2a7f Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 12:04:41 -0700 Subject: [PATCH 47/55] Fixed compile error on Xcode. --- libraries/metavoxels/src/Bitstream.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/metavoxels/src/Bitstream.cpp b/libraries/metavoxels/src/Bitstream.cpp index 297dfeb079..44342abe33 100644 --- a/libraries/metavoxels/src/Bitstream.cpp +++ b/libraries/metavoxels/src/Bitstream.cpp @@ -1901,9 +1901,9 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge int highestValue = 0; foreach (const QJsonValue& value, type.value("values").toArray()) { QJsonObject object = value.toObject(); - int value = object.value("value").toInt(); - highestValue = qMax(value, highestValue); - values.append(NameIntPair(object.value("key").toString().toLatin1(), value)); + int intValue = object.value("value").toInt(); + highestValue = qMax(intValue, highestValue); + values.append(NameIntPair(object.value("key").toString().toLatin1(), intValue)); } streamer = TypeStreamerPointer(new GenericEnumTypeStreamer(latinName, values, getBitsForHighestValue(highestValue), QByteArray())); @@ -1944,13 +1944,13 @@ JSONReader::JSONReader(const QJsonDocument& document, Bitstream::GenericsMode ge bool matches = (array.size() == metaEnum.keyCount()); foreach (const QJsonValue& value, array) { QJsonObject object = value.toObject(); - int value = object.value("value").toInt(); - highestValue = qMax(value, highestValue); + int intValue = object.value("value").toInt(); + highestValue = qMax(intValue, highestValue); int mapping = metaEnum.keyToValue(object.value("key").toString().toLatin1()); if (mapping != -1) { - mappings.insert(value, mapping); + mappings.insert(intValue, mapping); } - matches &= (value == mapping); + matches &= (intValue == mapping); } // if everything matches our built-in enum, we can use that, which will be faster if (matches) { From ed32ca721a19e0f1fd4b44f5ade96f36834e2000 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Wed, 18 Jun 2014 12:19:01 -0700 Subject: [PATCH 48/55] fix bug --- examples/editModels.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/editModels.js b/examples/editModels.js index 9278f32490..55fcfa6be8 100644 --- a/examples/editModels.js +++ b/examples/editModels.js @@ -1099,7 +1099,7 @@ function handeMenuEvent(menuItem){ var properties = Models.getModelProperties(editModelID); var oldValue = properties[propertyName]; var newValue = Window.prompt("New value for: " + propertyName, oldValue); - if (newValue != NULL) { + if (newValue != "") { properties[propertyName] = newValue; Models.editModel(editModelID, properties); } From eea45e98d7895494eec70e5700f9e0713e02264d Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 12:22:45 -0700 Subject: [PATCH 49/55] Fix to include GLM. --- tools/bitstream2json/CMakeLists.txt | 6 ++++++ tools/json2bitstream/CMakeLists.txt | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/tools/bitstream2json/CMakeLists.txt b/tools/bitstream2json/CMakeLists.txt index 6483c024a7..fde80b4d33 100644 --- a/tools/bitstream2json/CMakeLists.txt +++ b/tools/bitstream2json/CMakeLists.txt @@ -9,6 +9,9 @@ set(TARGET_NAME bitstream2json) set(ROOT_DIR ../..) set(MACRO_DIR "${ROOT_DIR}/cmake/macros") +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + find_package(Qt5 COMPONENTS Network Script Widgets) include(${MACRO_DIR}/SetupHifiProject.cmake) @@ -17,4 +20,7 @@ setup_hifi_project(${TARGET_NAME} TRUE) link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} "${ROOT_DIR}") + target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) diff --git a/tools/json2bitstream/CMakeLists.txt b/tools/json2bitstream/CMakeLists.txt index 799f3bcc4c..51382d5858 100644 --- a/tools/json2bitstream/CMakeLists.txt +++ b/tools/json2bitstream/CMakeLists.txt @@ -9,6 +9,9 @@ set(TARGET_NAME json2bitstream) set(ROOT_DIR ../..) set(MACRO_DIR "${ROOT_DIR}/cmake/macros") +# setup for find modules +set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake/modules/") + find_package(Qt5 COMPONENTS Network Script Widgets) include(${MACRO_DIR}/SetupHifiProject.cmake) @@ -17,4 +20,7 @@ setup_hifi_project(${TARGET_NAME} TRUE) link_hifi_library(metavoxels ${TARGET_NAME} "${ROOT_DIR}") link_hifi_library(shared ${TARGET_NAME} "${ROOT_DIR}") +include(${MACRO_DIR}/IncludeGLM.cmake) +include_glm(${TARGET_NAME} "${ROOT_DIR}") + target_link_libraries(${TARGET_NAME} Qt5::Network Qt5::Widgets Qt5::Script) From 26159a05d925236cbe152ecc5479788d27129a7a Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 13:09:45 -0700 Subject: [PATCH 50/55] Fix for sporadic test failure due to hash ordering. --- tests/metavoxels/src/MetavoxelTests.cpp | 34 +++++++++++++++---------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tests/metavoxels/src/MetavoxelTests.cpp b/tests/metavoxels/src/MetavoxelTests.cpp index 4a3010caf4..c9bce27cc3 100644 --- a/tests/metavoxels/src/MetavoxelTests.cpp +++ b/tests/metavoxels/src/MetavoxelTests.cpp @@ -83,7 +83,7 @@ static TestSharedObjectA::TestFlags getRandomTestFlags() { return flags; } -static QScriptValue createRandomScriptValue(bool complex = false) { +static QScriptValue createRandomScriptValue(bool complex = false, bool ensureHashOrder = false) { scriptObjectsCreated++; switch (randIntInRange(0, complex ? 5 : 3)) { case 0: @@ -108,31 +108,37 @@ static QScriptValue createRandomScriptValue(bool complex = false) { } default: { QScriptValue value = ScriptCache::getInstance()->getEngine()->newObject(); - if (randomBoolean()) { + if (ensureHashOrder) { + // we can't depend on the iteration order, so if we need it to be the same (as when comparing bytes), we + // can only have one property value.setProperty("foo", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("bar", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("baz", createRandomScriptValue()); - } - if (randomBoolean()) { - value.setProperty("bong", createRandomScriptValue()); + } else { + if (randomBoolean()) { + value.setProperty("foo", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("bar", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("baz", createRandomScriptValue()); + } + if (randomBoolean()) { + value.setProperty("bong", createRandomScriptValue()); + } } return value; } } } -static TestMessageC createRandomMessageC() { +static TestMessageC createRandomMessageC(bool ensureHashOrder = false) { TestMessageC message; message.foo = randomBoolean(); message.bar = rand(); message.baz = randFloat(); message.bong.foo = createRandomBytes(); message.bong.baz = getRandomTestEnum(); - message.bizzle = createRandomScriptValue(true); + message.bizzle = createRandomScriptValue(true, ensureHashOrder); return message; } @@ -146,7 +152,7 @@ static bool testSerialization(Bitstream::MetadataType metadataType) { SharedObjectPointer testObjectWrittenB = new TestSharedObjectB(randFloat(), createRandomBytes(), TestSharedObjectB::THIRD_TEST_ENUM, TestSharedObjectB::SECOND_TEST_FLAG); out << testObjectWrittenB; - TestMessageC messageWritten = createRandomMessageC(); + TestMessageC messageWritten = createRandomMessageC(true); out << QVariant::fromValue(messageWritten); QByteArray endWritten = "end"; out << endWritten; From 29723d0ef32d3dfc6467fea1cd68e4f2f1197eb5 Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 18 Jun 2014 11:06:15 -0700 Subject: [PATCH 51/55] changed octree data nacks to repeatedly nack missing seq nums --- .../octree/OctreeInboundPacketProcessor.cpp | 17 ++-- interface/src/Application.cpp | 86 ++++++++++--------- interface/src/Application.h | 2 +- libraries/octree/src/OctreeSceneStats.cpp | 16 +--- libraries/octree/src/OctreeSceneStats.h | 6 +- 5 files changed, 58 insertions(+), 69 deletions(-) diff --git a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp index 78ab9259fd..76a6845342 100644 --- a/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp +++ b/assignment-client/src/octree/OctreeInboundPacketProcessor.cpp @@ -184,7 +184,8 @@ void OctreeInboundPacketProcessor::trackInboundPacket(const QUuid& nodeUUID, uns int OctreeInboundPacketProcessor::sendNackPackets() { int packetsSent = 0; - + char packet[MAX_PACKET_SIZE]; + NodeToSenderStatsMapIterator i = _singleSenderStats.begin(); while (i != _singleSenderStats.end()) { @@ -206,15 +207,10 @@ int OctreeInboundPacketProcessor::sendNackPackets() { const SharedNodePointer& destinationNode = NodeList::getInstance()->getNodeHash().value(nodeUUID); const QSet& missingSequenceNumbers = nodeStats.getMissingSequenceNumbers(); - - // check if there are any sequence numbers that need to be nacked - int numSequenceNumbersAvailable = missingSequenceNumbers.size(); - + // construct nack packet(s) for this node - - QSet::const_iterator missingSequenceNumberIterator = missingSequenceNumbers.begin(); - char packet[MAX_PACKET_SIZE]; - + int numSequenceNumbersAvailable = missingSequenceNumbers.size(); + QSet::const_iterator missingSequenceNumberIterator = missingSequenceNumbers.constBegin(); while (numSequenceNumbersAvailable > 0) { char* dataAt = packet; @@ -243,8 +239,7 @@ int OctreeInboundPacketProcessor::sendNackPackets() { numSequenceNumbersAvailable -= numSequenceNumbers; // send it - qint64 bytesWritten = NodeList::getInstance()->writeUnverifiedDatagram(packet, dataAt - packet, destinationNode); - + NodeList::getInstance()->writeUnverifiedDatagram(packet, dataAt - packet, destinationNode); packetsSent++; } i++; diff --git a/interface/src/Application.cpp b/interface/src/Application.cpp index 02cf59ad87..efe98819bd 100644 --- a/interface/src/Application.cpp +++ b/interface/src/Application.cpp @@ -2102,19 +2102,19 @@ void Application::updateMyAvatar(float deltaTime) { const quint64 TOO_LONG_SINCE_LAST_NACK = 1 * USECS_PER_SECOND; if (sinceLastNack > TOO_LONG_SINCE_LAST_NACK) { _lastNackTime = now; - sendNack(); + sendNackPackets(); } } } -void Application::sendNack() { +int Application::sendNackPackets() { if (Menu::getInstance()->isOptionChecked(MenuOption::DisableNackPackets)) { - return; + return 0; } + int packetsSent = 0; char packet[MAX_PACKET_SIZE]; - NodeList* nodeList = NodeList::getInstance(); // iterates thru all nodes in NodeList foreach(const SharedNodePointer& node, NodeList::getInstance()->getNodeHash()) { @@ -2125,14 +2125,14 @@ void Application::sendNack() { || node->getType() == NodeType::ModelServer) ) { - // if there are octree packets from this node that are waiting to be processed, - // don't send a NACK since the missing packets may be among those waiting packets. - if (_octreeProcessor.hasPacketsToProcessFrom(node)) { - continue; - } - QUuid nodeUUID = node->getUUID(); + // if there are octree packets from this node that are waiting to be processed, + // don't send a NACK since the missing packets may be among those waiting packets. + if (_octreeProcessor.hasPacketsToProcessFrom(nodeUUID)) { + continue; + } + _octreeSceneStatsLock.lockForRead(); // retreive octree scene stats of this node @@ -2142,40 +2142,48 @@ void Application::sendNack() { } OctreeSceneStats& stats = _octreeServerSceneStats[nodeUUID]; - // check if there are any sequence numbers that need to be nacked - int numSequenceNumbersAvailable = stats.getNumSequenceNumbersToNack(); - if (numSequenceNumbersAvailable == 0) { - _octreeSceneStatsLock.unlock(); - continue; - } + // make copy of missing sequence numbers from stats + const QSet missingSequenceNumbers = stats.getMissingSequenceNumbers(); - char* dataAt = packet; - int bytesRemaining = MAX_PACKET_SIZE; - - // pack header - int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack); - dataAt += numBytesPacketHeader; - bytesRemaining -= numBytesPacketHeader; - int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(OCTREE_PACKET_SEQUENCE); - - // calculate and pack the number of sequence numbers - uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); - uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; - *numSequenceNumbersAt = numSequenceNumbers; - dataAt += sizeof(uint16_t); - - // pack sequence numbers - for (int i = 0; i < numSequenceNumbers; i++) { - OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; - *sequenceNumberAt = stats.getNextSequenceNumberToNack(); - dataAt += sizeof(OCTREE_PACKET_SEQUENCE); - } - _octreeSceneStatsLock.unlock(); - nodeList->writeUnverifiedDatagram(packet, dataAt - packet, node); + // construct nack packet(s) for this node + int numSequenceNumbersAvailable = missingSequenceNumbers.size(); + QSet::const_iterator missingSequenceNumbersIterator = missingSequenceNumbers.constBegin(); + while (numSequenceNumbersAvailable > 0) { + + char* dataAt = packet; + int bytesRemaining = MAX_PACKET_SIZE; + + // pack header + int numBytesPacketHeader = populatePacketHeader(packet, PacketTypeOctreeDataNack); + dataAt += numBytesPacketHeader; + bytesRemaining -= numBytesPacketHeader; + + // calculate and pack the number of sequence numbers + int numSequenceNumbersRoomFor = (bytesRemaining - sizeof(uint16_t)) / sizeof(OCTREE_PACKET_SEQUENCE); + uint16_t numSequenceNumbers = min(numSequenceNumbersAvailable, numSequenceNumbersRoomFor); + uint16_t* numSequenceNumbersAt = (uint16_t*)dataAt; + *numSequenceNumbersAt = numSequenceNumbers; + dataAt += sizeof(uint16_t); + + // pack sequence numbers + for (int i = 0; i < numSequenceNumbers; i++) { + OCTREE_PACKET_SEQUENCE* sequenceNumberAt = (OCTREE_PACKET_SEQUENCE*)dataAt; + *sequenceNumberAt = *missingSequenceNumbersIterator; + dataAt += sizeof(OCTREE_PACKET_SEQUENCE); + + missingSequenceNumbersIterator++; + } + numSequenceNumbersAvailable -= numSequenceNumbers; + + // send it + NodeList::getInstance()->writeUnverifiedDatagram(packet, dataAt - packet, node); + packetsSent++; + } } } + return packetsSent; } void Application::queryOctree(NodeType_t serverType, PacketType packetType, NodeToJurisdictionMap& jurisdictions) { diff --git a/interface/src/Application.h b/interface/src/Application.h index 2889dcb301..e63c7c35b8 100644 --- a/interface/src/Application.h +++ b/interface/src/Application.h @@ -411,7 +411,7 @@ private: static void attachNewHeadToNode(Node *newNode); static void* networkReceive(void* args); // network receive thread - void sendNack(); + int sendNackPackets(); MainWindow* _window; GLCanvas* _glWidget; // our GLCanvas has a couple extra features diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index d037ec79ad..1d6185fb3e 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -47,7 +47,6 @@ OctreeSceneStats::OctreeSceneStats() : _incomingReallyLate(0), _incomingPossibleDuplicate(0), _missingSequenceNumbers(), - _sequenceNumbersToNack(), _incomingFlightTimeAverage(samples), _jurisdictionRoot(NULL) { @@ -160,7 +159,6 @@ void OctreeSceneStats::copyFromOther(const OctreeSceneStats& other) { _incomingPossibleDuplicate = other._incomingPossibleDuplicate; _missingSequenceNumbers = other._missingSequenceNumbers; - _sequenceNumbersToNack = other._sequenceNumbersToNack; } @@ -946,7 +944,6 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, qDebug() << "found it in _missingSequenceNumbers"; } _missingSequenceNumbers.remove(sequence); - _sequenceNumbersToNack.remove(sequence); _incomingLikelyLost--; _incomingRecovered++; } @@ -986,7 +983,6 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, for (int missingSequenceInt = expectedInt; missingSequenceInt < sequenceInt; missingSequenceInt++) { OCTREE_PACKET_SEQUENCE missingSequence = missingSequenceInt >= 0 ? missingSequenceInt : missingSequenceInt + UINT16_RANGE; _missingSequenceNumbers << missingSequence; - _sequenceNumbersToNack << missingSequence; } _incomingLastSequence = sequence; @@ -1025,7 +1021,6 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, qDebug() << "pruning really old missing sequence:" << missingItem; } _missingSequenceNumbers.remove(missingItem); - _sequenceNumbersToNack.remove(missingItem); } } } @@ -1038,13 +1033,6 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, } } -int OctreeSceneStats::getNumSequenceNumbersToNack() const { - return _sequenceNumbersToNack.size(); -} - -uint16_t OctreeSceneStats::getNextSequenceNumberToNack() { - QSet::Iterator it = _sequenceNumbersToNack.begin(); - uint16_t sequenceNumber = *it; - _sequenceNumbersToNack.remove(sequenceNumber); - return sequenceNumber; +const QSet& OctreeSceneStats::getMissingSequenceNumbers() const { + return _missingSequenceNumbers; } diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index 182bd6c86c..088eb787f8 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -172,9 +172,8 @@ public: quint32 getIncomingReallyLate() const { return _incomingReallyLate; } quint32 getIncomingPossibleDuplicate() const { return _incomingPossibleDuplicate; } float getIncomingFlightTimeAverage() { return _incomingFlightTimeAverage.getAverage(); } - - int getNumSequenceNumbersToNack() const; - OCTREE_PACKET_SEQUENCE getNextSequenceNumberToNack(); + + const QSet& getMissingSequenceNumbers() const; private: @@ -277,7 +276,6 @@ private: quint32 _incomingReallyLate; /// out of order and later than MAX_MISSING_SEQUENCE_OLD_AGE late quint32 _incomingPossibleDuplicate; /// out of order possibly a duplicate QSet _missingSequenceNumbers; - QSet _sequenceNumbersToNack; SimpleMovingAverage _incomingFlightTimeAverage; // features related items From c7788b793c50ee0b3e49c31293bcdb432f8d90bd Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 18 Jun 2014 13:36:31 -0700 Subject: [PATCH 52/55] removed erroneous call updateNodeWithDataFromPacket for octree data nacks --- assignment-client/src/octree/OctreeServer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index c6ef4aa5aa..48c8674c03 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -851,7 +851,6 @@ void OctreeServer::readPendingDatagrams() { // If we got a nack packet, then we're talking to an agent, and we // need to make sure we have it in our nodeList. if (matchingNode) { - nodeList->updateNodeWithDataFromPacket(matchingNode, receivedPacket); OctreeQueryNode* nodeData = (OctreeQueryNode*)matchingNode->getLinkedData(); if (nodeData) { nodeData->parseNackPacket(receivedPacket); From f76b4dc1ab90b347543d8d005d7198ae9634254a Mon Sep 17 00:00:00 2001 From: wangyix Date: Wed, 18 Jun 2014 13:39:49 -0700 Subject: [PATCH 53/55] made getMissingSequenceNumbers() inline --- libraries/octree/src/OctreeSceneStats.cpp | 4 ---- libraries/octree/src/OctreeSceneStats.h | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/libraries/octree/src/OctreeSceneStats.cpp b/libraries/octree/src/OctreeSceneStats.cpp index 1d6185fb3e..28445ec327 100644 --- a/libraries/octree/src/OctreeSceneStats.cpp +++ b/libraries/octree/src/OctreeSceneStats.cpp @@ -1032,7 +1032,3 @@ void OctreeSceneStats::trackIncomingOctreePacket(const QByteArray& packet, _incomingWastedBytes += (MAX_PACKET_SIZE - packet.size()); } } - -const QSet& OctreeSceneStats::getMissingSequenceNumbers() const { - return _missingSequenceNumbers; -} diff --git a/libraries/octree/src/OctreeSceneStats.h b/libraries/octree/src/OctreeSceneStats.h index 088eb787f8..1c468a8dc6 100644 --- a/libraries/octree/src/OctreeSceneStats.h +++ b/libraries/octree/src/OctreeSceneStats.h @@ -173,7 +173,7 @@ public: quint32 getIncomingPossibleDuplicate() const { return _incomingPossibleDuplicate; } float getIncomingFlightTimeAverage() { return _incomingFlightTimeAverage.getAverage(); } - const QSet& getMissingSequenceNumbers() const; + const QSet& getMissingSequenceNumbers() const { return _missingSequenceNumbers; } private: From 900024eefe3e31df15be97954cc32cc60fcf40ea Mon Sep 17 00:00:00 2001 From: Atlante45 Date: Wed, 18 Jun 2014 14:22:46 -0700 Subject: [PATCH 54/55] More intuitive rotation UI --- examples/editModels.js | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/examples/editModels.js b/examples/editModels.js index 55fcfa6be8..eebcd075fa 100644 --- a/examples/editModels.js +++ b/examples/editModels.js @@ -974,17 +974,27 @@ function mouseMoveEvent(event) { var pixelPerDegrees = windowDimensions.y / (1 * 360); // the entire height of the window allow you to make 2 full rotations - var STEP = 15; - var delta = Math.floor((event.x - mouseLastPosition.x) / pixelPerDegrees); + //compute delta in pixel + var cameraForward = Quat.getFront(Camera.getOrientation()); + var rotationAxis = (!zIsPressed && xIsPressed) ? { x: 1, y: 0, z: 0 } : + (!zIsPressed && !xIsPressed) ? { x: 0, y: 1, z: 0 } : + { x: 0, y: 0, z: 1 }; + rotationAxis = Vec3.multiplyQbyV(selectedModelProperties.modelRotation, rotationAxis); + var orthogonalAxis = Vec3.cross(cameraForward, rotationAxis); + var mouseDelta = { x: event.x - mouseLastPosition + .x, y: mouseLastPosition.y - event.y, z: 0 }; + var transformedMouseDelta = Vec3.multiplyQbyV(Camera.getOrientation(), mouseDelta); + var delta = Math.floor(Vec3.dot(transformedMouseDelta, Vec3.normalize(orthogonalAxis)) / pixelPerDegrees); + var STEP = 15; if (!event.isShifted) { - delta = Math.floor(delta / STEP) * STEP; + delta = Math.round(delta / STEP) * STEP; } var rotation = Quat.fromVec3Degrees({ - x: (!zIsPressed && xIsPressed) ? delta : 0, // z is pressed - y: (!zIsPressed && !xIsPressed) ? delta : 0, // x is pressed - z: (zIsPressed && !xIsPressed) ? delta : 0 // neither is pressed + x: (!zIsPressed && xIsPressed) ? delta : 0, // x is pressed + y: (!zIsPressed && !xIsPressed) ? delta : 0, // neither is pressed + z: (zIsPressed && !xIsPressed) ? delta : 0 // z is pressed }); rotation = Quat.multiply(selectedModelProperties.oldRotation, rotation); From 6ec562931f3a8167920be923c523ab6a1bda23b6 Mon Sep 17 00:00:00 2001 From: Andrzej Kapolka Date: Wed, 18 Jun 2014 14:44:11 -0700 Subject: [PATCH 55/55] Some additional comments for DatagramSequencer. --- libraries/metavoxels/src/DatagramSequencer.h | 44 +++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/libraries/metavoxels/src/DatagramSequencer.h b/libraries/metavoxels/src/DatagramSequencer.h index cf6ded74da..5ac88556f0 100644 --- a/libraries/metavoxels/src/DatagramSequencer.h +++ b/libraries/metavoxels/src/DatagramSequencer.h @@ -23,18 +23,50 @@ class ReliableChannel; -/// Performs simple datagram sequencing, packet fragmentation and reassembly. +/// Performs datagram sequencing, packet fragmentation and reassembly. Works with Bitstream to provide methods to send and +/// receive data over UDP with varying reliability and latency characteristics. To use, create a DatagramSequencer with the +/// fixed-size header that will be included with all outgoing datagrams and expected in all incoming ones (the contents of the +/// header are not checked on receive, only skipped over, and may be modified by the party that actually send the +/// datagram--this means that the header may include dynamically generated data, as long as its size remains fixed). Connect +/// the readyToWrite signal to a slot that will actually transmit the datagram to the remote party. When a datagram is +/// received from that party, call receivedDatagram with its contents. +/// +/// A "packet" represents a batch of data sent at one time (split into one or more datagrams sized below the MTU). Packets are +/// received in full and in order or not at all (that is, a packet being assembled is dropped as soon as a fragment from the +/// next packet is received). Packets can be any size, but the larger a packet is, the more likely it is to be dropped--so, +/// it's better to keep packet sizes close to the MTU. To write a packet, call startPacket, write data to the returned +/// Bitstream, then call endPacket (which will result in one or more firings of readyToWrite). Data written in this way is not +/// guaranteed to be received, but if it is received, it will arrive in order. This is a good way to transmit delta state: +/// state that represents the change between the last acknowledged state and the current state (which, if not received, will +/// not be resent as-is; instead, it will be replaced by up-to-date new deltas). +/// +/// There are two methods for sending reliable data. The first, for small messages that require minimum-latency processing, is +/// the high priority messaging system. When you call sendHighPriorityMessage, the message that you send will be included with +/// every outgoing packet until it is acknowledged. When the receiving party first sees the message, it will fire a +/// receivedHighPriorityMessage signal. +/// +/// The second method employs a set of independent reliable channels multiplexed onto the packet stream. These channels are +/// created lazily through the getReliableOutputChannel/getReliableInputChannel functions. Output channels contain buffers +/// to which one may write either arbitrary data (as a QIODevice) or messages (as QVariants), or switch between the two. +/// Each time a packet is sent, data pending for reliable output channels is added, in proportion to their relative priorities, +/// until the packet size limit set by setMaxPacketSize is reached. On the receive side, the streams are reconstructed and +/// (again, depending on whether messages are enabled) either the QIODevice reports that data is available, or, when a complete +/// message is decoded, the receivedMessage signal is fired. class DatagramSequencer : public QObject { Q_OBJECT public: + /// Contains the content of a high-priority message along with the number of the first packet in which it was sent. class HighPriorityMessage { public: QVariant data; int firstPacketNumber; }; + /// Creates a new datagram sequencer. + /// \param datagramHeader the content of the header that will be prepended to each outgoing datagram and whose length + /// will be skipped over in each incoming datagram DatagramSequencer(const QByteArray& datagramHeader = QByteArray(), QObject* parent = NULL); /// Returns a reference to the weak hash mapping remote ids to shared objects. @@ -267,15 +299,24 @@ class ReliableChannel : public QObject { public: + /// Returns the channel's index in the sequencer's channel map. int getIndex() const { return _index; } + /// Returns a reference to the buffer used to write/read data to/from this channel. CircularBuffer& getBuffer() { return _buffer; } + + /// Returns a reference to the data stream created on this channel's buffer. QDataStream& getDataStream() { return _dataStream; } + + /// Returns a reference to the bitstream created on this channel's data stream. Bitstream& getBitstream() { return _bitstream; } + /// Sets the channel priority, which determines how much of this channel's data (in proportion to the other channels) to + /// include in each outgoing packet. void setPriority(float priority) { _priority = priority; } float getPriority() const { return _priority; } + /// Returns the number of bytes available to read from this channel. int getBytesAvailable() const; /// Sets whether we expect to write/read framed messages. @@ -287,6 +328,7 @@ public: signals: + /// Fired when a framed message has been received on this channel. void receivedMessage(const QVariant& message); private slots: