From 0de109e2e81b3d21ab110cc9001e50e9075a53b7 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 18 Aug 2017 11:29:47 -0700 Subject: [PATCH 1/8] cleanup and fix typos --- libraries/octree/src/Octree.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index cceecaeed9..cde2fe7026 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -431,7 +431,7 @@ int Octree::readElementData(const OctreeElementPointer& destinationElement, cons return bytesRead; } -void Octree::readBitstreamToTree(const unsigned char * bitstream, unsigned long int bufferSizeBytes, +void Octree::readBitstreamToTree(const unsigned char * bitstream, uint64_t bufferSizeBytes, ReadBitstreamToTreeParams& args) { int bytesRead = 0; const unsigned char* bitstreamAt = bitstream; @@ -925,8 +925,8 @@ int Octree::encodeTreeBitstream(const OctreeElementPointer& element, roomForOctalCode = packetData->startSubTree(newCode); if (newCode) { - delete[] newCode; codeLength = numberOfThreeBitSectionsInCode(newCode); + delete[] newCode; } else { codeLength = 1; } @@ -1152,7 +1152,6 @@ int Octree::encodeTreeBitstreamRecursion(const OctreeElementPointer& element, OctreeElementPointer sortedChildren[NUMBER_OF_CHILDREN] = { NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL }; float distancesToChildren[NUMBER_OF_CHILDREN] = { 0, 0, 0, 0, 0, 0, 0, 0 }; int indexOfChildren[NUMBER_OF_CHILDREN] = { 0, 0, 0, 0, 0, 0, 0, 0 }; - int currentCount = 0; for (int i = 0; i < NUMBER_OF_CHILDREN; i++) { OctreeElementPointer childElement = element->getChildAtIndex(i); @@ -1174,7 +1173,6 @@ int Octree::encodeTreeBitstreamRecursion(const OctreeElementPointer& element, sortedChildren[i] = childElement; indexOfChildren[i] = i; distancesToChildren[i] = 0.0f; - currentCount++; // track stats // must check childElement here, because it could be we got here with no childElement @@ -1185,7 +1183,7 @@ int Octree::encodeTreeBitstreamRecursion(const OctreeElementPointer& element, // for each child element in Distance sorted order..., check to see if they exist, are colored, and in view, and if so // add them to our distance ordered array of children - for (int i = 0; i < currentCount; i++) { + for (int i = 0; i < NUMBER_OF_CHILDREN; i++) { OctreeElementPointer childElement = sortedChildren[i]; int originalIndex = indexOfChildren[i]; @@ -1276,7 +1274,7 @@ int Octree::encodeTreeBitstreamRecursion(const OctreeElementPointer& element, } // NOTE: the childrenDataBits indicates that there is an array of child element data included in this packet. - // We wil write this bit mask but we may come back later and update the bits that are actually included + // We will write this bit mask but we may come back later and update the bits that are actually included packetData->releaseReservedBytes(sizeof(childrenDataBits)); continueThisLevel = packetData->appendBitMask(childrenDataBits); @@ -1441,7 +1439,7 @@ int Octree::encodeTreeBitstreamRecursion(const OctreeElementPointer& element, // for each child element in Distance sorted order..., check to see if they exist, are colored, and in view, and if so // add them to our distance ordered array of children - for (int indexByDistance = 0; indexByDistance < currentCount; indexByDistance++) { + for (int indexByDistance = 0; indexByDistance < NUMBER_OF_CHILDREN; indexByDistance++) { OctreeElementPointer childElement = sortedChildren[indexByDistance]; int originalIndex = indexOfChildren[indexByDistance]; From 075b8574fb34f3ecee744691c8512555b2c63de5 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 18 Aug 2017 11:31:03 -0700 Subject: [PATCH 2/8] using 0.0 for time stats is unambiguous --- assignment-client/src/octree/OctreeServer.cpp | 17 +++++------------ assignment-client/src/octree/OctreeServer.h | 2 -- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index db97da751f..227f9599e3 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -37,8 +37,6 @@ int OctreeServer::_clientCount = 0; const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000000; -float OctreeServer::SKIP_TIME = -1.0f; // use this for trackXXXTime() calls for non-times - SimpleMovingAverage OctreeServer::_averageLoopTime(MOVING_AVERAGE_SAMPLE_COUNTS); SimpleMovingAverage OctreeServer::_averageInsideTime(MOVING_AVERAGE_SAMPLE_COUNTS); @@ -134,9 +132,8 @@ void OctreeServer::trackEncodeTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; - if (time == SKIP_TIME) { + if (time == 0.0f) { _noEncode++; - time = 0.0f; } else if (time <= MAX_SHORT_TIME) { _shortEncode++; _averageShortEncodeTime.updateAverage(time); @@ -153,9 +150,8 @@ void OctreeServer::trackEncodeTime(float time) { void OctreeServer::trackTreeWaitTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; - if (time == SKIP_TIME) { + if (time == 0.0f) { _noTreeWait++; - time = 0.0f; } else if (time <= MAX_SHORT_TIME) { _shortTreeWait++; _averageTreeShortWaitTime.updateAverage(time); @@ -172,9 +168,8 @@ void OctreeServer::trackTreeWaitTime(float time) { void OctreeServer::trackCompressAndWriteTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; - if (time == SKIP_TIME) { + if (time == 0.0f) { _noCompress++; - time = 0.0f; } else if (time <= MAX_SHORT_TIME) { _shortCompress++; _averageShortCompressTime.updateAverage(time); @@ -189,9 +184,8 @@ void OctreeServer::trackCompressAndWriteTime(float time) { } void OctreeServer::trackPacketSendingTime(float time) { - if (time == SKIP_TIME) { + if (time == 0.0f) { _noSend++; - time = 0.0f; } _averagePacketSendingTime.updateAverage(time); } @@ -200,9 +194,8 @@ void OctreeServer::trackPacketSendingTime(float time) { void OctreeServer::trackProcessWaitTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; - if (time == SKIP_TIME) { + if (time == 0.0f) { _noProcessWait++; - time = 0.0f; } else if (time <= MAX_SHORT_TIME) { _shortProcessWait++; _averageProcessShortWaitTime.updateAverage(time); diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index 5043ea681c..ab2ef30da1 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -82,8 +82,6 @@ public: virtual void trackSend(const QUuid& dataID, quint64 dataLastEdited, const QUuid& viewerNode) { } virtual void trackViewerGone(const QUuid& viewerNode) { } - static float SKIP_TIME; // use this for trackXXXTime() calls for non-times - static void trackLoopTime(float time) { _averageLoopTime.updateAverage(time); } static float getAverageLoopTime() { return _averageLoopTime.getAverage(); } From 65cf7b4f61fe8df7c514d4b0e96176d84e9b503f Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 18 Aug 2017 11:32:05 -0700 Subject: [PATCH 3/8] add OctreeSendThread::traverseAndBuildPacket() cleanup and split data packing from bandwidth management --- .../src/octree/OctreeSendThread.cpp | 130 ++++++++---------- .../src/octree/OctreeSendThread.h | 4 +- 2 files changed, 64 insertions(+), 70 deletions(-) diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 868b377ced..5c4639a8e0 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -458,84 +458,77 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode* return _truePacketsSent; } +bool OctreeSendThread::traverseAndBuildPacket(EncodeBitstreamParams& params) { + bool somethingToSend = false; + OctreeQueryNode* nodeData = static_cast(params.nodeData); + if (!nodeData->elementBag.isEmpty()) { + quint64 encodeStart = usecTimestampNow(); + quint64 lockWaitStart = encodeStart; + + _myServer->getOctree()->withReadLock([&]{ + OctreeServer::trackTreeWaitTime((float)(usecTimestampNow() - lockWaitStart)); + + OctreeElementPointer subTree = nodeData->elementBag.extract(); + if (subTree) { + // NOTE: this is where the tree "contents" are actually packed + nodeData->stats.encodeStarted(); + _myServer->getOctree()->encodeTreeBitstream(subTree, &_packetData, nodeData->elementBag, params); + nodeData->stats.encodeStopped(); + + somethingToSend = true; + } + }); + + OctreeServer::trackEncodeTime((float)(usecTimestampNow() - encodeStart)); + } + return somethingToSend; +} + void OctreeSendThread::traverseTreeAndSendContents(SharedNodePointer node, OctreeQueryNode* nodeData, bool viewFrustumChanged, bool isFullScene) { // calculate max number of packets that can be sent during this interval int clientMaxPacketsPerInterval = std::max(1, (nodeData->getMaxQueryPacketsPerSecond() / INTERVALS_PER_SECOND)); int maxPacketsPerInterval = std::min(clientMaxPacketsPerInterval, _myServer->getPacketsPerClientPerInterval()); int extraPackingAttempts = 0; - bool completedScene = false; + + // init params once outside the while loop + int boundaryLevelAdjustClient = nodeData->getBoundaryLevelAdjust(); + int boundaryLevelAdjust = boundaryLevelAdjustClient + + (viewFrustumChanged ? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST); + float octreeSizeScale = nodeData->getOctreeSizeScale(); + EncodeBitstreamParams params(INT_MAX, WANT_EXISTS_BITS, DONT_CHOP, + viewFrustumChanged, boundaryLevelAdjust, octreeSizeScale, + isFullScene, _myServer->getJurisdiction(), nodeData); + // Our trackSend() function is implemented by the server subclass, and will be called back + // during the encodeTreeBitstream() as new entities/data elements are sent + params.trackSend = [this](const QUuid& dataID, quint64 dataEdited) { + _myServer->trackSend(dataID, dataEdited, _nodeUuid); + }; + nodeData->copyCurrentViewFrustum(params.viewFrustum); + if (viewFrustumChanged) { + nodeData->copyLastKnownViewFrustum(params.lastViewFrustum); + } bool somethingToSend = true; // assume we have something + bool bagHadSomething = !nodeData->elementBag.isEmpty(); while (somethingToSend && _packetsSentThisInterval < maxPacketsPerInterval && !nodeData->isShuttingDown()) { - float lockWaitElapsedUsec = OctreeServer::SKIP_TIME; - float encodeElapsedUsec = OctreeServer::SKIP_TIME; - float compressAndWriteElapsedUsec = OctreeServer::SKIP_TIME; - float packetSendingElapsedUsec = OctreeServer::SKIP_TIME; + float compressAndWriteElapsedUsec = 0.0f; + float packetSendingElapsedUsec = 0.0f; quint64 startInside = usecTimestampNow(); bool lastNodeDidntFit = false; // assume each node fits - if (!nodeData->elementBag.isEmpty()) { + params.stopReason = EncodeBitstreamParams::UNKNOWN; // reset params.stopReason before traversal - quint64 lockWaitStart = usecTimestampNow(); - _myServer->getOctree()->withReadLock([&]{ - quint64 lockWaitEnd = usecTimestampNow(); - lockWaitElapsedUsec = (float)(lockWaitEnd - lockWaitStart); - quint64 encodeStart = usecTimestampNow(); + somethingToSend = traverseAndBuildPacket(params); - OctreeElementPointer subTree = nodeData->elementBag.extract(); - if (!subTree) { - return; - } - - float octreeSizeScale = nodeData->getOctreeSizeScale(); - int boundaryLevelAdjustClient = nodeData->getBoundaryLevelAdjust(); - - int boundaryLevelAdjust = boundaryLevelAdjustClient + - (viewFrustumChanged ? LOW_RES_MOVING_ADJUST : NO_BOUNDARY_ADJUST); - - EncodeBitstreamParams params(INT_MAX, WANT_EXISTS_BITS, DONT_CHOP, - viewFrustumChanged, boundaryLevelAdjust, octreeSizeScale, - isFullScene, _myServer->getJurisdiction(), nodeData); - nodeData->copyCurrentViewFrustum(params.viewFrustum); - if (viewFrustumChanged) { - nodeData->copyLastKnownViewFrustum(params.lastViewFrustum); - } - - // Our trackSend() function is implemented by the server subclass, and will be called back - // during the encodeTreeBitstream() as new entities/data elements are sent - params.trackSend = [this](const QUuid& dataID, quint64 dataEdited) { - _myServer->trackSend(dataID, dataEdited, _nodeUuid); - }; - - // TODO: should this include the lock time or not? This stat is sent down to the client, - // it seems like it may be a good idea to include the lock time as part of the encode time - // are reported to client. Since you can encode without the lock - nodeData->stats.encodeStarted(); - - // NOTE: this is where the tree "contents" are actaully packed - _myServer->getOctree()->encodeTreeBitstream(subTree, &_packetData, nodeData->elementBag, params); - - quint64 encodeEnd = usecTimestampNow(); - encodeElapsedUsec = (float)(encodeEnd - encodeStart); - - // If after calling encodeTreeBitstream() there are no nodes left to send, then we know we've - // sent the entire scene. We want to know this below so we'll actually write this content into - // the packet and send it - completedScene = nodeData->elementBag.isEmpty(); - - if (params.stopReason == EncodeBitstreamParams::DIDNT_FIT) { - lastNodeDidntFit = true; - extraPackingAttempts++; - } - - nodeData->stats.encodeStopped(); - }); - } else { - somethingToSend = false; // this will cause us to drop out of the loop... + if (params.stopReason == EncodeBitstreamParams::DIDNT_FIT) { + lastNodeDidntFit = true; + extraPackingAttempts++; } + // If the bag had contents but is now empty then we know we've sent the entire scene. + bool completedScene = bagHadSomething && nodeData->elementBag.isEmpty(); if (completedScene || lastNodeDidntFit) { // we probably want to flush what has accumulated in nodeData but: // do we have more data to send? and is there room? @@ -562,8 +555,7 @@ void OctreeSendThread::traverseTreeAndSendContents(SharedNodePointer node, Octre if (sendNow) { quint64 packetSendingStart = usecTimestampNow(); _packetsSentThisInterval += handlePacketSend(node, nodeData); - quint64 packetSendingEnd = usecTimestampNow(); - packetSendingElapsedUsec = (float)(packetSendingEnd - packetSendingStart); + packetSendingElapsedUsec = (float)(usecTimestampNow() - packetSendingStart); targetSize = nodeData->getAvailable() - sizeof(OCTREE_PACKET_INTERNAL_SECTION_SIZE); extraPackingAttempts = 0; @@ -576,14 +568,14 @@ void OctreeSendThread::traverseTreeAndSendContents(SharedNodePointer node, Octre } _packetData.changeSettings(true, targetSize); // will do reset - NOTE: Always compressed } - OctreeServer::trackTreeWaitTime(lockWaitElapsedUsec); - OctreeServer::trackEncodeTime(encodeElapsedUsec); + if (!somethingToSend) { + // these stats were not updated so we record zero to record this fact + OctreeServer::trackTreeWaitTime(0.0f); + OctreeServer::trackEncodeTime(0.0f); + } OctreeServer::trackCompressAndWriteTime(compressAndWriteElapsedUsec); OctreeServer::trackPacketSendingTime(packetSendingElapsedUsec); - - quint64 endInside = usecTimestampNow(); - quint64 elapsedInsideUsecs = endInside - startInside; - OctreeServer::trackInsideTime((float)elapsedInsideUsecs); + OctreeServer::trackInsideTime((float)(usecTimestampNow() - startInside)); } if (somethingToSend && _myServer->wantsVerboseDebug()) { diff --git a/assignment-client/src/octree/OctreeSendThread.h b/assignment-client/src/octree/OctreeSendThread.h index d158539f57..583c30bdd3 100644 --- a/assignment-client/src/octree/OctreeSendThread.h +++ b/assignment-client/src/octree/OctreeSendThread.h @@ -53,7 +53,9 @@ protected: /// Called before a packetDistributor pass to allow for pre-distribution processing virtual void preDistributionProcessing() {}; - virtual void traverseTreeAndSendContents(SharedNodePointer node, OctreeQueryNode* nodeData, bool viewFrustumChanged, bool isFullScene); + virtual void traverseTreeAndSendContents(SharedNodePointer node, OctreeQueryNode* nodeData, + bool viewFrustumChanged, bool isFullScene); + virtual bool traverseAndBuildPacket(EncodeBitstreamParams& params); OctreeServer* _myServer { nullptr }; QWeakPointer _node; From 8ad41227cecf5a64120e30b153e456f96b38242e Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 18 Aug 2017 13:35:13 -0700 Subject: [PATCH 4/8] use more correct method name --- assignment-client/src/octree/OctreeSendThread.cpp | 4 ++-- assignment-client/src/octree/OctreeSendThread.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 5c4639a8e0..9e37be4b4e 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -458,7 +458,7 @@ int OctreeSendThread::packetDistributor(SharedNodePointer node, OctreeQueryNode* return _truePacketsSent; } -bool OctreeSendThread::traverseAndBuildPacket(EncodeBitstreamParams& params) { +bool OctreeSendThread::traverseTreeAndBuildNextPacketPayload(EncodeBitstreamParams& params) { bool somethingToSend = false; OctreeQueryNode* nodeData = static_cast(params.nodeData); if (!nodeData->elementBag.isEmpty()) { @@ -520,7 +520,7 @@ void OctreeSendThread::traverseTreeAndSendContents(SharedNodePointer node, Octre bool lastNodeDidntFit = false; // assume each node fits params.stopReason = EncodeBitstreamParams::UNKNOWN; // reset params.stopReason before traversal - somethingToSend = traverseAndBuildPacket(params); + somethingToSend = traverseTreeAndBuildNextPacketPayload(params); if (params.stopReason == EncodeBitstreamParams::DIDNT_FIT) { lastNodeDidntFit = true; diff --git a/assignment-client/src/octree/OctreeSendThread.h b/assignment-client/src/octree/OctreeSendThread.h index 583c30bdd3..8f75092528 100644 --- a/assignment-client/src/octree/OctreeSendThread.h +++ b/assignment-client/src/octree/OctreeSendThread.h @@ -55,7 +55,7 @@ protected: virtual void preDistributionProcessing() {}; virtual void traverseTreeAndSendContents(SharedNodePointer node, OctreeQueryNode* nodeData, bool viewFrustumChanged, bool isFullScene); - virtual bool traverseAndBuildPacket(EncodeBitstreamParams& params); + virtual bool traverseTreeAndBuildNextPacketPayload(EncodeBitstreamParams& params); OctreeServer* _myServer { nullptr }; QWeakPointer _node; From aee35b751ece12436a742ac387eacc687c53b5a8 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Fri, 18 Aug 2017 14:35:48 -0700 Subject: [PATCH 5/8] replace 'unsigned long' with uint64_t --- libraries/octree/src/Octree.cpp | 30 +++++++++++++++--------------- libraries/octree/src/Octree.h | 11 ++++++----- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp index cde2fe7026..ad16a97bf6 100644 --- a/libraries/octree/src/Octree.cpp +++ b/libraries/octree/src/Octree.cpp @@ -1636,7 +1636,7 @@ bool Octree::readFromFile(const char* fileName) { QDataStream fileInputStream(&file); QFileInfo fileInfo(qFileName); - unsigned long fileLength = fileInfo.size(); + uint64_t fileLength = fileInfo.size(); emit importSize(1.0f, 1.0f, 1.0f); emit importProgress(0); @@ -1713,7 +1713,7 @@ bool Octree::readFromURL(const QString& urlString) { } -bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream, const QString& marketplaceID) { +bool Octree::readFromStream(uint64_t streamLength, QDataStream& inputStream, const QString& marketplaceID) { // decide if this is binary SVO or JSON-formatted SVO QIODevice *device = inputStream.device(); char firstChar; @@ -1730,14 +1730,14 @@ bool Octree::readFromStream(unsigned long streamLength, QDataStream& inputStream } -bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStream) { +bool Octree::readSVOFromStream(uint64_t streamLength, QDataStream& inputStream) { qWarning() << "SVO file format depricated. Support for reading SVO files is no longer support and will be removed soon."; bool fileOk = false; PacketVersion gotVersion = 0; - unsigned long headerLength = 0; // bytes in the header + uint64_t headerLength = 0; // bytes in the header bool wantImportProgress = true; @@ -1749,14 +1749,14 @@ bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStr if (getWantSVOfileVersions()) { // read just enough of the file to parse the header... - const unsigned long HEADER_LENGTH = sizeof(int) + sizeof(PacketVersion); + const uint64_t HEADER_LENGTH = sizeof(int) + sizeof(PacketVersion); unsigned char fileHeader[HEADER_LENGTH]; inputStream.readRawData((char*)&fileHeader, HEADER_LENGTH); headerLength = HEADER_LENGTH; // we need this later to skip to the data unsigned char* dataAt = (unsigned char*)&fileHeader; - unsigned long dataLength = HEADER_LENGTH; + uint64_t dataLength = HEADER_LENGTH; // if so, read the first byte of the file and see if it matches the expected version code int intPacketType; @@ -1802,7 +1802,7 @@ bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStr if (!hasBufferBreaks) { // read the entire file into a buffer, WHAT!? Why not. - unsigned long dataLength = streamLength - headerLength; + uint64_t dataLength = streamLength - headerLength; unsigned char* entireFileDataSection = new unsigned char[dataLength]; inputStream.readRawData((char*)entireFileDataSection, dataLength); @@ -1817,9 +1817,9 @@ bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStr } else { - unsigned long dataLength = streamLength - headerLength; - unsigned long remainingLength = dataLength; - const unsigned long MAX_CHUNK_LENGTH = MAX_OCTREE_PACKET_SIZE * 2; + uint64_t dataLength = streamLength - headerLength; + uint64_t remainingLength = dataLength; + const uint64_t MAX_CHUNK_LENGTH = MAX_OCTREE_PACKET_SIZE * 2; unsigned char* fileChunk = new unsigned char[MAX_CHUNK_LENGTH]; while (remainingLength > 0) { @@ -1845,7 +1845,7 @@ bool Octree::readSVOFromStream(unsigned long streamLength, QDataStream& inputStr remainingLength -= chunkLength; unsigned char* dataAt = fileChunk; - unsigned long dataLength = chunkLength; + uint64_t dataLength = chunkLength; ReadBitstreamToTreeParams args(NO_EXISTS_BITS, NULL, 0, SharedNodePointer(), wantImportProgress, gotVersion); @@ -1885,7 +1885,7 @@ QJsonDocument addMarketplaceIDToDocumentEntities(QJsonDocument& doc, const QStri const int READ_JSON_BUFFER_SIZE = 2048; -bool Octree::readJSONFromStream(unsigned long streamLength, QDataStream& inputStream, const QString& marketplaceID /*=""*/) { +bool Octree::readJSONFromStream(uint64_t streamLength, QDataStream& inputStream, const QString& marketplaceID /*=""*/) { // if the data is gzipped we may not have a useful bytesAvailable() result, so just keep reading until // we get an eof. Leave streamLength parameter for consistency. @@ -1980,14 +1980,14 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e return success; } -unsigned long Octree::getOctreeElementsCount() { - unsigned long nodeCount = 0; +uint64_t Octree::getOctreeElementsCount() { + uint64_t nodeCount = 0; recurseTreeWithOperation(countOctreeElementsOperation, &nodeCount); return nodeCount; } bool Octree::countOctreeElementsOperation(const OctreeElementPointer& element, void* extraData) { - (*(unsigned long*)extraData)++; + (*(uint64_t*)extraData)++; return true; // keep going } diff --git a/libraries/octree/src/Octree.h b/libraries/octree/src/Octree.h index 38454d20b5..3b84618a56 100644 --- a/libraries/octree/src/Octree.h +++ b/libraries/octree/src/Octree.h @@ -14,6 +14,7 @@ #include #include +#include #include #include @@ -231,7 +232,7 @@ public: virtual void eraseAllOctreeElements(bool createNewRoot = true); - void readBitstreamToTree(const unsigned char* bitstream, unsigned long int bufferSizeBytes, ReadBitstreamToTreeParams& args); + void readBitstreamToTree(const unsigned char* bitstream, uint64_t bufferSizeBytes, ReadBitstreamToTreeParams& args); void deleteOctalCodeFromTree(const unsigned char* codeBuffer, bool collapseEmptyTrees = DONT_COLLAPSE); void reaverageOctreeElements(OctreeElementPointer startElement = OctreeElementPointer()); @@ -301,13 +302,13 @@ public: // Octree importers bool readFromFile(const char* filename); bool readFromURL(const QString& url); // will support file urls as well... - bool readFromStream(unsigned long streamLength, QDataStream& inputStream, const QString& marketplaceID=""); - bool readSVOFromStream(unsigned long streamLength, QDataStream& inputStream); - bool readJSONFromStream(unsigned long streamLength, QDataStream& inputStream, const QString& marketplaceID=""); + bool readFromStream(uint64_t streamLength, QDataStream& inputStream, const QString& marketplaceID=""); + bool readSVOFromStream(uint64_t streamLength, QDataStream& inputStream); + bool readJSONFromStream(uint64_t streamLength, QDataStream& inputStream, const QString& marketplaceID=""); bool readJSONFromGzippedFile(QString qFileName); virtual bool readFromMap(QVariantMap& entityDescription) = 0; - unsigned long getOctreeElementsCount(); + uint64_t getOctreeElementsCount(); bool getShouldReaverage() const { return _shouldReaverage; } From 4be119b729066ebc1b7259179b169e5568fcaa6b Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 23 Aug 2017 14:04:30 -0700 Subject: [PATCH 6/8] make SKIP_TIME work as intended --- assignment-client/src/octree/OctreeServer.cpp | 95 +++++++++++-------- assignment-client/src/octree/OctreeServer.h | 2 + 2 files changed, 55 insertions(+), 42 deletions(-) diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index 227f9599e3..55fc405d1b 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -37,6 +37,8 @@ int OctreeServer::_clientCount = 0; const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000000; +float OctreeServer::SKIP_TIME = -1.0f; // use this for trackXXXTime() calls for non-times + SimpleMovingAverage OctreeServer::_averageLoopTime(MOVING_AVERAGE_SAMPLE_COUNTS); SimpleMovingAverage OctreeServer::_averageInsideTime(MOVING_AVERAGE_SAMPLE_COUNTS); @@ -132,81 +134,90 @@ void OctreeServer::trackEncodeTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; - if (time == 0.0f) { + if (time == SKIP_TIME) { _noEncode++; - } else if (time <= MAX_SHORT_TIME) { - _shortEncode++; - _averageShortEncodeTime.updateAverage(time); - } else if (time <= MAX_LONG_TIME) { - _longEncode++; - _averageLongEncodeTime.updateAverage(time); } else { - _extraLongEncode++; - _averageExtraLongEncodeTime.updateAverage(time); + if (time <= MAX_SHORT_TIME) { + _shortEncode++; + _averageShortEncodeTime.updateAverage(time); + } else if (time <= MAX_LONG_TIME) { + _longEncode++; + _averageLongEncodeTime.updateAverage(time); + } else { + _extraLongEncode++; + _averageExtraLongEncodeTime.updateAverage(time); + } + _averageEncodeTime.updateAverage(time); } - _averageEncodeTime.updateAverage(time); } void OctreeServer::trackTreeWaitTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; - if (time == 0.0f) { + if (time == SKIP_TIME) { _noTreeWait++; - } else if (time <= MAX_SHORT_TIME) { - _shortTreeWait++; - _averageTreeShortWaitTime.updateAverage(time); - } else if (time <= MAX_LONG_TIME) { - _longTreeWait++; - _averageTreeLongWaitTime.updateAverage(time); } else { - _extraLongTreeWait++; - _averageTreeExtraLongWaitTime.updateAverage(time); + if (time <= MAX_SHORT_TIME) { + _shortTreeWait++; + _averageTreeShortWaitTime.updateAverage(time); + } else if (time <= MAX_LONG_TIME) { + _longTreeWait++; + _averageTreeLongWaitTime.updateAverage(time); + } else { + _extraLongTreeWait++; + _averageTreeExtraLongWaitTime.updateAverage(time); + } + _averageTreeWaitTime.updateAverage(time); } - _averageTreeWaitTime.updateAverage(time); } void OctreeServer::trackCompressAndWriteTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; - if (time == 0.0f) { + if (time == SKIP_TIME) { _noCompress++; - } else if (time <= MAX_SHORT_TIME) { - _shortCompress++; - _averageShortCompressTime.updateAverage(time); - } else if (time <= MAX_LONG_TIME) { - _longCompress++; - _averageLongCompressTime.updateAverage(time); } else { - _extraLongCompress++; - _averageExtraLongCompressTime.updateAverage(time); + if (time <= MAX_SHORT_TIME) { + _shortCompress++; + _averageShortCompressTime.updateAverage(time); + } else if (time <= MAX_LONG_TIME) { + _longCompress++; + _averageLongCompressTime.updateAverage(time); + } else { + _extraLongCompress++; + _averageExtraLongCompressTime.updateAverage(time); + } + _averageCompressAndWriteTime.updateAverage(time); } - _averageCompressAndWriteTime.updateAverage(time); } void OctreeServer::trackPacketSendingTime(float time) { - if (time == 0.0f) { + if (time == SKIP_TIME) { _noSend++; + } else { + _averagePacketSendingTime.updateAverage(time); } - _averagePacketSendingTime.updateAverage(time); } void OctreeServer::trackProcessWaitTime(float time) { const float MAX_SHORT_TIME = 10.0f; const float MAX_LONG_TIME = 100.0f; - if (time == 0.0f) { + if (time == SKIP_TIME) { _noProcessWait++; - } else if (time <= MAX_SHORT_TIME) { - _shortProcessWait++; - _averageProcessShortWaitTime.updateAverage(time); - } else if (time <= MAX_LONG_TIME) { - _longProcessWait++; - _averageProcessLongWaitTime.updateAverage(time); } else { - _extraLongProcessWait++; - _averageProcessExtraLongWaitTime.updateAverage(time); + if (time <= MAX_SHORT_TIME) { + _shortProcessWait++; + _averageProcessShortWaitTime.updateAverage(time); + } else if (time <= MAX_LONG_TIME) { + _longProcessWait++; + _averageProcessLongWaitTime.updateAverage(time); + } else { + _extraLongProcessWait++; + _averageProcessExtraLongWaitTime.updateAverage(time); + } + _averageProcessWaitTime.updateAverage(time); } - _averageProcessWaitTime.updateAverage(time); } OctreeServer::OctreeServer(ReceivedMessage& message) : diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h index ab2ef30da1..5043ea681c 100644 --- a/assignment-client/src/octree/OctreeServer.h +++ b/assignment-client/src/octree/OctreeServer.h @@ -82,6 +82,8 @@ public: virtual void trackSend(const QUuid& dataID, quint64 dataLastEdited, const QUuid& viewerNode) { } virtual void trackViewerGone(const QUuid& viewerNode) { } + static float SKIP_TIME; // use this for trackXXXTime() calls for non-times + static void trackLoopTime(float time) { _averageLoopTime.updateAverage(time); } static float getAverageLoopTime() { return _averageLoopTime.getAverage(); } From 45f194f6bc18dc9bcae1974721b9ec284878c77f Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Wed, 23 Aug 2017 14:05:05 -0700 Subject: [PATCH 7/8] use SKIP_TIME again instead of 0.0 --- assignment-client/src/octree/OctreeSendThread.cpp | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/assignment-client/src/octree/OctreeSendThread.cpp b/assignment-client/src/octree/OctreeSendThread.cpp index 9e37be4b4e..9345f96b1d 100644 --- a/assignment-client/src/octree/OctreeSendThread.cpp +++ b/assignment-client/src/octree/OctreeSendThread.cpp @@ -480,6 +480,9 @@ bool OctreeSendThread::traverseTreeAndBuildNextPacketPayload(EncodeBitstreamPara }); OctreeServer::trackEncodeTime((float)(usecTimestampNow() - encodeStart)); + } else { + OctreeServer::trackTreeWaitTime(OctreeServer::SKIP_TIME); + OctreeServer::trackEncodeTime(OctreeServer::SKIP_TIME); } return somethingToSend; } @@ -512,8 +515,8 @@ void OctreeSendThread::traverseTreeAndSendContents(SharedNodePointer node, Octre bool somethingToSend = true; // assume we have something bool bagHadSomething = !nodeData->elementBag.isEmpty(); while (somethingToSend && _packetsSentThisInterval < maxPacketsPerInterval && !nodeData->isShuttingDown()) { - float compressAndWriteElapsedUsec = 0.0f; - float packetSendingElapsedUsec = 0.0f; + float compressAndWriteElapsedUsec = OctreeServer::SKIP_TIME; + float packetSendingElapsedUsec = OctreeServer::SKIP_TIME; quint64 startInside = usecTimestampNow(); @@ -568,11 +571,6 @@ void OctreeSendThread::traverseTreeAndSendContents(SharedNodePointer node, Octre } _packetData.changeSettings(true, targetSize); // will do reset - NOTE: Always compressed } - if (!somethingToSend) { - // these stats were not updated so we record zero to record this fact - OctreeServer::trackTreeWaitTime(0.0f); - OctreeServer::trackEncodeTime(0.0f); - } OctreeServer::trackCompressAndWriteTime(compressAndWriteElapsedUsec); OctreeServer::trackPacketSendingTime(packetSendingElapsedUsec); OctreeServer::trackInsideTime((float)(usecTimestampNow() - startInside)); From d6b914283c4a12d8d00526568386830345725b60 Mon Sep 17 00:00:00 2001 From: Andrew Meadows Date: Thu, 24 Aug 2017 13:56:34 -0700 Subject: [PATCH 8/8] use sane running average window --- assignment-client/src/octree/OctreeServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp index 55fc405d1b..974f00326b 100644 --- a/assignment-client/src/octree/OctreeServer.cpp +++ b/assignment-client/src/octree/OctreeServer.cpp @@ -35,7 +35,7 @@ #include int OctreeServer::_clientCount = 0; -const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000000; +const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000; float OctreeServer::SKIP_TIME = -1.0f; // use this for trackXXXTime() calls for non-times