From b5f6a51728919ba82b8320d26d0a33d3d53d6227 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 7 Nov 2013 13:03:04 -0800 Subject: [PATCH 1/2] fix thread contention issues and VS crash on domain server shutdown and restart --- .../src/VoxelNodeData.cpp | 1 + .../voxel-server-library/src/VoxelNodeData.h | 2 +- .../src/VoxelPersistThread.cpp | 29 ++++----- .../src/VoxelPersistThread.h | 11 +++- .../src/VoxelSendThread.cpp | 44 ++++++++------ .../voxel-server-library/src/VoxelServer.cpp | 59 ++++++++++++++++++- .../voxel-server-library/src/VoxelServer.h | 4 ++ .../src/VoxelServerPacketProcessor.cpp | 2 +- libraries/voxels/src/VoxelQuery.cpp | 1 + libraries/voxels/src/VoxelQuery.h | 2 +- libraries/voxels/src/VoxelTree.cpp | 3 +- libraries/voxels/src/VoxelTree.h | 2 + 12 files changed, 116 insertions(+), 44 deletions(-) diff --git a/libraries/voxel-server-library/src/VoxelNodeData.cpp b/libraries/voxel-server-library/src/VoxelNodeData.cpp index 8151839b7d..2cede147b3 100644 --- a/libraries/voxel-server-library/src/VoxelNodeData.cpp +++ b/libraries/voxel-server-library/src/VoxelNodeData.cpp @@ -129,6 +129,7 @@ VoxelNodeData::~VoxelNodeData() { if (_voxelSendThread) { _voxelSendThread->terminate(); delete _voxelSendThread; + qDebug("VoxelNodeData::~VoxelNodeData() DELETED _voxelSendThread=%p\n", _voxelSendThread); } } diff --git a/libraries/voxel-server-library/src/VoxelNodeData.h b/libraries/voxel-server-library/src/VoxelNodeData.h index 1ad4558b81..c21b99a308 100644 --- a/libraries/voxel-server-library/src/VoxelNodeData.h +++ b/libraries/voxel-server-library/src/VoxelNodeData.h @@ -24,7 +24,7 @@ class VoxelServer; class VoxelNodeData : public VoxelQuery { public: VoxelNodeData(Node* owningNode); - ~VoxelNodeData(); + virtual ~VoxelNodeData(); void resetVoxelPacket(); // resets voxel packet to after "V" header diff --git a/libraries/voxel-server-library/src/VoxelPersistThread.cpp b/libraries/voxel-server-library/src/VoxelPersistThread.cpp index 0a22b0a2ec..0f2718fe1f 100644 --- a/libraries/voxel-server-library/src/VoxelPersistThread.cpp +++ b/libraries/voxel-server-library/src/VoxelPersistThread.cpp @@ -20,34 +20,28 @@ VoxelPersistThread::VoxelPersistThread(VoxelTree* tree, const char* filename, in _tree(tree), _filename(filename), _persistInterval(persistInterval), - _initialLoad(false) { + _initialLoadComplete(false), + _loadTimeUSecs(0) { } bool VoxelPersistThread::process() { - if (!_initialLoad) { - _initialLoad = true; + if (!_initialLoadComplete) { + uint64_t loadStarted = usecTimestampNow(); qDebug("loading voxels from file: %s...\n", _filename); bool persistantFileRead; - + + _tree->lockForWrite(); { PerformanceWarning warn(true, "Loading Voxel File", true); - _tree->lockForRead(); persistantFileRead = _tree->readFromSVOFile(_filename); - _tree->unlock(); } + _tree->unlock(); - if (persistantFileRead) { - PerformanceWarning warn(true, "Voxels Re-Averaging", true); - - // after done inserting all these voxels, then reaverage colors - qDebug("BEGIN Voxels Re-Averaging\n"); - _tree->lockForWrite(); - _tree->reaverageVoxelColors(_tree->rootNode); - _tree->unlock(); - qDebug("DONE WITH Voxels Re-Averaging\n"); - } + _loadCompleted = time(0); + uint64_t loadDone = usecTimestampNow(); + _loadTimeUSecs = loadDone - loadStarted; _tree->clearDirtyBit(); // the tree is clean since we just loaded it qDebug("DONE loading voxels from file... fileRead=%s\n", debug::valueOf(persistantFileRead)); @@ -65,6 +59,7 @@ bool VoxelPersistThread::process() { qDebug("setChildAtIndexCalls=%llu setChildAtIndexTime=%llu perSet=%lf\n", VoxelNode::getSetChildAtIndexTime(), VoxelNode::getSetChildAtIndexCalls(), usecPerSet); + _initialLoadComplete = true; } uint64_t MSECS_TO_USECS = 1000; @@ -74,9 +69,7 @@ bool VoxelPersistThread::process() { // check the dirty bit and persist here... if (_tree->isDirty()) { qDebug("saving voxels to file %s...\n",_filename); - _tree->lockForRead(); _tree->writeToSVOFile(_filename); - _tree->unlock(); _tree->clearDirtyBit(); // tree is clean after saving qDebug("DONE saving voxels to file...\n"); } diff --git a/libraries/voxel-server-library/src/VoxelPersistThread.h b/libraries/voxel-server-library/src/VoxelPersistThread.h index 3219633702..91e65a2daf 100644 --- a/libraries/voxel-server-library/src/VoxelPersistThread.h +++ b/libraries/voxel-server-library/src/VoxelPersistThread.h @@ -21,6 +21,12 @@ public: static const int DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds VoxelPersistThread(VoxelTree* tree, const char* filename, int persistInterval = DEFAULT_PERSIST_INTERVAL); + + bool isInitialLoadComplete() const { return _initialLoadComplete; } + + time_t* getLoadCompleted() { return &_loadCompleted; } + uint64_t getLoadElapsedTime() const { return _loadTimeUSecs; } + protected: /// Implements generic processing behavior for this thread. virtual bool process(); @@ -28,7 +34,10 @@ private: VoxelTree* _tree; const char* _filename; int _persistInterval; - bool _initialLoad; + bool _initialLoadComplete; + + time_t _loadCompleted; + uint64_t _loadTimeUSecs; }; #endif // __voxel_server__VoxelPersistThread__ diff --git a/libraries/voxel-server-library/src/VoxelSendThread.cpp b/libraries/voxel-server-library/src/VoxelSendThread.cpp index 4455f1e379..4bfed9c9ad 100644 --- a/libraries/voxel-server-library/src/VoxelSendThread.cpp +++ b/libraries/voxel-server-library/src/VoxelSendThread.cpp @@ -26,28 +26,35 @@ VoxelSendThread::VoxelSendThread(const QUuid& nodeUUID, VoxelServer* myServer) : bool VoxelSendThread::process() { uint64_t start = usecTimestampNow(); - Node* node = NodeList::getInstance()->nodeWithUUID(_nodeUUID); + // don't do any send processing until the initial load of the voxels is complete... + if (_myServer->isInitialLoadComplete()) { + Node* node = NodeList::getInstance()->nodeWithUUID(_nodeUUID); - if (node) { - node->lock(); // make sure the node list doesn't kill our node while we're using it - VoxelNodeData* nodeData = NULL; + if (node) { + node->lock(); // make sure the node list doesn't kill our node while we're using it + VoxelNodeData* nodeData = NULL; - nodeData = (VoxelNodeData*) node->getLinkedData(); + nodeData = (VoxelNodeData*) node->getLinkedData(); - int packetsSent = 0; + int packetsSent = 0; - // Sometimes the node data has not yet been linked, in which case we can't really do anything - if (nodeData) { - bool viewFrustumChanged = nodeData->updateCurrentViewFrustum(); - if (_myServer->wantsDebugVoxelSending()) { - printf("nodeData->updateCurrentViewFrustum() changed=%s\n", debug::valueOf(viewFrustumChanged)); + // Sometimes the node data has not yet been linked, in which case we can't really do anything + if (nodeData) { + bool viewFrustumChanged = nodeData->updateCurrentViewFrustum(); + if (_myServer->wantsDebugVoxelSending()) { + printf("nodeData->updateCurrentViewFrustum() changed=%s\n", debug::valueOf(viewFrustumChanged)); + } + packetsSent = deepestLevelVoxelDistributor(node, nodeData, viewFrustumChanged); } - packetsSent = deepestLevelVoxelDistributor(node, nodeData, viewFrustumChanged); + + node->unlock(); // we're done with this node for now. + } + } else { + if (_myServer->wantsDebugVoxelSending()) { + qDebug("VoxelSendThread::process() waiting for isInitialLoadComplete()\n"); } - - node->unlock(); // we're done with this node for now. } - + // dynamically sleep until we need to fire off the next set of voxels int elapsed = (usecTimestampNow() - start); int usecToSleep = VOXEL_SEND_INTERVAL_USECS - elapsed; @@ -117,8 +124,6 @@ int VoxelSendThread::handlePacketSend(Node* node, VoxelNodeData* nodeData, int& /// Version of voxel distributor that sends the deepest LOD level at once int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nodeData, bool viewFrustumChanged) { - _myServer->getServerTree().lockForRead(); - int truePacketsSent = 0; int trueBytesSent = 0; int packetsSentThisInterval = 0; @@ -289,10 +294,13 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod nodeData->getLastTimeBagEmpty(), isFullScene, &nodeData->stats, _myServer->getJurisdiction()); + + _myServer->getServerTree().lockForRead(); nodeData->stats.encodeStarted(); bytesWritten = _myServer->getServerTree().encodeTreeBitstream(subTree, _tempOutputBuffer, MAX_VOXEL_PACKET_SIZE - 1, nodeData->nodeBag, params); nodeData->stats.encodeStopped(); + _myServer->getServerTree().unlock(); if (nodeData->getAvailable() >= bytesWritten) { nodeData->writeToPacket(_tempOutputBuffer, bytesWritten); @@ -360,8 +368,6 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod } // end if bag wasn't empty, and so we sent stuff... - _myServer->getServerTree().unlock(); - return truePacketsSent; } diff --git a/libraries/voxel-server-library/src/VoxelServer.cpp b/libraries/voxel-server-library/src/VoxelServer.cpp index a5cb8f9620..88fed0e921 100644 --- a/libraries/voxel-server-library/src/VoxelServer.cpp +++ b/libraries/voxel-server-library/src/VoxelServer.cpp @@ -175,6 +175,47 @@ int VoxelServer::civetwebRequestHandler(struct mg_connection* connection) { mg_printf(connection, "%s", "\r\n"); mg_printf(connection, "%s", "\r\n"); + + // display voxel file load time + if (GetInstance()->isInitialLoadComplete()) { + tm* voxelsLoadedAtLocal = localtime(GetInstance()->getLoadCompleted()); + const int MAX_TIME_LENGTH = 128; + char buffer[MAX_TIME_LENGTH]; + strftime(buffer, MAX_TIME_LENGTH, "%m/%d/%Y %X", voxelsLoadedAtLocal); + mg_printf(connection, "Voxels Loaded At: %s", buffer); + + // Convert now to tm struct for UTC + tm* voxelsLoadedAtUTM = gmtime(GetInstance()->getLoadCompleted()); + if (gmtm != NULL) { + strftime(buffer, MAX_TIME_LENGTH, "%m/%d/%Y %X", voxelsLoadedAtUTM); + mg_printf(connection, " [%s UTM] ", buffer); + } + mg_printf(connection, "%s", "\r\n"); + + + uint64_t msecsElapsed = GetInstance()->getLoadElapsedTime() / USECS_PER_MSEC;; + float seconds = (msecsElapsed % MSECS_PER_MIN)/(float)MSECS_PER_SEC; + int minutes = (msecsElapsed/(MSECS_PER_MIN)) % MIN_PER_HOUR; + int hours = (msecsElapsed/(MSECS_PER_MIN * MIN_PER_HOUR)); + + mg_printf(connection, "%s", "Voxels Load Took: "); + if (hours > 0) { + mg_printf(connection, "%d hour%s ", hours, (hours > 1) ? "s" : "" ); + } + if (minutes > 0) { + mg_printf(connection, "%d minute%s ", minutes, (minutes > 1) ? "s" : ""); + } + if (seconds > 0) { + mg_printf(connection, "%.3f seconds", seconds); + } + mg_printf(connection, "%s", "\r\n"); + + } else { + mg_printf(connection, "%s", "Voxels not yet loaded...\r\n"); + } + + mg_printf(connection, "%s", "\r\n"); + mg_printf(connection, "%s", "\r\n"); mg_printf(connection, "%s", "Configuration: \r\n "); for (int i = 1; i < GetInstance()->_argc; i++) { @@ -512,14 +553,15 @@ void VoxelServer::run() { // loop to send to nodes requesting data while (true) { - if (NodeList::getInstance()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS) { + qDebug() << "Exit loop... getInstance()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS\n"; break; } // send a check in packet to the domain server if DOMAIN_SERVER_CHECK_IN_USECS has elapsed if (usecTimestampNow() - usecTimestamp(&lastDomainServerCheckIn) >= DOMAIN_SERVER_CHECK_IN_USECS) { gettimeofday(&lastDomainServerCheckIn, NULL); + //qDebug() << "VoxelServer::run()... NodeList::getInstance()->sendDomainServerCheckIn()\n"; NodeList::getInstance()->sendDomainServerCheckIn(); } @@ -569,26 +611,39 @@ void VoxelServer::run() { } } } + qDebug() << "VoxelServer::run()... AFTER loop...\n"; - delete _jurisdiction; + // call NodeList::clear() so that all of our node specific objects, including our sending threads, are + // properly shutdown and cleaned up. + NodeList::getInstance()->clear(); + qDebug() << "VoxelServer::run()... terminating _jurisdictionSender\n"; if (_jurisdictionSender) { _jurisdictionSender->terminate(); delete _jurisdictionSender; } + qDebug() << "VoxelServer::run()... terminating _voxelServerPacketProcessor\n"; if (_voxelServerPacketProcessor) { _voxelServerPacketProcessor->terminate(); delete _voxelServerPacketProcessor; } + qDebug() << "VoxelServer::run()... terminating _voxelPersistThread\n"; if (_voxelPersistThread) { _voxelPersistThread->terminate(); delete _voxelPersistThread; } // tell our NodeList we're done with notifications + qDebug() << "VoxelServer::run()... nodeList->removeHook(&_nodeWatcher)\n"; nodeList->removeHook(&_nodeWatcher); + + qDebug() << "VoxelServer::run()... deleting _jurisdiction\n"; + delete _jurisdiction; + _jurisdiction = NULL; + + qDebug() << "VoxelServer::run()... DONE\n"; } diff --git a/libraries/voxel-server-library/src/VoxelServer.h b/libraries/voxel-server-library/src/VoxelServer.h index f938bebd7e..92590489f8 100644 --- a/libraries/voxel-server-library/src/VoxelServer.h +++ b/libraries/voxel-server-library/src/VoxelServer.h @@ -55,6 +55,10 @@ public: static VoxelServer* GetInstance() { return _theInstance; } + bool isInitialLoadComplete() const { return (_voxelPersistThread) ? _voxelPersistThread->isInitialLoadComplete() : true; } + time_t* getLoadCompleted() { return (_voxelPersistThread) ? _voxelPersistThread->getLoadCompleted() : NULL; } + uint64_t getLoadElapsedTime() const { return (_voxelPersistThread) ? _voxelPersistThread->getLoadElapsedTime() : 0; } + private: int _argc; const char** _argv; diff --git a/libraries/voxel-server-library/src/VoxelServerPacketProcessor.cpp b/libraries/voxel-server-library/src/VoxelServerPacketProcessor.cpp index 5fd2028f50..672c4f0d50 100644 --- a/libraries/voxel-server-library/src/VoxelServerPacketProcessor.cpp +++ b/libraries/voxel-server-library/src/VoxelServerPacketProcessor.cpp @@ -85,7 +85,7 @@ void VoxelServerPacketProcessor::processPacket(sockaddr& senderAddress, unsigned printf("inserting voxel: %f,%f,%f r=%d,g=%d,b=%d\n", vertices[0], vertices[1], vertices[2], red, green, blue); delete[] vertices; } - + _myServer->getServerTree().lockForWrite(); _myServer->getServerTree().readCodeColorBufferToTree(voxelData, destructive); _myServer->getServerTree().unlock(); diff --git a/libraries/voxels/src/VoxelQuery.cpp b/libraries/voxels/src/VoxelQuery.cpp index 463b806eba..9b5d65e0c5 100644 --- a/libraries/voxels/src/VoxelQuery.cpp +++ b/libraries/voxels/src/VoxelQuery.cpp @@ -42,6 +42,7 @@ VoxelQuery::VoxelQuery(Node* owningNode) : } VoxelQuery::~VoxelQuery() { + qDebug("VoxelQuery::~VoxelQuery()\n"); } int VoxelQuery::getBroadcastData(unsigned char* destinationBuffer) { diff --git a/libraries/voxels/src/VoxelQuery.h b/libraries/voxels/src/VoxelQuery.h index bedbfdac4e..c95b632eef 100644 --- a/libraries/voxels/src/VoxelQuery.h +++ b/libraries/voxels/src/VoxelQuery.h @@ -35,7 +35,7 @@ class VoxelQuery : public NodeData { public: VoxelQuery(Node* owningNode = NULL); - ~VoxelQuery(); + virtual ~VoxelQuery(); int getBroadcastData(unsigned char* destinationBuffer); int parseData(unsigned char* sourceBuffer, int numBytes); diff --git a/libraries/voxels/src/VoxelTree.cpp b/libraries/voxels/src/VoxelTree.cpp index cfe7828c08..e0e1ac5367 100644 --- a/libraries/voxels/src/VoxelTree.cpp +++ b/libraries/voxels/src/VoxelTree.cpp @@ -1806,9 +1806,10 @@ void VoxelTree::writeToSVOFile(const char* fileName, VoxelNode* node) { while (!nodeBag.isEmpty()) { VoxelNode* subTree = nodeBag.extract(); + lockForRead(); // do tree locking down here so that we have shorter slices and less thread contention EncodeBitstreamParams params(INT_MAX, IGNORE_VIEW_FRUSTUM, WANT_COLOR, NO_EXISTS_BITS); bytesWritten = encodeTreeBitstream(subTree, &outputBuffer[0], MAX_VOXEL_PACKET_SIZE - 1, nodeBag, params); - + unlock(); file.write((const char*)&outputBuffer[0], bytesWritten); } } diff --git a/libraries/voxels/src/VoxelTree.h b/libraries/voxels/src/VoxelTree.h index f9a5c38eb9..f8e0458fa1 100644 --- a/libraries/voxels/src/VoxelTree.h +++ b/libraries/voxels/src/VoxelTree.h @@ -189,7 +189,9 @@ public: // VoxelTree does not currently handle its own locking, caller must use these to lock/unlock void lockForRead() { lock.lockForRead(); } + void tryLockForRead() { lock.tryLockForRead(); } void lockForWrite() { lock.lockForWrite(); } + void tryLockForWrite() { lock.tryLockForWrite(); } void unlock() { lock.unlock(); } unsigned long getVoxelCount(); From 77ba88e56dbfaae9157e761c4d8bf234de8f9581 Mon Sep 17 00:00:00 2001 From: ZappoMan Date: Thu, 7 Nov 2013 13:06:57 -0800 Subject: [PATCH 2/2] removed dead comment --- libraries/voxel-server-library/src/VoxelServer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/voxel-server-library/src/VoxelServer.cpp b/libraries/voxel-server-library/src/VoxelServer.cpp index 88fed0e921..de421341f3 100644 --- a/libraries/voxel-server-library/src/VoxelServer.cpp +++ b/libraries/voxel-server-library/src/VoxelServer.cpp @@ -561,7 +561,6 @@ void VoxelServer::run() { // send a check in packet to the domain server if DOMAIN_SERVER_CHECK_IN_USECS has elapsed if (usecTimestampNow() - usecTimestamp(&lastDomainServerCheckIn) >= DOMAIN_SERVER_CHECK_IN_USECS) { gettimeofday(&lastDomainServerCheckIn, NULL); - //qDebug() << "VoxelServer::run()... NodeList::getInstance()->sendDomainServerCheckIn()\n"; NodeList::getInstance()->sendDomainServerCheckIn(); }