Merge pull request #1215 from ZappoMan/bugfixes

fix thread contention issues and VS crash on domain server shutdown and restart

libraries/voxel-server-library/src/VoxelNodeData.cpp

@@ -129,6 +129,7 @@ VoxelNodeData::~VoxelNodeData() {
     if (_voxelSendThread) {
         _voxelSendThread->terminate();
         delete _voxelSendThread;
+        qDebug("VoxelNodeData::~VoxelNodeData() DELETED _voxelSendThread=%p\n", _voxelSendThread);
     }
 }
 

libraries/voxel-server-library/src/VoxelNodeData.h

@@ -24,7 +24,7 @@ class VoxelServer;
 class VoxelNodeData : public VoxelQuery {
 public:
     VoxelNodeData(Node* owningNode);
-    ~VoxelNodeData();
+    virtual ~VoxelNodeData();
 
     void resetVoxelPacket();  // resets voxel packet to after "V" header
 

libraries/voxel-server-library/src/VoxelPersistThread.cpp

@@ -20,34 +20,28 @@ VoxelPersistThread::VoxelPersistThread(VoxelTree* tree, const char* filename, in
     _tree(tree),
     _filename(filename),
     _persistInterval(persistInterval),
-    _initialLoad(false) {
+    _initialLoadComplete(false),
+    _loadTimeUSecs(0) {
 }
 
 bool VoxelPersistThread::process() {
 
-    if (!_initialLoad) {
+    if (!_initialLoadComplete) {
-        _initialLoad = true;
+        uint64_t loadStarted = usecTimestampNow();
         qDebug("loading voxels from file: %s...\n", _filename);
 
         bool persistantFileRead;
-        
+
+        _tree->lockForWrite();
         {
             PerformanceWarning warn(true, "Loading Voxel File", true);
-            _tree->lockForRead();
             persistantFileRead = _tree->readFromSVOFile(_filename);
-            _tree->unlock();
         }
+        _tree->unlock();
 
-        if (persistantFileRead) {
+        _loadCompleted = time(0);
-            PerformanceWarning warn(true, "Voxels Re-Averaging", true);
+        uint64_t loadDone = usecTimestampNow();
-            
+        _loadTimeUSecs = loadDone - loadStarted;
-            // after done inserting all these voxels, then reaverage colors
-            qDebug("BEGIN Voxels Re-Averaging\n");
-            _tree->lockForWrite();
-            _tree->reaverageVoxelColors(_tree->rootNode);
-            _tree->unlock();
-            qDebug("DONE WITH Voxels Re-Averaging\n");
-        }
         
         _tree->clearDirtyBit(); // the tree is clean since we just loaded it
         qDebug("DONE loading voxels from file... fileRead=%s\n", debug::valueOf(persistantFileRead));
@@ -65,6 +59,7 @@ bool VoxelPersistThread::process() {
         qDebug("setChildAtIndexCalls=%llu setChildAtIndexTime=%llu perSet=%lf\n",
             VoxelNode::getSetChildAtIndexTime(), VoxelNode::getSetChildAtIndexCalls(), usecPerSet);
 
+        _initialLoadComplete = true;
     }
     
     uint64_t MSECS_TO_USECS = 1000;
@@ -74,9 +69,7 @@ bool VoxelPersistThread::process() {
     // check the dirty bit and persist here...
     if (_tree->isDirty()) {
         qDebug("saving voxels to file %s...\n",_filename);
-        _tree->lockForRead();
         _tree->writeToSVOFile(_filename);
-        _tree->unlock();
         _tree->clearDirtyBit(); // tree is clean after saving
         qDebug("DONE saving voxels to file...\n");
     }

libraries/voxel-server-library/src/VoxelPersistThread.h

@@ -21,6 +21,12 @@ public:
     static const int DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
 
     VoxelPersistThread(VoxelTree* tree, const char* filename, int persistInterval = DEFAULT_PERSIST_INTERVAL);
+    
+    bool isInitialLoadComplete() const { return _initialLoadComplete; }
+
+    time_t* getLoadCompleted() { return &_loadCompleted; }
+    uint64_t getLoadElapsedTime() const { return _loadTimeUSecs; }
+
 protected:
     /// Implements generic processing behavior for this thread.
     virtual bool process();
@@ -28,7 +34,10 @@ private:
     VoxelTree* _tree;
     const char* _filename;
     int _persistInterval;
-    bool _initialLoad;
+    bool _initialLoadComplete;
+
+    time_t _loadCompleted;
+    uint64_t _loadTimeUSecs;
 };
 
 #endif // __voxel_server__VoxelPersistThread__

libraries/voxel-server-library/src/VoxelSendThread.cpp

@@ -26,28 +26,35 @@ VoxelSendThread::VoxelSendThread(const QUuid& nodeUUID, VoxelServer* myServer) :
 bool VoxelSendThread::process() {
     uint64_t  start = usecTimestampNow();
     
-    Node* node = NodeList::getInstance()->nodeWithUUID(_nodeUUID);
+    // don't do any send processing until the initial load of the voxels is complete...
+    if (_myServer->isInitialLoadComplete()) {
+        Node* node = NodeList::getInstance()->nodeWithUUID(_nodeUUID);
     
-    if (node) {
+        if (node) {
-        node->lock(); // make sure the node list doesn't kill our node while we're using it
+            node->lock(); // make sure the node list doesn't kill our node while we're using it
-        VoxelNodeData* nodeData = NULL;
+            VoxelNodeData* nodeData = NULL;
     
-        nodeData = (VoxelNodeData*) node->getLinkedData();
+            nodeData = (VoxelNodeData*) node->getLinkedData();
     
-        int packetsSent = 0;
+            int packetsSent = 0;
 
-        // Sometimes the node data has not yet been linked, in which case we can't really do anything
+            // Sometimes the node data has not yet been linked, in which case we can't really do anything
-        if (nodeData) {
+            if (nodeData) {
-            bool viewFrustumChanged = nodeData->updateCurrentViewFrustum();
+                bool viewFrustumChanged = nodeData->updateCurrentViewFrustum();
-            if (_myServer->wantsDebugVoxelSending()) {
+                if (_myServer->wantsDebugVoxelSending()) {
-                printf("nodeData->updateCurrentViewFrustum() changed=%s\n", debug::valueOf(viewFrustumChanged));
+                    printf("nodeData->updateCurrentViewFrustum() changed=%s\n", debug::valueOf(viewFrustumChanged));
+                }
+                packetsSent = deepestLevelVoxelDistributor(node, nodeData, viewFrustumChanged);
             }
-            packetsSent = deepestLevelVoxelDistributor(node, nodeData, viewFrustumChanged);
+    
+            node->unlock(); // we're done with this node for now.
+        }
+    } else {
+        if (_myServer->wantsDebugVoxelSending()) {
+            qDebug("VoxelSendThread::process() waiting for isInitialLoadComplete()\n");
         }
-    
-        node->unlock(); // we're done with this node for now.
     }
-    
+        
     // dynamically sleep until we need to fire off the next set of voxels
     int elapsed = (usecTimestampNow() - start);
     int usecToSleep =  VOXEL_SEND_INTERVAL_USECS - elapsed;
@@ -117,8 +124,6 @@ int VoxelSendThread::handlePacketSend(Node* node, VoxelNodeData* nodeData, int&
 /// Version of voxel distributor that sends the deepest LOD level at once
 int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nodeData, bool viewFrustumChanged) {
 
-    _myServer->getServerTree().lockForRead();
-
     int truePacketsSent = 0;
     int trueBytesSent = 0;
     int packetsSentThisInterval = 0;
@@ -289,10 +294,13 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
                                              nodeData->getLastTimeBagEmpty(),
                                              isFullScene, &nodeData->stats, _myServer->getJurisdiction());
                       
+
+                _myServer->getServerTree().lockForRead();
                 nodeData->stats.encodeStarted();
                 bytesWritten = _myServer->getServerTree().encodeTreeBitstream(subTree, _tempOutputBuffer, MAX_VOXEL_PACKET_SIZE - 1,
                                                               nodeData->nodeBag, params);
                 nodeData->stats.encodeStopped();
+                _myServer->getServerTree().unlock();
 
                 if (nodeData->getAvailable() >= bytesWritten) {
                     nodeData->writeToPacket(_tempOutputBuffer, bytesWritten);
@@ -360,8 +368,6 @@ int VoxelSendThread::deepestLevelVoxelDistributor(Node* node, VoxelNodeData* nod
         
     } // end if bag wasn't empty, and so we sent stuff...
 
-    _myServer->getServerTree().unlock();
-
     return truePacketsSent;
 }
 

libraries/voxel-server-library/src/VoxelServer.cpp

@@ -175,6 +175,47 @@ int VoxelServer::civetwebRequestHandler(struct mg_connection* connection) {
         mg_printf(connection, "%s", "\r\n");
         mg_printf(connection, "%s", "\r\n");
 
+
+        // display voxel file load time
+        if (GetInstance()->isInitialLoadComplete()) {
+            tm* voxelsLoadedAtLocal = localtime(GetInstance()->getLoadCompleted());
+            const int MAX_TIME_LENGTH = 128;
+            char buffer[MAX_TIME_LENGTH];
+            strftime(buffer, MAX_TIME_LENGTH, "%m/%d/%Y %X", voxelsLoadedAtLocal);
+            mg_printf(connection, "Voxels Loaded At: %s", buffer);
+
+            // Convert now to tm struct for UTC
+            tm* voxelsLoadedAtUTM = gmtime(GetInstance()->getLoadCompleted());
+            if (gmtm != NULL) {
+                strftime(buffer, MAX_TIME_LENGTH, "%m/%d/%Y %X", voxelsLoadedAtUTM);
+                mg_printf(connection, " [%s UTM] ", buffer);
+            }
+            mg_printf(connection, "%s", "\r\n");
+
+
+            uint64_t msecsElapsed = GetInstance()->getLoadElapsedTime() / USECS_PER_MSEC;;
+            float seconds = (msecsElapsed % MSECS_PER_MIN)/(float)MSECS_PER_SEC;
+            int minutes = (msecsElapsed/(MSECS_PER_MIN)) % MIN_PER_HOUR;
+            int hours = (msecsElapsed/(MSECS_PER_MIN * MIN_PER_HOUR));
+
+            mg_printf(connection, "%s", "Voxels Load Took: ");
+            if (hours > 0) {
+                mg_printf(connection, "%d hour%s ", hours, (hours > 1) ? "s" : "" );
+            }
+            if (minutes > 0) {
+                mg_printf(connection, "%d minute%s ", minutes, (minutes > 1) ? "s" : "");
+            }
+            if (seconds > 0) {
+                mg_printf(connection, "%.3f seconds", seconds);
+            }
+            mg_printf(connection, "%s", "\r\n");
+
+        } else {
+            mg_printf(connection, "%s", "Voxels not yet loaded...\r\n");
+        }
+
+        mg_printf(connection, "%s", "\r\n");
+
         mg_printf(connection, "%s", "\r\n");
         mg_printf(connection, "%s", "Configuration: \r\n     ");
         for (int i = 1; i < GetInstance()->_argc; i++) {
@@ -512,8 +553,8 @@ void VoxelServer::run() {
     
     // loop to send to nodes requesting data
     while (true) {
-    
         if (NodeList::getInstance()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS) {
+            qDebug() << "Exit loop... getInstance()->getNumNoReplyDomainCheckIns() == MAX_SILENT_DOMAIN_SERVER_CHECK_INS\n";
             break;
         }
         
@@ -569,26 +610,39 @@ void VoxelServer::run() {
             }
         }
     }
+    qDebug() << "VoxelServer::run()... AFTER loop...\n";
     
-    delete _jurisdiction;
+    // call NodeList::clear() so that all of our node specific objects, including our sending threads, are
+    // properly shutdown and cleaned up.
+    NodeList::getInstance()->clear();
     
+    qDebug() << "VoxelServer::run()... terminating _jurisdictionSender\n";
     if (_jurisdictionSender) {
         _jurisdictionSender->terminate();
         delete _jurisdictionSender;
     }
 
+    qDebug() << "VoxelServer::run()... terminating _voxelServerPacketProcessor\n";
     if (_voxelServerPacketProcessor) {
         _voxelServerPacketProcessor->terminate();
         delete _voxelServerPacketProcessor;
     }
 
+    qDebug() << "VoxelServer::run()... terminating _voxelPersistThread\n";
     if (_voxelPersistThread) {
         _voxelPersistThread->terminate();
         delete _voxelPersistThread;
     }
     
     // tell our NodeList we're done with notifications
+    qDebug() << "VoxelServer::run()... nodeList->removeHook(&_nodeWatcher)\n";
     nodeList->removeHook(&_nodeWatcher);
+
+    qDebug() << "VoxelServer::run()... deleting _jurisdiction\n";
+    delete _jurisdiction;
+    _jurisdiction = NULL;
+
+    qDebug() << "VoxelServer::run()... DONE\n";
 }
 
 

libraries/voxel-server-library/src/VoxelServer.h

@@ -55,6 +55,10 @@ public:
     
     static VoxelServer* GetInstance() { return _theInstance; }
     
+    bool isInitialLoadComplete() const { return (_voxelPersistThread) ? _voxelPersistThread->isInitialLoadComplete() : true; }
+    time_t* getLoadCompleted() { return (_voxelPersistThread) ? _voxelPersistThread->getLoadCompleted() : NULL; }
+    uint64_t getLoadElapsedTime() const { return (_voxelPersistThread) ? _voxelPersistThread->getLoadElapsedTime() : 0; }
+    
 private:
     int _argc;
     const char** _argv;

libraries/voxel-server-library/src/VoxelServerPacketProcessor.cpp

@@ -85,7 +85,7 @@ void VoxelServerPacketProcessor::processPacket(sockaddr& senderAddress, unsigned
                     printf("inserting voxel: %f,%f,%f r=%d,g=%d,b=%d\n", vertices[0], vertices[1], vertices[2], red, green, blue);
                     delete[] vertices;
                 }
-        
+
                 _myServer->getServerTree().lockForWrite();
                 _myServer->getServerTree().readCodeColorBufferToTree(voxelData, destructive);
                 _myServer->getServerTree().unlock();

libraries/voxels/src/VoxelQuery.cpp

@@ -42,6 +42,7 @@ VoxelQuery::VoxelQuery(Node* owningNode) :
 }
 
 VoxelQuery::~VoxelQuery() {
+    qDebug("VoxelQuery::~VoxelQuery()\n");
 }
 
 int VoxelQuery::getBroadcastData(unsigned char* destinationBuffer) {

libraries/voxels/src/VoxelQuery.h

@@ -35,7 +35,7 @@ class VoxelQuery : public NodeData {
     
 public:
     VoxelQuery(Node* owningNode = NULL);
-    ~VoxelQuery();
+    virtual ~VoxelQuery();
 
     int getBroadcastData(unsigned char* destinationBuffer);
     int parseData(unsigned char* sourceBuffer, int numBytes);

libraries/voxels/src/VoxelTree.cpp

@@ -1806,9 +1806,10 @@ void VoxelTree::writeToSVOFile(const char* fileName, VoxelNode* node) {
         while (!nodeBag.isEmpty()) {
             VoxelNode* subTree = nodeBag.extract();
 
+            lockForRead(); // do tree locking down here so that we have shorter slices and less thread contention
             EncodeBitstreamParams params(INT_MAX, IGNORE_VIEW_FRUSTUM, WANT_COLOR, NO_EXISTS_BITS);
             bytesWritten = encodeTreeBitstream(subTree, &outputBuffer[0], MAX_VOXEL_PACKET_SIZE - 1, nodeBag, params);
-
+            unlock();
             file.write((const char*)&outputBuffer[0], bytesWritten);
         }
     }

libraries/voxels/src/VoxelTree.h

@@ -189,7 +189,9 @@ public:
     
     // VoxelTree does not currently handle its own locking, caller must use these to lock/unlock
     void lockForRead() { lock.lockForRead(); }
+    void tryLockForRead() { lock.tryLockForRead(); }
     void lockForWrite() { lock.lockForWrite(); }
+    void tryLockForWrite() { lock.tryLockForWrite(); }
     void unlock() { lock.unlock(); }
 
     unsigned long getVoxelCount();