Merge pull request #3798 from ZappoMan/persistThreadImprovements

Persist thread improvements/Backup support

assignment-client/src/octree/OctreeServer.cpp

@@ -1020,7 +1020,7 @@ void OctreeServer::readConfiguration() {
     bool noPersist;
     readOptionBool(QString("NoPersist"), settingsSectionObject, noPersist);
     _wantPersist = !noPersist;
-    qDebug("wantPersist=%s", debug::valueOf(_wantPersist));
+    qDebug() << "wantPersist=" << _wantPersist;
 
     if (_wantPersist) {
         QString persistFilename;
@@ -1029,6 +1029,26 @@ void OctreeServer::readConfiguration() {
         }
         strcpy(_persistFilename, qPrintable(persistFilename));
         qDebug("persistFilename=%s", _persistFilename);
+
+        _persistInterval = OctreePersistThread::DEFAULT_PERSIST_INTERVAL;
+        readOptionInt(QString("persistInterval"), settingsSectionObject, _persistInterval);
+        qDebug() << "persistInterval=" << _persistInterval;
+
+        bool noBackup;
+        readOptionBool(QString("NoBackup"), settingsSectionObject, noBackup);
+        _wantBackup = !noBackup;
+        qDebug() << "wantBackup=" << _wantBackup;
+
+        if (_wantBackup) {
+            _backupExtensionFormat = OctreePersistThread::DEFAULT_BACKUP_EXTENSION_FORMAT;
+            readOptionString(QString("backupExtensionFormat"), settingsSectionObject, _backupExtensionFormat);
+            qDebug() << "backupExtensionFormat=" << _backupExtensionFormat;
+
+            _backupInterval = OctreePersistThread::DEFAULT_BACKUP_INTERVAL;
+            readOptionInt(QString("backupInterval"), settingsSectionObject, _backupInterval);
+            qDebug() << "backupInterval=" << _backupInterval;
+        }
+
     } else {
         qDebug("persistFilename= DISABLED");
     }
@@ -1112,7 +1132,8 @@ void OctreeServer::run() {
     if (_wantPersist) {
 
         // now set up PersistThread
-        _persistThread = new OctreePersistThread(_tree, _persistFilename);
+        _persistThread = new OctreePersistThread(_tree, _persistFilename, _persistInterval,
+                                    _wantBackup, _backupInterval, _backupExtensionFormat);
         if (_persistThread) {
             _persistThread->initialize(true);
         }
@@ -1197,6 +1218,9 @@ void OctreeServer::aboutToFinish() {
         qDebug() << qPrintable(_safeServerName) << "server about to finish while node still connected node:" << *node;
         forceNodeShutdown(node);
     }
+    if (_persistThread) {
+        _persistThread->aboutToFinish();
+    }
     qDebug() << qPrintable(_safeServerName) << "server ENDING about to finish...";
 }
 

assignment-client/src/octree/OctreeServer.h

@@ -168,6 +168,11 @@ protected:
     OctreeInboundPacketProcessor* _octreeInboundPacketProcessor;
     OctreePersistThread* _persistThread;
     
+    int _persistInterval;
+    bool _wantBackup;
+    QString _backupExtensionFormat;
+    int _backupInterval;
+
     static OctreeServer* _instance;
 
     time_t _started;

domain-server/resources/describe-settings.json

@@ -276,6 +276,52 @@
     "label": "Entity Server Settings",
     "assignment-types": [6],
     "settings": [
+      {
+        "name": "persistFilename",
+        "label": "Persistant Filename",
+        "help": "the filename for your entities",
+        "placeholder": "resources/models.svo",
+        "default": "resources/models.svo",
+        "advanced": true
+      },
+      {
+        "name": "persistInterval",
+        "label": "Persist Interval",
+        "help": "Interval between persist checks in msecs.",
+        "placeholder": "30000",
+        "default": "30000",
+        "advanced": true
+      },
+      {
+        "name": "NoPersist",
+        "type": "checkbox",
+        "help": "Don't persist your entities to a file.",
+        "default": false,
+        "advanced": true
+      },
+      {
+        "name": "backupExtensionFormat",
+        "label": "Backup File Extension Format:",
+        "help": "Format used to create the extension for the backup of your persisted entities.",
+        "placeholder": ".backup.%Y-%m-%d.%H:%M:%S.%z",
+        "default": ".backup.%Y-%m-%d.%H:%M:%S.%z",
+        "advanced": true
+      },
+      {
+        "name": "backupInterval",
+        "label": "Backup Interval",
+        "help": "Interval between backup checks in msecs.",
+        "placeholder": "1800000",
+        "default": "1800000",
+        "advanced": true
+      },
+      {
+        "name": "NoBackup",
+        "type": "checkbox",
+        "help": "Don't regularly backup your persisted entities to a backup file.",
+        "default": false,
+        "advanced": true
+      },
       {
         "name": "statusHost",
         "label": "Status Hostname",
@@ -330,7 +376,52 @@
     "label": "Voxel Server Settings",
     "assignment-types": [3],
     "settings": [
-
+      {
+        "name": "persistFilename",
+        "label": "Persistant Filename",
+        "help": "the filename for your voxels",
+        "placeholder": "resources/voxels.svo",
+        "default": "resources/voxels.svo",
+        "advanced": true
+      },
+      {
+        "name": "persistInterval",
+        "label": "Persist Interval",
+        "help": "Interval between persist checks in msecs.",
+        "placeholder": "30000",
+        "default": "30000",
+        "advanced": true
+      },
+      {
+        "name": "NoPersist",
+        "type": "checkbox",
+        "help": "Don't persist your voxels to a file.",
+        "default": false,
+        "advanced": true
+      },
+      {
+        "name": "backupExtensionFormat",
+        "label": "Backup File Extension Format:",
+        "help": "Format used to create the extension for the backup of your persisted voxels.",
+        "placeholder": ".backup.%Y-%m-%d.%H:%M:%S.%z",
+        "default": ".backup.%Y-%m-%d.%H:%M:%S.%z",
+        "advanced": true
+      },
+      {
+        "name": "backupInterval",
+        "label": "Backup Interval",
+        "help": "Interval between backup checks in msecs.",
+        "placeholder": "1800000",
+        "default": "1800000",
+        "advanced": true
+      },
+      {
+        "name": "NoBackup",
+        "type": "checkbox",
+        "help": "Don't regularly backup your persisted voxels to a backup file.",
+        "default": false,
+        "advanced": true
+      },
       {
         "name": "statusHost",
         "label": "Status Hostname",

libraries/entities/src/EntityTree.cpp

@@ -1089,7 +1089,6 @@ bool DebugOperator::preRecursion(OctreeElement* element) {
 }
 
 void EntityTree::dumpTree() {
-    // First, look for the existing entity in the tree..
     DebugOperator theOperator;
     recurseTreeWithOperator(&theOperator);
 }
@@ -1107,7 +1106,6 @@ bool PruneOperator::postRecursion(OctreeElement* element) {
 }
 
 void EntityTree::pruneTree() {
-    // First, look for the existing entity in the tree..
     PruneOperator theOperator;
     recurseTreeWithOperator(&theOperator);
 }

libraries/octree/src/OctreePersistThread.cpp

@@ -9,18 +9,31 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include <time.h>
+
 #include <QDebug>
 #include <PerfStat.h>
 #include <SharedUtil.h>
 
 #include "OctreePersistThread.h"
 
-OctreePersistThread::OctreePersistThread(Octree* tree, const QString& filename, int persistInterval) :
+const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
+const int OctreePersistThread::DEFAULT_BACKUP_INTERVAL = 1000 * 60 * 30; // every 30 minutes
+const QString OctreePersistThread::DEFAULT_BACKUP_EXTENSION_FORMAT(".backup.%Y-%m-%d.%H:%M:%S.%z");
+
+
+OctreePersistThread::OctreePersistThread(Octree* tree, const QString& filename, int persistInterval, 
+                                                bool wantBackup, int backupInterval, const QString& backupExtensionFormat) :
     _tree(tree),
     _filename(filename),
+    _backupExtensionFormat(backupExtensionFormat),
     _persistInterval(persistInterval),
+    _backupInterval(backupInterval),
     _initialLoadComplete(false),
-    _loadTimeUSecs(0) 
+    _loadTimeUSecs(0),
+    _lastCheck(0),
+    _lastBackup(0),
+    _wantBackup(wantBackup)
 {
 }
 
@@ -51,16 +64,22 @@ bool OctreePersistThread::process() {
         unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
         qDebug("Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
 
-        double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime() / (double)OctreeElement::getGetChildAtIndexCalls();
+        bool wantDebug = false;
-        qDebug() << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
+        if (wantDebug) {
-                << " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
+            double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime() 
+                                    / (double)OctreeElement::getGetChildAtIndexCalls();
+            qDebug() << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
+                    << " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
 
-        double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime() / (double)OctreeElement::getSetChildAtIndexCalls();
+            double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime() 
-        qDebug() << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
+                                    / (double)OctreeElement::getSetChildAtIndexCalls();
-                << " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perset=" << usecPerSet;
+            qDebug() << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
+                    << " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
+        }
 
         _initialLoadComplete = true;
-        _lastCheck = usecTimestampNow(); // we just loaded, no need to save again
+        _lastBackup = _lastCheck = usecTimestampNow(); // we just loaded, no need to save again
+        time(&_lastPersistTime);
 
         emit loadCompleted();
     }
@@ -78,19 +97,62 @@ bool OctreePersistThread::process() {
         quint64 intervalToCheck = _persistInterval * MSECS_TO_USECS;
 
         if (sinceLastSave > intervalToCheck) {
-            // check the dirty bit and persist here...
+            _lastCheck = now;
-            _lastCheck = usecTimestampNow();
+            persist();
-            if (_tree->isDirty()) {
-                qDebug() << "pruning Octree before saving...";
-                _tree->pruneTree();
-                qDebug() << "DONE pruning Octree before saving...";
-            
-                qDebug() << "saving Octree to file " << _filename << "...";
-                _tree->writeToSVOFile(_filename.toLocal8Bit().constData());
-                _tree->clearDirtyBit(); // tree is clean after saving
-                qDebug("DONE saving Octree to file...");
-            }
         }
     }
     return isStillRunning();  // keep running till they terminate us
 }
+
+
+void OctreePersistThread::aboutToFinish() {
+    qDebug() << "Persist thread about to finish...";
+    persist();
+    qDebug() << "Persist thread done with about to finish...";
+}
+
+void OctreePersistThread::persist() {
+    if (_tree->isDirty()) {
+        _tree->lockForWrite();
+        {
+            qDebug() << "pruning Octree before saving...";
+            _tree->pruneTree();
+            qDebug() << "DONE pruning Octree before saving...";
+        }
+        _tree->unlock();
+
+        backup(); // handle backup if requested        
+
+        qDebug() << "saving Octree to file " << _filename << "...";
+        _tree->writeToSVOFile(qPrintable(_filename));
+        time(&_lastPersistTime);
+        _tree->clearDirtyBit(); // tree is clean after saving
+        qDebug() << "DONE saving Octree to file...";
+    }
+}
+
+void OctreePersistThread::backup() {
+    if (_wantBackup) {
+        quint64 now = usecTimestampNow();
+        quint64 sinceLastBackup = now - _lastBackup;
+        quint64 MSECS_TO_USECS = 1000;
+        quint64 intervalToBackup = _backupInterval * MSECS_TO_USECS;
+        
+        if (sinceLastBackup > intervalToBackup) {
+            struct tm* localTime = localtime(&_lastPersistTime);
+
+            char backupExtension[256];
+            strftime(backupExtension, sizeof(backupExtension), qPrintable(_backupExtensionFormat), localTime);
+
+            QString backupFileName = _filename + backupExtension;
+
+            qDebug() << "backing up persist file " << _filename << "to" << backupFileName << "...";
+            int result = rename(qPrintable(_filename), qPrintable(backupFileName));
+            if (result == 0) {
+                qDebug() << "DONE backing up persist file...";
+            } else {
+                qDebug() << "ERROR in backing up persist file...";
+            }
+        }
+    }
+}

libraries/octree/src/OctreePersistThread.h

@@ -22,27 +22,41 @@
 class OctreePersistThread : public GenericThread {
     Q_OBJECT
 public:
-    static const int DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
+    static const int DEFAULT_PERSIST_INTERVAL;
+    static const int DEFAULT_BACKUP_INTERVAL;
+    static const QString DEFAULT_BACKUP_EXTENSION_FORMAT;
 
-    OctreePersistThread(Octree* tree, const QString& filename, int persistInterval = DEFAULT_PERSIST_INTERVAL);
+    OctreePersistThread(Octree* tree, const QString& filename, int persistInterval = DEFAULT_PERSIST_INTERVAL, 
+                                bool wantBackup = false, int backupInterval = DEFAULT_BACKUP_INTERVAL,
+                                const QString& backupExtensionFormat = DEFAULT_BACKUP_EXTENSION_FORMAT);
 
     bool isInitialLoadComplete() const { return _initialLoadComplete; }
     quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
 
+    void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
+
 signals:
     void loadCompleted();
 
 protected:
     /// Implements generic processing behavior for this thread.
     virtual bool process();
+    
+    void persist();
+    void backup();
 private:
     Octree* _tree;
     QString _filename;
+    QString _backupExtensionFormat;
     int _persistInterval;
+    int _backupInterval;
     bool _initialLoadComplete;
 
     quint64 _loadTimeUSecs;
     quint64 _lastCheck;
+    quint64 _lastBackup;
+    bool _wantBackup;
+    time_t _lastPersistTime;
 };
 
 #endif // hifi_OctreePersistThread_h