Merge pull request #13144 from huffman/fix/remove-entity-server-backups

Remove old entity backups and OctreePersistThread refactoring

assignment-client/src/octree/OctreeServer.cpp

@@ -231,18 +231,19 @@ void OctreeServer::trackProcessWaitTime(float time) {
 OctreeServer::OctreeServer(ReceivedMessage& message) :
     ThreadedAssignment(message),
     _argc(0),
-    _argv(NULL),
+    _argv(nullptr),
-    _parsedArgV(NULL),
+    _parsedArgV(nullptr),
+    _httpManager(nullptr),
     _statusPort(0),
     _packetsPerClientPerInterval(10),
     _packetsTotalPerInterval(DEFAULT_PACKETS_PER_INTERVAL),
-    _tree(NULL),
+    _tree(nullptr),
     _wantPersist(true),
     _debugSending(false),
     _debugReceiving(false),
     _verboseDebug(false),
-    _octreeInboundPacketProcessor(NULL),
+    _octreeInboundPacketProcessor(nullptr),
-    _persistThread(NULL),
+    _persistManager(nullptr),
     _started(time(0)),
     _startedUSecs(usecTimestampNow())
 {
@@ -265,11 +266,8 @@ OctreeServer::~OctreeServer() {
         _octreeInboundPacketProcessor->deleteLater();
     }
 
-    if (_persistThread) {
+    qDebug() << "Waiting for persist thread to come down";
-        _persistThread->terminating();
+    _persistThread.wait();
-        _persistThread->terminate();
-        _persistThread->deleteLater();
-    }
 
     // cleanup our tree here...
     qDebug() << qPrintable(_safeServerName) << "server START cleaning up octree... [" << this << "]";
@@ -1052,19 +1050,13 @@ void OctreeServer::readConfiguration() {
         _persistAsFileType = "json.gz";
 
         _persistInterval = OctreePersistThread::DEFAULT_PERSIST_INTERVAL;
-        readOptionInt(QString("persistInterval"), settingsSectionObject, _persistInterval);
+        int result { -1 };
-        qDebug() << "persistInterval=" << _persistInterval;
+        readOptionInt(QString("persistInterval"), settingsSectionObject, result);
-
+        if (result != -1) {
-        bool noBackup;
+            _persistInterval = std::chrono::milliseconds(result);
-        readOptionBool(QString("NoBackup"), settingsSectionObject, noBackup);
-        _wantBackup = !noBackup;
-        qDebug() << "wantBackup=" << _wantBackup;
-
-        if (!readOptionString("backupDirectoryPath", settingsSectionObject, _backupDirectoryPath)) {
-            _backupDirectoryPath = "";
         }
 
-        qDebug() << "backupDirectoryPath=" << _backupDirectoryPath;
+        qDebug() << "persistInterval=" << _persistInterval.count();
 
         readOptionBool(QString("persistFileDownload"), settingsSectionObject, _persistFileDownload);
         qDebug() << "persistFileDownload=" << _persistFileDownload;
@@ -1128,111 +1120,14 @@ void OctreeServer::run() {
 }
 
 void OctreeServer::domainSettingsRequestComplete() {
-    if (_state != OctreeServerState::WaitingForDomainSettings) {
-        qCWarning(octree_server) << "Received domain settings after they have already been received";
-        return;
-    }
-
     auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
     packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
-
+    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
-    packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
 
     qDebug(octree_server) << "Received domain settings";
 
     readConfiguration();
 
-    _state = OctreeServerState::WaitingForOctreeDataNegotation;
-
-    auto nodeList = DependencyManager::get<NodeList>();
-    const DomainHandler& domainHandler = nodeList->getDomainHandler();
-
-    auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
-
-    OctreeUtils::RawOctreeData data;
-    qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
-    if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
-        qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
-        packet->writePrimitive(true);
-        auto id = data.id.toRfc4122();
-        packet->write(id);
-        packet->writePrimitive(data.version);
-    } else {
-        qCWarning(octree_server) << "No octree data found";
-        packet->writePrimitive(false);
-    }
-
-    qCDebug(octree_server) << "Sending request for octree data to DS";
-    nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
-} 
-
-void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
-    if (_state != OctreeServerState::WaitingForOctreeDataNegotation) {
-        qCWarning(octree_server) << "Server received ocree data file reply but is not currently negotiating.";
-        return;
-    }
-
-    bool includesNewData;
-    message->readPrimitive(&includesNewData);
-    QByteArray replaceData;
-    if (includesNewData) {
-        replaceData = message->readAll();
-        qDebug() << "Got reply to octree data file request, new data sent";
-    } else {
-        qDebug() << "Got reply to octree data file request, current entity data is sufficient";
-        
-        OctreeUtils::RawEntityData data;
-        qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
-        if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
-            if (data.id.isNull()) {
-                qCDebug(octree_server) << "Current octree data has a null id, updating";
-                data.resetIdAndVersion();
-
-                QFile file(_persistAbsoluteFilePath);
-                if (file.open(QIODevice::WriteOnly)) {
-                    auto entityData = data.toGzippedByteArray();
-                    file.write(entityData);
-                    file.close();
-                } else {
-                    qCDebug(octree_server) << "Failed to update octree data";
-                }
-            }
-        }
-    }
-
-    _state = OctreeServerState::Running;
-    beginRunning(replaceData);
-}
-
-void OctreeServer::beginRunning(QByteArray replaceData) {
-    if (_state != OctreeServerState::Running) {
-        qCWarning(octree_server) << "Server is not running";
-        return;
-    }
-
-    auto nodeList = DependencyManager::get<NodeList>();
-
-    // we need to ask the DS about agents so we can ping/reply with them
-    nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
-
-    beforeRun(); // after payload has been processed
-
-    connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
-    connect(nodeList.data(), SIGNAL(nodeKilled(SharedNodePointer)), SLOT(nodeKilled(SharedNodePointer)));
-
-#ifndef WIN32
-    setvbuf(stdout, NULL, _IOLBF, 0);
-#endif
-
-    nodeList->linkedDataCreateCallback = [this](Node* node) {
-        auto queryNodeData = createOctreeQueryNode();
-        queryNodeData->init();
-        node->setLinkedData(std::move(queryNodeData));
-    };
-
-    srand((unsigned)time(0));
-
     // if we want Persistence, set up the local file and persist thread
     if (_wantPersist) {
         static const QString ENTITY_PERSIST_EXTENSION = ".json.gz";
@@ -1288,40 +1183,40 @@ void OctreeServer::beginRunning(QByteArray replaceData) {
         }
 
         auto persistFileDirectory = QFileInfo(_persistAbsoluteFilePath).absolutePath();
-        if (_backupDirectoryPath.isEmpty()) {
-            // Use the persist file's directory to store backups
-            _backupDirectoryPath = persistFileDirectory;
-        } else {
-            // The backup directory has been set.
-            //   If relative, make it relative to the entities directory in the application data directory
-            //   If absolute, no resolution is necessary
-            QDir backupDirectory { _backupDirectoryPath };
-            QString absoluteBackupDirectory;
-            if (backupDirectory.isRelative()) {
-                absoluteBackupDirectory = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_backupDirectoryPath);
-                absoluteBackupDirectory = QDir(absoluteBackupDirectory).absolutePath();
-            } else {
-                absoluteBackupDirectory = backupDirectory.absolutePath();
-            }
-            backupDirectory = QDir(absoluteBackupDirectory);
-            if (!backupDirectory.exists()) {
-                if (backupDirectory.mkpath(".")) {
-                    qDebug() << "Created backup directory";
-                } else {
-                    qDebug() << "ERROR creating backup directory, using persist file directory";
-                    _backupDirectoryPath = persistFileDirectory;
-                }
-            } else {
-                _backupDirectoryPath = absoluteBackupDirectory;
-            }
-        }
-        qDebug() << "Backups will be stored in: " << _backupDirectoryPath;
 
         // now set up PersistThread
-        _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval,
+        _persistManager = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _persistInterval, _debugTimestampNow,
-                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData);
+                                                 _persistAsFileType);
-        _persistThread->initialize(true);
+        _persistManager->moveToThread(&_persistThread);
+        connect(&_persistThread, &QThread::finished, _persistManager, &QObject::deleteLater);
+        connect(&_persistThread, &QThread::started, _persistManager, &OctreePersistThread::start);
+        connect(_persistManager, &OctreePersistThread::loadCompleted, this, [this]() {
+            beginRunning();
+        });
+        _persistThread.start();
+    } else {
+        beginRunning();
     }
+} 
+
+void OctreeServer::beginRunning() {
+    auto nodeList = DependencyManager::get<NodeList>();
+
+    // we need to ask the DS about agents so we can ping/reply with them
+    nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
+
+    beforeRun(); // after payload has been processed
+
+    connect(nodeList.data(), &NodeList::nodeAdded, this, &OctreeServer::nodeAdded);
+    connect(nodeList.data(), &NodeList::nodeKilled, this, &OctreeServer::nodeKilled);
+
+    nodeList->linkedDataCreateCallback = [this](Node* node) {
+        auto queryNodeData = createOctreeQueryNode();
+        queryNodeData->init();
+        node->setLinkedData(std::move(queryNodeData));
+    };
+
+    srand((unsigned)time(0));
 
     // set up our OctreeServerPacketProcessor
     _octreeInboundPacketProcessor = new OctreeInboundPacketProcessor(this);
@@ -1384,7 +1279,7 @@ void OctreeServer::aboutToFinish() {
 
     qDebug() << qPrintable(_safeServerName) << "inform Octree Inbound Packet Processor that we are shutting down...";
 
-    // we're going down - set the NodeList linkedDataCallback to NULL so we do not create any more OctreeQueryNode objects.
+    // we're going down - set the NodeList linkedDataCallback to nullptr so we do not create any more OctreeQueryNode objects.
     // This ensures that we don't get any more newly connecting nodes
     DependencyManager::get<NodeList>()->linkedDataCreateCallback = nullptr;
 
@@ -1402,9 +1297,8 @@ void OctreeServer::aboutToFinish() {
     // which waits on the thread to be done before returning
     _sendThreads.clear(); // Cleans up all the send threads.
 
-    if (_persistThread) {
+    if (_persistManager) {
-        _persistThread->aboutToFinish();
+        _persistThread.quit();
-        _persistThread->terminating();
     }
 
     qDebug() << qPrintable(_safeServerName) << "server ENDING about to finish...";

assignment-client/src/octree/OctreeServer.h

@@ -33,12 +33,6 @@ Q_DECLARE_LOGGING_CATEGORY(octree_server)
 
 const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total
 
-enum class OctreeServerState {
-    WaitingForDomainSettings,
-    WaitingForOctreeDataNegotation,
-    Running
-};
-
 /// Handles assignments of type OctreeServer - sending octrees to various clients.
 class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler {
     Q_OBJECT
@@ -46,8 +40,6 @@ public:
     OctreeServer(ReceivedMessage& message);
     ~OctreeServer();
 
-    OctreeServerState _state { OctreeServerState::WaitingForDomainSettings };
-
     /// allows setting of run arguments
     void setArguments(int argc, char** argv);
 
@@ -68,12 +60,12 @@ public:
     static void clientConnected() { _clientCount++; }
     static void clientDisconnected() { _clientCount--; }
 
-    bool isInitialLoadComplete() const { return (_persistThread) ? _persistThread->isInitialLoadComplete() : true; }
+    bool isInitialLoadComplete() const { return (_persistManager) ? _persistManager->isInitialLoadComplete() : true; }
-    bool isPersistEnabled() const { return (_persistThread) ? true : false; }
+    bool isPersistEnabled() const { return (_persistManager) ? true : false; }
-    quint64 getLoadElapsedTime() const { return (_persistThread) ? _persistThread->getLoadElapsedTime() : 0; }
+    quint64 getLoadElapsedTime() const { return (_persistManager) ? _persistManager->getLoadElapsedTime() : 0; }
-    QString getPersistFilename() const { return (_persistThread) ? _persistThread->getPersistFilename() : ""; }
+    QString getPersistFilename() const { return (_persistManager) ? _persistManager->getPersistFilename() : ""; }
-    QString getPersistFileMimeType() const { return (_persistThread) ? _persistThread->getPersistFileMimeType() : "text/plain"; }
+    QString getPersistFileMimeType() const { return (_persistManager) ? _persistManager->getPersistFileMimeType() : "text/plain"; }
-    QByteArray getPersistFileContents() const { return (_persistThread) ? _persistThread->getPersistFileContents() : QByteArray(); }
+    QByteArray getPersistFileContents() const { return (_persistManager) ? _persistManager->getPersistFileContents() : QByteArray(); }
 
     // Subclasses must implement these methods
     virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() = 0;
@@ -149,7 +141,6 @@ private slots:
     void domainSettingsRequestComplete();
     void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
     void removeSendThread();
 
 protected:
@@ -171,7 +162,7 @@ protected:
     QString getConfiguration();
     QString getStatusLink();
 
-    void beginRunning(QByteArray replaceData);
+    void beginRunning();
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
     virtual UniqueSendThread newSendThread(const SharedNodePointer& node) = 0;
@@ -190,7 +181,6 @@ protected:
     QString _persistFilePath;
     QString _persistAbsoluteFilePath;
     QString _persistAsFileType;
-    QString _backupDirectoryPath;
     int _packetsPerClientPerInterval;
     int _packetsTotalPerInterval;
     OctreePointer _tree; // this IS a reaveraging tree
@@ -200,13 +190,11 @@ protected:
     bool _debugTimestampNow;
     bool _verboseDebug;
     OctreeInboundPacketProcessor* _octreeInboundPacketProcessor;
-    OctreePersistThread* _persistThread;
+    OctreePersistThread* _persistManager;
+    QThread _persistThread;
 
-    int _persistInterval;
+    std::chrono::milliseconds _persistInterval;
-    bool _wantBackup;
     bool _persistFileDownload;
-    QString _backupExtensionFormat;
-    int _backupInterval;
     int _maxBackupVersions;
 
     time_t _started;

libraries/octree/src/Octree.cpp

@@ -30,6 +30,7 @@
 #include <QJsonObject>
 #include <QJsonArray>
 #include <QFileInfo>
+#include <QSaveFile>
 #include <QString>
 #include <QRegularExpression>
 #include <QRegularExpressionMatch>
@@ -972,12 +973,19 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
         return false;
     }
 
-    QFile persistFile(fileName);
+    QSaveFile persistFile(fileName);
     bool success = false;
     if (persistFile.open(QIODevice::WriteOnly)) {
-        success = persistFile.write(jsonDataForFile) != -1;
+        if (persistFile.write(jsonDataForFile) != -1) {
+            success = persistFile.commit();
+            if (!success) {
+                qCritical() << "Failed to commit to JSON save file:" << persistFile.errorString();
+            }
+        } else {
+            qCritical("Failed to write to JSON file.");
+        }
     } else {
-        qCritical("Could not write to JSON description of entities.");
+        qCritical("Failed to open JSON file for writing.");
     }
 
     return success;

libraries/octree/src/OctreePersistThread.cpp

@@ -26,6 +26,7 @@
 #include <QJsonArray>
 #include <QJsonObject>
 #include <QJsonDocument>
+#include <QRegExp>
 
 #include <NumericalConstants.h>
 #include <PerfStat.h>
@@ -35,32 +36,152 @@
 #include "OctreeUtils.h"
 #include "OctreeDataUtils.h"
 
-const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
+constexpr std::chrono::seconds OctreePersistThread::DEFAULT_PERSIST_INTERVAL { 30 };
+constexpr std::chrono::milliseconds TIME_BETWEEN_PROCESSING { 10 };
 
-OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval,
+constexpr int MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT { 20 };
-                                         bool wantBackup, const QJsonObject& settings, bool debugTimestampNow,
+constexpr int64_t MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 50 * 1000 * 1000 };
-                                         QString persistAsFileType, const QByteArray& replacementData) :
+
+OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, std::chrono::milliseconds persistInterval,
+                                         bool debugTimestampNow, QString persistAsFileType) :
     _tree(tree),
     _filename(filename),
-    _backupDirectory(backupDirectory),
     _persistInterval(persistInterval),
+    _lastPersistCheck(std::chrono::steady_clock::now()),
     _initialLoadComplete(false),
-    _replacementData(replacementData),
     _loadTimeUSecs(0),
-    _lastCheck(0),
-    _wantBackup(wantBackup),
     _debugTimestampNow(debugTimestampNow),
     _lastTimeDebug(0),
     _persistAsFileType(persistAsFileType)
 {
-    parseSettings(settings);
-
-
     // in case the persist filename has an extension that doesn't match the file type
     QString sansExt = fileNameWithoutExtension(_filename, PERSIST_EXTENSIONS);
     _filename = sansExt + "." + _persistAsFileType;
 }
 
+void OctreePersistThread::start() {
+    cleanupOldReplacementBackups();
+
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    const DomainHandler& domainHandler = nodeList->getDomainHandler();
+
+    auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
+
+    OctreeUtils::RawOctreeData data;
+    qCDebug(octree) << "Reading octree data from" << _filename;
+    if (data.readOctreeDataInfoFromFile(_filename)) {
+        qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
+        packet->writePrimitive(true);
+        auto id = data.id.toRfc4122();
+        packet->write(id);
+        packet->writePrimitive(data.version);
+    } else {
+        qCWarning(octree) << "No octree data found";
+        packet->writePrimitive(false);
+    }
+
+    qCDebug(octree) << "Sending OctreeDataFileRequest to DS";
+    nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
+}
+
+void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
+    if (_initialLoadComplete) {
+        qCWarning(octree) << "Received OctreeDataFileReply after initial load had completed";
+        return;
+    }
+
+    bool includesNewData;
+    message->readPrimitive(&includesNewData);
+    QByteArray replacementData;
+    OctreeUtils::RawOctreeData data;
+    bool hasValidOctreeData { false };
+    if (includesNewData) {
+        replacementData = message->readAll();
+        replaceData(replacementData);
+        hasValidOctreeData = data.readOctreeDataInfoFromFile(_filename);
+        qDebug() << "Got OctreeDataFileReply, new data sent";
+    } else {
+        qDebug() << "Got OctreeDataFileReply, current entity data is sufficient";
+        
+        OctreeUtils::RawEntityData data;
+        qCDebug(octree) << "Reading octree data from" << _filename;
+        if (data.readOctreeDataInfoFromFile(_filename)) {
+            hasValidOctreeData = true;
+            if (data.id.isNull()) {
+                qCDebug(octree) << "Current octree data has a null id, updating";
+                data.resetIdAndVersion();
+
+                QFile file(_filename);
+                if (file.open(QIODevice::WriteOnly)) {
+                    auto entityData = data.toGzippedByteArray();
+                    file.write(entityData);
+                    file.close();
+                } else {
+                    qCDebug(octree) << "Failed to update octree data";
+                }
+            }
+        }
+    }
+
+    quint64 loadStarted = usecTimestampNow();
+    qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
+
+    if (hasValidOctreeData) {
+        qDebug() << "Setting entity version info to: " << data.id << data.version;
+        _tree->setOctreeVersionInfo(data.id, data.version);
+    }
+
+    bool persistentFileRead;
+
+    _tree->withWriteLock([&] {
+        PerformanceWarning warn(true, "Loading Octree File", true);
+
+        persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
+        _tree->pruneTree();
+    });
+
+    quint64 loadDone = usecTimestampNow();
+    _loadTimeUSecs = loadDone - loadStarted;
+
+    _tree->clearDirtyBit(); // the tree is clean since we just loaded it
+    qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
+
+    unsigned long nodeCount = OctreeElement::getNodeCount();
+    unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
+    unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
+    qCDebug(octree, "Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
+
+    bool wantDebug = false;
+    if (wantDebug) {
+        double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime() 
+                                / (double)OctreeElement::getGetChildAtIndexCalls();
+        qCDebug(octree) << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
+                << " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
+
+        double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime() 
+                                / (double)OctreeElement::getSetChildAtIndexCalls();
+        qCDebug(octree) << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
+                << " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
+    }
+
+    _initialLoadComplete = true;
+
+    // Since we just loaded the persistent file, we can consider ourselves as having just persisted
+    _lastPersistCheck = std::chrono::steady_clock::now();
+
+    if (replacementData.isNull()) {
+        sendLatestEntityDataToDS();
+    }
+
+    QTimer::singleShot(TIME_BETWEEN_PROCESSING.count(), this, &OctreePersistThread::process);
+
+    emit loadCompleted();
+}
+
+
 QString OctreePersistThread::getPersistFileMimeType() const {
     if (_persistAsFileType == "json") {
         return "application/json";
@@ -70,72 +191,6 @@ QString OctreePersistThread::getPersistFileMimeType() const {
     return "";
 }
 
-void OctreePersistThread::parseSettings(const QJsonObject& settings) {
-    if (settings["backups"].isArray()) {
-        const QJsonArray& backupRules = settings["backups"].toArray();
-        qCDebug(octree) << "BACKUP RULES:";
-
-        foreach (const QJsonValue& value, backupRules) {
-
-            QJsonObject obj = value.toObject();
-
-            int interval = 0;
-            int count = 0;
-            
-            QJsonValue intervalVal = obj["backupInterval"];
-            if (intervalVal.isString()) {
-                interval = intervalVal.toString().toInt();
-            } else {
-                interval = intervalVal.toInt();
-            }
-
-            QJsonValue countVal = obj["maxBackupVersions"];
-            if (countVal.isString()) {
-                count = countVal.toString().toInt();
-            } else {
-                count = countVal.toInt();
-            }
-
-            qCDebug(octree) << "    Name:" << obj["Name"].toString();
-            qCDebug(octree) << "        format:" << obj["format"].toString();
-            qCDebug(octree) << "        interval:" << interval;
-            qCDebug(octree) << "        count:" << count;
-
-            BackupRule newRule = { obj["Name"].toString(), interval, obj["format"].toString(), count, 0};
-                                    
-            newRule.lastBackup = getMostRecentBackupTimeInUsecs(obj["format"].toString());
-            
-            if (newRule.lastBackup > 0) {
-                quint64 now = usecTimestampNow();
-                quint64 sinceLastBackup = now - newRule.lastBackup;
-                qCDebug(octree) << "        lastBackup:" << qPrintable(formatUsecTime(sinceLastBackup)) << "ago";
-            } else {
-                qCDebug(octree) << "        lastBackup: NEVER";
-            }
-            
-            _backupRules << newRule;
-        }        
-    } else {
-        qCDebug(octree) << "BACKUP RULES: NONE";
-    }
-}
-
-quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& format) {
-
-    quint64 mostRecentBackupInUsecs = 0;
-
-    QString mostRecentBackupFileName;
-    QDateTime mostRecentBackupTime;
-    
-    bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
-    
-    if (recentBackup) {
-        mostRecentBackupInUsecs = mostRecentBackupTime.toMSecsSinceEpoch() * USECS_PER_MSEC;
-    }
-
-    return mostRecentBackupInUsecs;
-}
-
 void OctreePersistThread::replaceData(QByteArray data) {
     backupCurrentFile();
 
@@ -167,123 +222,24 @@ bool OctreePersistThread::backupCurrentFile() {
     return true;
 }
 
-bool OctreePersistThread::process() {
+void OctreePersistThread::process() {
+    _tree->update();
 
-    if (!_initialLoadComplete) {
+    auto now = std::chrono::steady_clock::now();
-        quint64 loadStarted = usecTimestampNow();
+    auto timeSinceLastPersist = now - _lastPersistCheck;
-        qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
 
-        if (!_replacementData.isNull()) {
+    if (timeSinceLastPersist > _persistInterval) {
-            replaceData(_replacementData);
+        _lastPersistCheck = now;
-        }
+        persist();
-
-        OctreeUtils::RawOctreeData data;
-        if (data.readOctreeDataInfoFromFile(_filename)) {
-            qDebug() << "Setting entity version info to: " << data.id << data.dataVersion;
-            _tree->setOctreeVersionInfo(data.id, data.dataVersion);
-        }
-
-        bool persistentFileRead;
-
-        _tree->withWriteLock([&] {
-            PerformanceWarning warn(true, "Loading Octree File", true);
-
-            // First check to make sure "lock" file doesn't exist. If it does exist, then
-            // our last save crashed during the save, and we want to load our most recent backup.
-            QString lockFileName = _filename + ".lock";
-            std::ifstream lockFile(qPrintable(lockFileName), std::ios::in | std::ios::binary | std::ios::ate);
-            if (lockFile.is_open()) {
-                qCDebug(octree) << "WARNING: Octree lock file detected at startup:" << lockFileName;
-
-                lockFile.close();
-                qCDebug(octree) << "Removing lock file:" << lockFileName;
-                remove(qPrintable(lockFileName));
-                qCDebug(octree) << "Lock file removed:" << lockFileName;
-            }
-
-            persistentFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
-            _tree->pruneTree();
-        });
-
-        quint64 loadDone = usecTimestampNow();
-        _loadTimeUSecs = loadDone - loadStarted;
-
-        _tree->clearDirtyBit(); // the tree is clean since we just loaded it
-        qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
-
-        unsigned long nodeCount = OctreeElement::getNodeCount();
-        unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
-        unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
-        qCDebug(octree, "Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
-
-        bool wantDebug = false;
-        if (wantDebug) {
-            double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime() 
-                                    / (double)OctreeElement::getGetChildAtIndexCalls();
-            qCDebug(octree) << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
-                    << " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
-
-            double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime() 
-                                    / (double)OctreeElement::getSetChildAtIndexCalls();
-            qCDebug(octree) << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
-                    << " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
-        }
-
-        _initialLoadComplete = true;
-
-        // Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance.
-        _lastCheck = usecTimestampNow(); // we just loaded, no need to save again
-        
-        // This last persist time is not really used until the file is actually persisted. It is only
-        // used in formatting the backup filename in cases of non-rolling backup names. However, we don't
-        // want an uninitialized value for this, so we set it to the current time (startup of the server)
-        time(&_lastPersistTime);
-
-        if (_replacementData.isNull()) {
-            sendLatestEntityDataToDS();
-        }
-        _replacementData.clear();
-
-        emit loadCompleted();
     }
 
-    if (isStillRunning()) {
+    QTimer::singleShot(TIME_BETWEEN_PROCESSING.count(), this, &OctreePersistThread::process);
-        quint64 MSECS_TO_USECS = 1000;
-        quint64 USECS_TO_SLEEP = 10 * MSECS_TO_USECS; // every 10ms
-        std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
-
-        // do our updates then check to save...
-        _tree->update();
-
-        quint64 now = usecTimestampNow();
-        quint64 sinceLastSave = now - _lastCheck;
-        quint64 intervalToCheck = _persistInterval * MSECS_TO_USECS;
-
-        if (sinceLastSave > intervalToCheck) {
-            _lastCheck = now;
-            persist();
-        }
-    }
-    
-    // if we were asked to debugTimestampNow do that now...
-    if (_debugTimestampNow) {
-        quint64 now = usecTimestampNow();
-        quint64 sinceLastDebug = now - _lastTimeDebug;
-        quint64 DEBUG_TIMESTAMP_INTERVAL = 600000000; // every 10 minutes
-
-        if (sinceLastDebug > DEBUG_TIMESTAMP_INTERVAL) {
-            _lastTimeDebug = usecTimestampNow(true); // ask for debug output
-        }
-        
-    }
-    return isStillRunning();  // keep running till they terminate us
 }
 
 void OctreePersistThread::aboutToFinish() {
     qCDebug(octree) << "Persist thread about to finish...";
     persist();
     qCDebug(octree) << "Persist thread done with about to finish...";
-    _stopThread = true;
 }
 
 QByteArray OctreePersistThread::getPersistFileContents() const {
@@ -295,6 +251,36 @@ QByteArray OctreePersistThread::getPersistFileContents() const {
     return fileContents;
 }
 
+void OctreePersistThread::cleanupOldReplacementBackups() {
+    QRegExp filenameRegex { ".*\\.backup\\.\\d{8}-\\d{6}$" };
+    QFileInfo persistFile { _filename };
+    QDir backupDir { persistFile.absolutePath() };
+    backupDir.setSorting(QDir::SortFlag::Time);
+    backupDir.setFilter(QDir::Filter::Files);
+    qDebug() << "Scanning backups for cleanup:" << backupDir.absolutePath();
+
+    int count = 0;
+    int64_t totalSize = 0;
+    for (auto fileInfo : backupDir.entryInfoList()) {
+        auto absPath = fileInfo.absoluteFilePath();
+        qDebug() << "  Found:" << absPath;
+        if (filenameRegex.exactMatch(absPath)) {
+            if (count >= MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT || totalSize > MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES) {
+                qDebug() << "  Removing:" << absPath;
+                QFile backup(absPath);
+                if (backup.remove()) {
+                    qDebug() << "  Removed backup:" << absPath;
+                } else {
+                    qWarning() << "  Failed to remove backup:" << absPath;
+                }
+            }
+            totalSize += fileInfo.size();
+            count++;
+        }
+    }
+    qDebug() << "Found" << count << "backups";
+}
+
 void OctreePersistThread::persist() {
     if (_tree->isDirty() && _initialLoadComplete) {
 
@@ -304,27 +290,14 @@ void OctreePersistThread::persist() {
             qCDebug(octree) << "DONE pruning Octree before saving...";
         });
 
-        qCDebug(octree) << "persist operation calling backup...";
-        backup(); // handle backup if requested        
-        qCDebug(octree) << "persist operation DONE with backup...";
-
         _tree->incrementPersistDataVersion();
 
-        // create our "lock" file to indicate we're saving.
+        qCDebug(octree) << "Saving Octree data to:" << _filename;
-        QString lockFileName = _filename + ".lock";
+        if (_tree->writeToFile(_filename.toLocal8Bit().constData(), nullptr, _persistAsFileType)) {
-        std::ofstream lockFile(qPrintable(lockFileName), std::ios::out|std::ios::binary);
-        if(lockFile.is_open()) {
-            qCDebug(octree) << "saving Octree lock file created at:" << lockFileName;
-
-            _tree->writeToFile(qPrintable(_filename), NULL, _persistAsFileType);
-            time(&_lastPersistTime);
             _tree->clearDirtyBit(); // tree is clean after saving
-            qCDebug(octree) << "DONE saving Octree to file...";
+            qCDebug(octree) << "DONE persisting Octree data to" << _filename;
-
+        } else {
-            lockFile.close();
+            qCWarning(octree) << "Failed to persist Octree data to" << _filename;
-            qCDebug(octree) << "saving Octree lock file closed:" << lockFileName;
-            remove(qPrintable(lockFileName));
-            qCDebug(octree) << "saving Octree lock file removed:" << lockFileName;
         }
 
         sendLatestEntityDataToDS();
@@ -345,197 +318,3 @@ void OctreePersistThread::sendLatestEntityDataToDS() {
         qCWarning(octree) << "Failed to persist octree to DS";
     }
 }
-
-void OctreePersistThread::restoreFromMostRecentBackup() {
-    qCDebug(octree) << "Restoring from most recent backup...";
-    
-    QString mostRecentBackupFileName;
-    QDateTime mostRecentBackupTime;
-    
-    bool recentBackup = getMostRecentBackup(QString(""), mostRecentBackupFileName, mostRecentBackupTime);
-
-    // If we found a backup file, restore from that file.
-    if (recentBackup) {
-        qCDebug(octree) << "BEST backup file:" << mostRecentBackupFileName << " last modified:" << mostRecentBackupTime.toString();
-
-        qCDebug(octree) << "Removing old file:" << _filename;
-        remove(qPrintable(_filename));
-
-        qCDebug(octree) << "Restoring backup file " << mostRecentBackupFileName << "...";
-        bool result = QFile::copy(mostRecentBackupFileName, _filename);
-        if (result) {
-            qCDebug(octree) << "DONE restoring backup file " << mostRecentBackupFileName << "to" << _filename << "...";
-        } else {
-            qCDebug(octree) << "ERROR while restoring backup file " << mostRecentBackupFileName << "to" << _filename << "...";
-            perror("ERROR while restoring backup file");
-        }
-    } else {
-        qCDebug(octree) << "NO BEST backup file found.";
-    }
-}
-
-bool OctreePersistThread::getMostRecentBackup(const QString& format, 
-                            QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime) {
-
-    // Based on our backup file name, determine the path and file name pattern for backup files
-    QFileInfo persistFileInfo(_filename);
-    QString path = _backupDirectory;
-    QString fileNamePart = persistFileInfo.fileName();
-
-    QStringList filters;
-
-    if (format.isEmpty()) {
-        // Create a file filter that will find all backup files of this extension format
-        foreach(const BackupRule& rule, _backupRules) {
-            QString backupExtension = rule.extensionFormat;
-            backupExtension.replace(QRegExp("%."), "*");
-            QString backupFileNamePart = fileNamePart + backupExtension;
-            filters << backupFileNamePart;
-        }
-    } else {
-        QString backupExtension = format;
-        backupExtension.replace(QRegExp("%."), "*");
-        QString backupFileNamePart = fileNamePart + backupExtension;
-        filters << backupFileNamePart;
-    }
-    
-    bool bestBackupFound = false;        
-    QString bestBackupFile;
-    QDateTime bestBackupFileTime;
-
-    // Iterate over all of the backup files in the persist location
-    QDirIterator dirIterator(path, filters, QDir::Files|QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
-    while(dirIterator.hasNext()) {
-
-        dirIterator.next();
-        QDateTime lastModified = dirIterator.fileInfo().lastModified();
-
-        // Based on last modified date, track the most recently modified file as the best backup
-        if (lastModified > bestBackupFileTime) {
-            bestBackupFound = true;
-            bestBackupFile = dirIterator.filePath();
-            bestBackupFileTime = lastModified;
-        }
-    }
-
-    // If we found a backup then return the results    
-    if (bestBackupFound) {
-        mostRecentBackupFileName = bestBackupFile;
-        mostRecentBackupTime = bestBackupFileTime;
-    }
-    return bestBackupFound;
-}
-
-void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) {
-
-    if (rule.extensionFormat.contains("%N")) {
-        if (rule.maxBackupVersions > 0) {
-            qCDebug(octree) << "Rolling old backup versions for rule" << rule.name << "...";
-
-            QString backupFileName = _backupDirectory + "/" + QUrl(_filename).fileName();
-
-            // Delete maximum rolling file because rename() fails on Windows if target exists
-            QString backupMaxExtensionN = rule.extensionFormat;
-            backupMaxExtensionN.replace(QString("%N"), QString::number(rule.maxBackupVersions));
-            QString backupMaxFilenameN = backupFileName + backupMaxExtensionN;
-            QFile backupMaxFileN(backupMaxFilenameN);
-            if (backupMaxFileN.exists()) {
-                int result = remove(qPrintable(backupMaxFilenameN));
-                if (result != 0) {
-                    qCDebug(octree) << "ERROR deleting old backup file " << backupMaxFilenameN;
-                }
-            }
-
-            for(int n = rule.maxBackupVersions - 1; n > 0; n--) {
-                QString backupExtensionN = rule.extensionFormat;
-                QString backupExtensionNplusOne = rule.extensionFormat;
-                backupExtensionN.replace(QString("%N"), QString::number(n));
-                backupExtensionNplusOne.replace(QString("%N"), QString::number(n+1));
-
-                QString backupFilenameN = findMostRecentFileExtension(backupFileName, PERSIST_EXTENSIONS) + backupExtensionN;
-                QString backupFilenameNplusOne = backupFileName + backupExtensionNplusOne;
-
-                QFile backupFileN(backupFilenameN);
-
-                if (backupFileN.exists()) {
-                    qCDebug(octree) << "rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
-                    int result = rename(qPrintable(backupFilenameN), qPrintable(backupFilenameNplusOne));
-                    if (result == 0) {
-                        qCDebug(octree) << "DONE rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
-                    } else {
-                        qCDebug(octree) << "ERROR in rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
-                        perror("ERROR in rolling backup file");
-                    }
-                }
-            }
-            qCDebug(octree) << "Done rolling old backup versions...";
-        } else {
-            qCDebug(octree) << "Rolling backups for rule" << rule.name << "."
-                     << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
-                     << " No need to roll backups...";
-        }
-    }
-}
-
-void OctreePersistThread::backup() {
-    qCDebug(octree) << "backup operation wantBackup:" << _wantBackup;
-    if (_wantBackup) {
-        quint64 now = usecTimestampNow();
-        
-        for(int i = 0; i < _backupRules.count(); i++) {
-            BackupRule& rule = _backupRules[i];
-
-            quint64 sinceLastBackup = now - rule.lastBackup;
-            quint64 SECS_TO_USECS = 1000 * 1000;
-            quint64 intervalToBackup = rule.interval * SECS_TO_USECS;
-
-            qCDebug(octree) << "Checking [" << rule.name << "] - Time since last backup [" << sinceLastBackup << "] " << 
-                        "compared to backup interval [" << intervalToBackup << "]...";
-
-            if (sinceLastBackup > intervalToBackup) {
-                qCDebug(octree) << "Time since last backup [" << sinceLastBackup << "] for rule [" << rule.name 
-                                        << "] exceeds backup interval [" << intervalToBackup << "] doing backup now...";
-
-                struct tm* localTime = localtime(&_lastPersistTime);
-
-                QString backupFileName = _backupDirectory + "/" + QUrl(_filename).fileName();
-            
-                // check to see if they asked for version rolling format
-                if (rule.extensionFormat.contains("%N")) {
-                    rollOldBackupVersions(rule); // rename all the old backup files accordingly
-                    QString backupExtension = rule.extensionFormat;
-                    backupExtension.replace(QString("%N"), QString("1"));
-                    backupFileName += backupExtension;
-                } else {
-                    char backupExtension[256];
-                    strftime(backupExtension, sizeof(backupExtension), qPrintable(rule.extensionFormat), localTime);
-                    backupFileName += backupExtension;
-                }
-
-                if (rule.maxBackupVersions > 0) {
-                    QFile persistFile(_filename);
-                    if (persistFile.exists()) {
-                        qCDebug(octree) << "backing up persist file " << _filename << "to" << backupFileName << "...";
-                        bool result = QFile::copy(_filename, backupFileName);
-                        if (result) {
-                            qCDebug(octree) << "DONE backing up persist file...";
-                            rule.lastBackup = now; // only record successful backup in this case.
-                        } else {
-                            qCDebug(octree) << "ERROR in backing up persist file...";
-                            perror("ERROR in backing up persist file");
-                        }
-                    } else {
-                        qCDebug(octree) << "persist file " << _filename << " does not exist. " << 
-                                    "nothing to backup for this rule ["<< rule.name << "]...";
-                    }
-                } else {
-                    qCDebug(octree) << "This backup rule" << rule.name 
-                             << " has Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
-                             << " There are no backups to be done...";
-                }
-            } else {
-                qCDebug(octree) << "Backup not needed for this rule ["<< rule.name << "]...";
-            }
-        }
-    }
-}

libraries/octree/src/OctreePersistThread.h

@@ -18,7 +18,7 @@
 #include <GenericThread.h>
 #include "Octree.h"
 
-class OctreePersistThread : public GenericThread {
+class OctreePersistThread : public QObject {
     Q_OBJECT
 public:
     class BackupRule {
@@ -30,37 +30,37 @@ public:
         quint64 lastBackup;
     };
 
-    static const int DEFAULT_PERSIST_INTERVAL;
+    static const std::chrono::seconds DEFAULT_PERSIST_INTERVAL;
 
-    OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory,
+    OctreePersistThread(OctreePointer tree,
-                        int persistInterval = DEFAULT_PERSIST_INTERVAL, bool wantBackup = false,
+                        const QString& filename,
-                        const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false,
+                        std::chrono::milliseconds persistInterval = DEFAULT_PERSIST_INTERVAL,
-                        QString persistAsFileType = "json.gz", const QByteArray& replacementData = QByteArray());
+                        bool debugTimestampNow = false,
+                        QString persistAsFileType = "json.gz");
 
     bool isInitialLoadComplete() const { return _initialLoadComplete; }
     quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
 
-    void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
-
     QString getPersistFilename() const { return _filename; }
     QString getPersistFileMimeType() const;
     QByteArray getPersistFileContents() const;
 
+    void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
+
+public slots:
+    void start();
+
 signals:
     void loadCompleted();
 
-protected:
+protected slots:
-    /// Implements generic processing behavior for this thread.
+    void process();
-    virtual bool process() override;
+    void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
 
+protected:
     void persist();
-    void backup();
-    void rollOldBackupVersions(const BackupRule& rule);
-    void restoreFromMostRecentBackup();
-    bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
-    quint64 getMostRecentBackupTimeInUsecs(const QString& format);
-    void parseSettings(const QJsonObject& settings);
     bool backupCurrentFile();
+    void cleanupOldReplacementBackups();
 
     void replaceData(QByteArray data);
     void sendLatestEntityDataToDS();
@@ -68,18 +68,12 @@ protected:
 private:
     OctreePointer _tree;
     QString _filename;
-    QString _backupDirectory;
+    std::chrono::milliseconds _persistInterval;
-    int _persistInterval;
+    std::chrono::steady_clock::time_point _lastPersistCheck;
     bool _initialLoadComplete;
-    QByteArray _replacementData;
 
     quint64 _loadTimeUSecs;
 
-    time_t _lastPersistTime;
-    quint64 _lastCheck;
-    bool _wantBackup;
-    QVector<BackupRule> _backupRules;
-
     bool _debugTimestampNow;
     quint64 _lastTimeDebug;