From f3f29bb6611146319b3e9860ccf10cb052014314 Mon Sep 17 00:00:00 2001
From: Stephen Birarda <commit@birarda.com>
Date: Wed, 25 Apr 2018 12:13:46 -0700
Subject: [PATCH 01/18] allow avatar data to use full segment size

---
 .../src/avatars/AvatarMixerSlave.cpp          | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/assignment-client/src/avatars/AvatarMixerSlave.cpp b/assignment-client/src/avatars/AvatarMixerSlave.cpp
index fb4b65726a..ece0ab9616 100644
--- a/assignment-client/src/avatars/AvatarMixerSlave.cpp
+++ b/assignment-client/src/avatars/AvatarMixerSlave.cpp
@@ -392,21 +392,26 @@ void AvatarMixerSlave::broadcastAvatarDataToAgent(const SharedNodePointer& node)
         quint64 end = usecTimestampNow();
         _stats.toByteArrayElapsedTime += (end - start);
 
-        static const int MAX_ALLOWED_AVATAR_DATA = (1400 - NUM_BYTES_RFC4122_UUID);
-        if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
-            qCWarning(avatars) << "otherAvatar.toByteArray() resulted in very large buffer:" << bytes.size() << "... attempt to drop facial data";
+        auto maxAvatarDataBytes = avatarPacketList->getMaxSegmentSize() - NUM_BYTES_RFC4122_UUID;
+        if (bytes.size() > maxAvatarDataBytes) {
+            qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID()
+                << "resulted in very large buffer of" << bytes.size() << "bytes - dropping facial data";
 
             dropFaceTracking = true; // first try dropping the facial data
             bytes = otherAvatar->toByteArray(detail, lastEncodeForOther, lastSentJointsForOther,
                                              hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
 
-            if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
-                qCWarning(avatars) << "otherAvatar.toByteArray() without facial data resulted in very large buffer:" << bytes.size() << "... reduce to MinimumData";
+            if (bytes.size() > maxAvatarDataBytes) {
+                qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID()
+                    << "without facial data resulted in very large buffer of" << bytes.size()
+                    << "bytes - reducing to MinimumData";
                 bytes = otherAvatar->toByteArray(AvatarData::MinimumData, lastEncodeForOther, lastSentJointsForOther,
                                                  hasFlagsOut, dropFaceTracking, distanceAdjust, viewerPosition, &lastSentJointsForOther);
 
-                if (bytes.size() > MAX_ALLOWED_AVATAR_DATA) {
-                    qCWarning(avatars) << "otherAvatar.toByteArray() MinimumData resulted in very large buffer:" << bytes.size() << "... FAIL!!";
+                if (bytes.size() > maxAvatarDataBytes) {
+                    qCWarning(avatars) << "otherAvatar.toByteArray() for" << otherNode->getUUID()
+                        << "MinimumData resulted in very large buffer of" << bytes.size()
+                        << "bytes - refusing to send avatar";
                     includeThisAvatar = false;
                 }
             }

From 2143bae1007fe1552224e37697d03b4dda114874 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Thu, 5 Apr 2018 10:23:46 -0700
Subject: [PATCH 02/18] Remove unused entity server backups

The domain server now handles entity backups so this code is no longer
used.

This also fixes a bug where, if the entity server went down while
writing the persist file, it would leave a lock file behind. When
starting up the next time it would try to recover from the last backup.
Because the entity server no longer gets any backup rules, and the way
it searches for backups, it would end up finding the lock file and
treating it as the most recent backup. This meant that the
models.json.gz file was replaced with an empty file.
---
 assignment-client/src/octree/OctreeServer.cpp |  43 +--
 assignment-client/src/octree/OctreeServer.h   |   4 -
 libraries/octree/src/OctreePersistThread.cpp  | 301 +-----------------
 libraries/octree/src/OctreePersistThread.h    |  24 +-
 4 files changed, 16 insertions(+), 356 deletions(-)

diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp
index acf0abeed1..83fbf6f3d8 100644
--- a/assignment-client/src/octree/OctreeServer.cpp
+++ b/assignment-client/src/octree/OctreeServer.cpp
@@ -1055,17 +1055,6 @@ void OctreeServer::readConfiguration() {
         readOptionInt(QString("persistInterval"), settingsSectionObject, _persistInterval);
         qDebug() << "persistInterval=" << _persistInterval;
 
-        bool noBackup;
-        readOptionBool(QString("NoBackup"), settingsSectionObject, noBackup);
-        _wantBackup = !noBackup;
-        qDebug() << "wantBackup=" << _wantBackup;
-
-        if (!readOptionString("backupDirectoryPath", settingsSectionObject, _backupDirectoryPath)) {
-            _backupDirectoryPath = "";
-        }
-
-        qDebug() << "backupDirectoryPath=" << _backupDirectoryPath;
-
         readOptionBool(QString("persistFileDownload"), settingsSectionObject, _persistFileDownload);
         qDebug() << "persistFileDownload=" << _persistFileDownload;
 
@@ -1288,38 +1277,10 @@ void OctreeServer::beginRunning(QByteArray replaceData) {
         }
 
         auto persistFileDirectory = QFileInfo(_persistAbsoluteFilePath).absolutePath();
-        if (_backupDirectoryPath.isEmpty()) {
-            // Use the persist file's directory to store backups
-            _backupDirectoryPath = persistFileDirectory;
-        } else {
-            // The backup directory has been set.
-            //   If relative, make it relative to the entities directory in the application data directory
-            //   If absolute, no resolution is necessary
-            QDir backupDirectory { _backupDirectoryPath };
-            QString absoluteBackupDirectory;
-            if (backupDirectory.isRelative()) {
-                absoluteBackupDirectory = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_backupDirectoryPath);
-                absoluteBackupDirectory = QDir(absoluteBackupDirectory).absolutePath();
-            } else {
-                absoluteBackupDirectory = backupDirectory.absolutePath();
-            }
-            backupDirectory = QDir(absoluteBackupDirectory);
-            if (!backupDirectory.exists()) {
-                if (backupDirectory.mkpath(".")) {
-                    qDebug() << "Created backup directory";
-                } else {
-                    qDebug() << "ERROR creating backup directory, using persist file directory";
-                    _backupDirectoryPath = persistFileDirectory;
-                }
-            } else {
-                _backupDirectoryPath = absoluteBackupDirectory;
-            }
-        }
-        qDebug() << "Backups will be stored in: " << _backupDirectoryPath;
 
         // now set up PersistThread
-        _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval,
-                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData);
+        _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _persistInterval, _debugTimestampNow,
+                                                 _persistAsFileType, replaceData);
         _persistThread->initialize(true);
     }
 
diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h
index 87145dd46e..3595fe1129 100644
--- a/assignment-client/src/octree/OctreeServer.h
+++ b/assignment-client/src/octree/OctreeServer.h
@@ -190,7 +190,6 @@ protected:
     QString _persistFilePath;
     QString _persistAbsoluteFilePath;
     QString _persistAsFileType;
-    QString _backupDirectoryPath;
     int _packetsPerClientPerInterval;
     int _packetsTotalPerInterval;
     OctreePointer _tree; // this IS a reaveraging tree
@@ -203,10 +202,7 @@ protected:
     OctreePersistThread* _persistThread;
 
     int _persistInterval;
-    bool _wantBackup;
     bool _persistFileDownload;
-    QString _backupExtensionFormat;
-    int _backupInterval;
     int _maxBackupVersions;
 
     time_t _started;
diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 3dc051675d..5f05c3cc85 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -37,25 +37,19 @@
 
 const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
 
-OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval,
-                                         bool wantBackup, const QJsonObject& settings, bool debugTimestampNow,
-                                         QString persistAsFileType, const QByteArray& replacementData) :
+OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, int persistInterval,
+                                         bool debugTimestampNow, QString persistAsFileType, const QByteArray& replacementData) :
     _tree(tree),
     _filename(filename),
-    _backupDirectory(backupDirectory),
     _persistInterval(persistInterval),
     _initialLoadComplete(false),
     _replacementData(replacementData),
     _loadTimeUSecs(0),
     _lastCheck(0),
-    _wantBackup(wantBackup),
     _debugTimestampNow(debugTimestampNow),
     _lastTimeDebug(0),
     _persistAsFileType(persistAsFileType)
 {
-    parseSettings(settings);
-
-
     // in case the persist filename has an extension that doesn't match the file type
     QString sansExt = fileNameWithoutExtension(_filename, PERSIST_EXTENSIONS);
     _filename = sansExt + "." + _persistAsFileType;
@@ -70,72 +64,6 @@ QString OctreePersistThread::getPersistFileMimeType() const {
     return "";
 }
 
-void OctreePersistThread::parseSettings(const QJsonObject& settings) {
-    if (settings["backups"].isArray()) {
-        const QJsonArray& backupRules = settings["backups"].toArray();
-        qCDebug(octree) << "BACKUP RULES:";
-
-        foreach (const QJsonValue& value, backupRules) {
-
-            QJsonObject obj = value.toObject();
-
-            int interval = 0;
-            int count = 0;
-            
-            QJsonValue intervalVal = obj["backupInterval"];
-            if (intervalVal.isString()) {
-                interval = intervalVal.toString().toInt();
-            } else {
-                interval = intervalVal.toInt();
-            }
-
-            QJsonValue countVal = obj["maxBackupVersions"];
-            if (countVal.isString()) {
-                count = countVal.toString().toInt();
-            } else {
-                count = countVal.toInt();
-            }
-
-            qCDebug(octree) << "    Name:" << obj["Name"].toString();
-            qCDebug(octree) << "        format:" << obj["format"].toString();
-            qCDebug(octree) << "        interval:" << interval;
-            qCDebug(octree) << "        count:" << count;
-
-            BackupRule newRule = { obj["Name"].toString(), interval, obj["format"].toString(), count, 0};
-                                    
-            newRule.lastBackup = getMostRecentBackupTimeInUsecs(obj["format"].toString());
-            
-            if (newRule.lastBackup > 0) {
-                quint64 now = usecTimestampNow();
-                quint64 sinceLastBackup = now - newRule.lastBackup;
-                qCDebug(octree) << "        lastBackup:" << qPrintable(formatUsecTime(sinceLastBackup)) << "ago";
-            } else {
-                qCDebug(octree) << "        lastBackup: NEVER";
-            }
-            
-            _backupRules << newRule;
-        }        
-    } else {
-        qCDebug(octree) << "BACKUP RULES: NONE";
-    }
-}
-
-quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& format) {
-
-    quint64 mostRecentBackupInUsecs = 0;
-
-    QString mostRecentBackupFileName;
-    QDateTime mostRecentBackupTime;
-    
-    bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
-    
-    if (recentBackup) {
-        mostRecentBackupInUsecs = mostRecentBackupTime.toMSecsSinceEpoch() * USECS_PER_MSEC;
-    }
-
-    return mostRecentBackupInUsecs;
-}
-
 void OctreePersistThread::replaceData(QByteArray data) {
     backupCurrentFile();
 
@@ -188,20 +116,7 @@ bool OctreePersistThread::process() {
         _tree->withWriteLock([&] {
             PerformanceWarning warn(true, "Loading Octree File", true);
 
-            // First check to make sure "lock" file doesn't exist. If it does exist, then
-            // our last save crashed during the save, and we want to load our most recent backup.
-            QString lockFileName = _filename + ".lock";
-            std::ifstream lockFile(qPrintable(lockFileName), std::ios::in | std::ios::binary | std::ios::ate);
-            if (lockFile.is_open()) {
-                qCDebug(octree) << "WARNING: Octree lock file detected at startup:" << lockFileName;
-
-                lockFile.close();
-                qCDebug(octree) << "Removing lock file:" << lockFileName;
-                remove(qPrintable(lockFileName));
-                qCDebug(octree) << "Lock file removed:" << lockFileName;
-            }
-
-            persistentFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
+            persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
             _tree->pruneTree();
         });
 
@@ -233,11 +148,6 @@ bool OctreePersistThread::process() {
 
         // Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance.
         _lastCheck = usecTimestampNow(); // we just loaded, no need to save again
-        
-        // This last persist time is not really used until the file is actually persisted. It is only
-        // used in formatting the backup filename in cases of non-rolling backup names. However, we don't
-        // want an uninitialized value for this, so we set it to the current time (startup of the server)
-        time(&_lastPersistTime);
 
         if (_replacementData.isNull()) {
             sendLatestEntityDataToDS();
@@ -304,26 +214,21 @@ void OctreePersistThread::persist() {
             qCDebug(octree) << "DONE pruning Octree before saving...";
         });
 
-        qCDebug(octree) << "persist operation calling backup...";
-        backup(); // handle backup if requested        
-        qCDebug(octree) << "persist operation DONE with backup...";
-
         _tree->incrementPersistDataVersion();
 
         // create our "lock" file to indicate we're saving.
         QString lockFileName = _filename + ".lock";
-        std::ofstream lockFile(qPrintable(lockFileName), std::ios::out|std::ios::binary);
+        std::ofstream lockFile(lockFileName.toLocal8Bit().constData(), std::ios::out|std::ios::binary);
         if(lockFile.is_open()) {
             qCDebug(octree) << "saving Octree lock file created at:" << lockFileName;
 
-            _tree->writeToFile(qPrintable(_filename), NULL, _persistAsFileType);
-            time(&_lastPersistTime);
+            _tree->writeToFile(_filename.toLocal8Bit().constData(), NULL, _persistAsFileType);
             _tree->clearDirtyBit(); // tree is clean after saving
             qCDebug(octree) << "DONE saving Octree to file...";
 
             lockFile.close();
             qCDebug(octree) << "saving Octree lock file closed:" << lockFileName;
-            remove(qPrintable(lockFileName));
+            remove(lockFileName.toLocal8Bit().constData());
             qCDebug(octree) << "saving Octree lock file removed:" << lockFileName;
         }
 
@@ -345,197 +250,3 @@ void OctreePersistThread::sendLatestEntityDataToDS() {
         qCWarning(octree) << "Failed to persist octree to DS";
     }
 }
-
-void OctreePersistThread::restoreFromMostRecentBackup() {
-    qCDebug(octree) << "Restoring from most recent backup...";
-    
-    QString mostRecentBackupFileName;
-    QDateTime mostRecentBackupTime;
-    
-    bool recentBackup = getMostRecentBackup(QString(""), mostRecentBackupFileName, mostRecentBackupTime);
-
-    // If we found a backup file, restore from that file.
-    if (recentBackup) {
-        qCDebug(octree) << "BEST backup file:" << mostRecentBackupFileName << " last modified:" << mostRecentBackupTime.toString();
-
-        qCDebug(octree) << "Removing old file:" << _filename;
-        remove(qPrintable(_filename));
-
-        qCDebug(octree) << "Restoring backup file " << mostRecentBackupFileName << "...";
-        bool result = QFile::copy(mostRecentBackupFileName, _filename);
-        if (result) {
-            qCDebug(octree) << "DONE restoring backup file " << mostRecentBackupFileName << "to" << _filename << "...";
-        } else {
-            qCDebug(octree) << "ERROR while restoring backup file " << mostRecentBackupFileName << "to" << _filename << "...";
-            perror("ERROR while restoring backup file");
-        }
-    } else {
-        qCDebug(octree) << "NO BEST backup file found.";
-    }
-}
-
-bool OctreePersistThread::getMostRecentBackup(const QString& format, 
-                            QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime) {
-
-    // Based on our backup file name, determine the path and file name pattern for backup files
-    QFileInfo persistFileInfo(_filename);
-    QString path = _backupDirectory;
-    QString fileNamePart = persistFileInfo.fileName();
-
-    QStringList filters;
-
-    if (format.isEmpty()) {
-        // Create a file filter that will find all backup files of this extension format
-        foreach(const BackupRule& rule, _backupRules) {
-            QString backupExtension = rule.extensionFormat;
-            backupExtension.replace(QRegExp("%."), "*");
-            QString backupFileNamePart = fileNamePart + backupExtension;
-            filters << backupFileNamePart;
-        }
-    } else {
-        QString backupExtension = format;
-        backupExtension.replace(QRegExp("%."), "*");
-        QString backupFileNamePart = fileNamePart + backupExtension;
-        filters << backupFileNamePart;
-    }
-    
-    bool bestBackupFound = false;        
-    QString bestBackupFile;
-    QDateTime bestBackupFileTime;
-
-    // Iterate over all of the backup files in the persist location
-    QDirIterator dirIterator(path, filters, QDir::Files|QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
-    while(dirIterator.hasNext()) {
-
-        dirIterator.next();
-        QDateTime lastModified = dirIterator.fileInfo().lastModified();
-
-        // Based on last modified date, track the most recently modified file as the best backup
-        if (lastModified > bestBackupFileTime) {
-            bestBackupFound = true;
-            bestBackupFile = dirIterator.filePath();
-            bestBackupFileTime = lastModified;
-        }
-    }
-
-    // If we found a backup then return the results    
-    if (bestBackupFound) {
-        mostRecentBackupFileName = bestBackupFile;
-        mostRecentBackupTime = bestBackupFileTime;
-    }
-    return bestBackupFound;
-}
-
-void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) {
-
-    if (rule.extensionFormat.contains("%N")) {
-        if (rule.maxBackupVersions > 0) {
-            qCDebug(octree) << "Rolling old backup versions for rule" << rule.name << "...";
-
-            QString backupFileName = _backupDirectory + "/" + QUrl(_filename).fileName();
-
-            // Delete maximum rolling file because rename() fails on Windows if target exists
-            QString backupMaxExtensionN = rule.extensionFormat;
-            backupMaxExtensionN.replace(QString("%N"), QString::number(rule.maxBackupVersions));
-            QString backupMaxFilenameN = backupFileName + backupMaxExtensionN;
-            QFile backupMaxFileN(backupMaxFilenameN);
-            if (backupMaxFileN.exists()) {
-                int result = remove(qPrintable(backupMaxFilenameN));
-                if (result != 0) {
-                    qCDebug(octree) << "ERROR deleting old backup file " << backupMaxFilenameN;
-                }
-            }
-
-            for(int n = rule.maxBackupVersions - 1; n > 0; n--) {
-                QString backupExtensionN = rule.extensionFormat;
-                QString backupExtensionNplusOne = rule.extensionFormat;
-                backupExtensionN.replace(QString("%N"), QString::number(n));
-                backupExtensionNplusOne.replace(QString("%N"), QString::number(n+1));
-
-                QString backupFilenameN = findMostRecentFileExtension(backupFileName, PERSIST_EXTENSIONS) + backupExtensionN;
-                QString backupFilenameNplusOne = backupFileName + backupExtensionNplusOne;
-
-                QFile backupFileN(backupFilenameN);
-
-                if (backupFileN.exists()) {
-                    qCDebug(octree) << "rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
-                    int result = rename(qPrintable(backupFilenameN), qPrintable(backupFilenameNplusOne));
-                    if (result == 0) {
-                        qCDebug(octree) << "DONE rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
-                    } else {
-                        qCDebug(octree) << "ERROR in rolling backup file " << backupFilenameN << "to" << backupFilenameNplusOne << "...";
-                        perror("ERROR in rolling backup file");
-                    }
-                }
-            }
-            qCDebug(octree) << "Done rolling old backup versions...";
-        } else {
-            qCDebug(octree) << "Rolling backups for rule" << rule.name << "."
-                     << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
-                     << " No need to roll backups...";
-        }
-    }
-}
-
-void OctreePersistThread::backup() {
-    qCDebug(octree) << "backup operation wantBackup:" << _wantBackup;
-    if (_wantBackup) {
-        quint64 now = usecTimestampNow();
-        
-        for(int i = 0; i < _backupRules.count(); i++) {
-            BackupRule& rule = _backupRules[i];
-
-            quint64 sinceLastBackup = now - rule.lastBackup;
-            quint64 SECS_TO_USECS = 1000 * 1000;
-            quint64 intervalToBackup = rule.interval * SECS_TO_USECS;
-
-            qCDebug(octree) << "Checking [" << rule.name << "] - Time since last backup [" << sinceLastBackup << "] " << 
-                        "compared to backup interval [" << intervalToBackup << "]...";
-
-            if (sinceLastBackup > intervalToBackup) {
-                qCDebug(octree) << "Time since last backup [" << sinceLastBackup << "] for rule [" << rule.name 
-                                        << "] exceeds backup interval [" << intervalToBackup << "] doing backup now...";
-
-                struct tm* localTime = localtime(&_lastPersistTime);
-
-                QString backupFileName = _backupDirectory + "/" + QUrl(_filename).fileName();
-            
-                // check to see if they asked for version rolling format
-                if (rule.extensionFormat.contains("%N")) {
-                    rollOldBackupVersions(rule); // rename all the old backup files accordingly
-                    QString backupExtension = rule.extensionFormat;
-                    backupExtension.replace(QString("%N"), QString("1"));
-                    backupFileName += backupExtension;
-                } else {
-                    char backupExtension[256];
-                    strftime(backupExtension, sizeof(backupExtension), qPrintable(rule.extensionFormat), localTime);
-                    backupFileName += backupExtension;
-                }
-
-                if (rule.maxBackupVersions > 0) {
-                    QFile persistFile(_filename);
-                    if (persistFile.exists()) {
-                        qCDebug(octree) << "backing up persist file " << _filename << "to" << backupFileName << "...";
-                        bool result = QFile::copy(_filename, backupFileName);
-                        if (result) {
-                            qCDebug(octree) << "DONE backing up persist file...";
-                            rule.lastBackup = now; // only record successful backup in this case.
-                        } else {
-                            qCDebug(octree) << "ERROR in backing up persist file...";
-                            perror("ERROR in backing up persist file");
-                        }
-                    } else {
-                        qCDebug(octree) << "persist file " << _filename << " does not exist. " << 
-                                    "nothing to backup for this rule ["<< rule.name << "]...";
-                    }
-                } else {
-                    qCDebug(octree) << "This backup rule" << rule.name 
-                             << " has Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
-                             << " There are no backups to be done...";
-                }
-            } else {
-                qCDebug(octree) << "Backup not needed for this rule ["<< rule.name << "]...";
-            }
-        }
-    }
-}
diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h
index bde207001f..990a3b1fad 100644
--- a/libraries/octree/src/OctreePersistThread.h
+++ b/libraries/octree/src/OctreePersistThread.h
@@ -32,20 +32,22 @@ public:
 
     static const int DEFAULT_PERSIST_INTERVAL;
 
-    OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory,
-                        int persistInterval = DEFAULT_PERSIST_INTERVAL, bool wantBackup = false,
-                        const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false,
-                        QString persistAsFileType = "json.gz", const QByteArray& replacementData = QByteArray());
+    OctreePersistThread(OctreePointer tree,
+                        const QString& filename,
+                        int persistInterval = DEFAULT_PERSIST_INTERVAL,
+                        bool debugTimestampNow = false,
+                        QString persistAsFileType = "json.gz",
+                        const QByteArray& replacementData = QByteArray());
 
     bool isInitialLoadComplete() const { return _initialLoadComplete; }
     quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
 
-    void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
-
     QString getPersistFilename() const { return _filename; }
     QString getPersistFileMimeType() const;
     QByteArray getPersistFileContents() const;
 
+    void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
+
 signals:
     void loadCompleted();
 
@@ -54,12 +56,6 @@ protected:
     virtual bool process() override;
 
     void persist();
-    void backup();
-    void rollOldBackupVersions(const BackupRule& rule);
-    void restoreFromMostRecentBackup();
-    bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
-    quint64 getMostRecentBackupTimeInUsecs(const QString& format);
-    void parseSettings(const QJsonObject& settings);
     bool backupCurrentFile();
 
     void replaceData(QByteArray data);
@@ -68,17 +64,13 @@ protected:
 private:
     OctreePointer _tree;
     QString _filename;
-    QString _backupDirectory;
     int _persistInterval;
     bool _initialLoadComplete;
     QByteArray _replacementData;
 
     quint64 _loadTimeUSecs;
 
-    time_t _lastPersistTime;
     quint64 _lastCheck;
-    bool _wantBackup;
-    QVector<BackupRule> _backupRules;
 
     bool _debugTimestampNow;
     quint64 _lastTimeDebug;

From d476146a56fab23afd4b01e33b86d35af690ca76 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Thu, 10 May 2018 10:47:49 -0700
Subject: [PATCH 03/18] Move OctreePersistThread off GenericThread and add
 octree negotiation

---
 assignment-client/src/octree/OctreeServer.cpp | 162 ++++----------
 assignment-client/src/octree/OctreeServer.h   |  26 +--
 libraries/octree/src/OctreePersistThread.cpp  | 205 ++++++++++++------
 libraries/octree/src/OctreePersistThread.h    |  19 +-
 4 files changed, 208 insertions(+), 204 deletions(-)

diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp
index 83fbf6f3d8..b8329b4d3e 100644
--- a/assignment-client/src/octree/OctreeServer.cpp
+++ b/assignment-client/src/octree/OctreeServer.cpp
@@ -231,18 +231,19 @@ void OctreeServer::trackProcessWaitTime(float time) {
 OctreeServer::OctreeServer(ReceivedMessage& message) :
     ThreadedAssignment(message),
     _argc(0),
-    _argv(NULL),
-    _parsedArgV(NULL),
+    _argv(nullptr),
+    _parsedArgV(nullptr),
+    _httpManager(nullptr),
     _statusPort(0),
     _packetsPerClientPerInterval(10),
     _packetsTotalPerInterval(DEFAULT_PACKETS_PER_INTERVAL),
-    _tree(NULL),
+    _tree(nullptr),
     _wantPersist(true),
     _debugSending(false),
     _debugReceiving(false),
     _verboseDebug(false),
-    _octreeInboundPacketProcessor(NULL),
-    _persistThread(NULL),
+    _octreeInboundPacketProcessor(nullptr),
+    _persistManager(nullptr),
     _started(time(0)),
     _startedUSecs(usecTimestampNow())
 {
@@ -265,11 +266,8 @@ OctreeServer::~OctreeServer() {
         _octreeInboundPacketProcessor->deleteLater();
     }
 
-    if (_persistThread) {
-        _persistThread->terminating();
-        _persistThread->terminate();
-        _persistThread->deleteLater();
-    }
+    qDebug() << "Waiting for persist thread to come down";
+    _persistThread.wait();
 
     // cleanup our tree here...
     qDebug() << qPrintable(_safeServerName) << "server START cleaning up octree... [" << this << "]";
@@ -1117,111 +1115,14 @@ void OctreeServer::run() {
 }
 
 void OctreeServer::domainSettingsRequestComplete() {
-    if (_state != OctreeServerState::WaitingForDomainSettings) {
-        qCWarning(octree_server) << "Received domain settings after they have already been received";
-        return;
-    }
-
     auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
     packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
-
-    packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
+    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
 
     qDebug(octree_server) << "Received domain settings";
 
     readConfiguration();
 
-    _state = OctreeServerState::WaitingForOctreeDataNegotation;
-
-    auto nodeList = DependencyManager::get<NodeList>();
-    const DomainHandler& domainHandler = nodeList->getDomainHandler();
-
-    auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
-
-    OctreeUtils::RawOctreeData data;
-    qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
-    if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
-        qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
-        packet->writePrimitive(true);
-        auto id = data.id.toRfc4122();
-        packet->write(id);
-        packet->writePrimitive(data.version);
-    } else {
-        qCWarning(octree_server) << "No octree data found";
-        packet->writePrimitive(false);
-    }
-
-    qCDebug(octree_server) << "Sending request for octree data to DS";
-    nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
-} 
-
-void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
-    if (_state != OctreeServerState::WaitingForOctreeDataNegotation) {
-        qCWarning(octree_server) << "Server received ocree data file reply but is not currently negotiating.";
-        return;
-    }
-
-    bool includesNewData;
-    message->readPrimitive(&includesNewData);
-    QByteArray replaceData;
-    if (includesNewData) {
-        replaceData = message->readAll();
-        qDebug() << "Got reply to octree data file request, new data sent";
-    } else {
-        qDebug() << "Got reply to octree data file request, current entity data is sufficient";
-        
-        OctreeUtils::RawEntityData data;
-        qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
-        if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
-            if (data.id.isNull()) {
-                qCDebug(octree_server) << "Current octree data has a null id, updating";
-                data.resetIdAndVersion();
-
-                QFile file(_persistAbsoluteFilePath);
-                if (file.open(QIODevice::WriteOnly)) {
-                    auto entityData = data.toGzippedByteArray();
-                    file.write(entityData);
-                    file.close();
-                } else {
-                    qCDebug(octree_server) << "Failed to update octree data";
-                }
-            }
-        }
-    }
-
-    _state = OctreeServerState::Running;
-    beginRunning(replaceData);
-}
-
-void OctreeServer::beginRunning(QByteArray replaceData) {
-    if (_state != OctreeServerState::Running) {
-        qCWarning(octree_server) << "Server is not running";
-        return;
-    }
-
-    auto nodeList = DependencyManager::get<NodeList>();
-
-    // we need to ask the DS about agents so we can ping/reply with them
-    nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
-
-    beforeRun(); // after payload has been processed
-
-    connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
-    connect(nodeList.data(), SIGNAL(nodeKilled(SharedNodePointer)), SLOT(nodeKilled(SharedNodePointer)));
-
-#ifndef WIN32
-    setvbuf(stdout, NULL, _IOLBF, 0);
-#endif
-
-    nodeList->linkedDataCreateCallback = [this](Node* node) {
-        auto queryNodeData = createOctreeQueryNode();
-        queryNodeData->init();
-        node->setLinkedData(std::move(queryNodeData));
-    };
-
-    srand((unsigned)time(0));
-
     // if we want Persistence, set up the local file and persist thread
     if (_wantPersist) {
         static const QString ENTITY_PERSIST_EXTENSION = ".json.gz";
@@ -1279,10 +1180,42 @@ void OctreeServer::beginRunning(QByteArray replaceData) {
         auto persistFileDirectory = QFileInfo(_persistAbsoluteFilePath).absolutePath();
 
         // now set up PersistThread
-        _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _persistInterval, _debugTimestampNow,
-                                                 _persistAsFileType, replaceData);
-        _persistThread->initialize(true);
+        _persistManager = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _persistInterval, _debugTimestampNow,
+                                                 _persistAsFileType);
+        _persistManager->moveToThread(&_persistThread);
+        connect(&_persistThread, &QThread::finished, _persistManager, &QObject::deleteLater);
+        connect(&_persistThread, &QThread::started, _persistManager, &OctreePersistThread::start);
+        connect(_persistManager, &OctreePersistThread::loadCompleted, this, [this]() {
+            beginRunning();
+        });
+        _persistThread.start();
+    } else {
+        beginRunning();
     }
+} 
+
+void OctreeServer::beginRunning() {
+    auto nodeList = DependencyManager::get<NodeList>();
+
+    // we need to ask the DS about agents so we can ping/reply with them
+    nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
+
+    beforeRun(); // after payload has been processed
+
+    connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
+    connect(nodeList.data(), SIGNAL(nodeKilled(SharedNodePointer)), SLOT(nodeKilled(SharedNodePointer)));
+
+#ifndef WIN32
+    setvbuf(stdout, nullptr, _IOLBF, 0);
+#endif
+
+    nodeList->linkedDataCreateCallback = [this](Node* node) {
+        auto queryNodeData = createOctreeQueryNode();
+        queryNodeData->init();
+        node->setLinkedData(std::move(queryNodeData));
+    };
+
+    srand((unsigned)time(0));
 
     // set up our OctreeServerPacketProcessor
     _octreeInboundPacketProcessor = new OctreeInboundPacketProcessor(this);
@@ -1345,7 +1278,7 @@ void OctreeServer::aboutToFinish() {
 
     qDebug() << qPrintable(_safeServerName) << "inform Octree Inbound Packet Processor that we are shutting down...";
 
-    // we're going down - set the NodeList linkedDataCallback to NULL so we do not create any more OctreeQueryNode objects.
+    // we're going down - set the NodeList linkedDataCallback to nullptr so we do not create any more OctreeQueryNode objects.
     // This ensures that we don't get any more newly connecting nodes
     DependencyManager::get<NodeList>()->linkedDataCreateCallback = nullptr;
 
@@ -1363,9 +1296,8 @@ void OctreeServer::aboutToFinish() {
     // which waits on the thread to be done before returning
     _sendThreads.clear(); // Cleans up all the send threads.
 
-    if (_persistThread) {
-        _persistThread->aboutToFinish();
-        _persistThread->terminating();
+    if (_persistManager) {
+        _persistThread.quit();
     }
 
     qDebug() << qPrintable(_safeServerName) << "server ENDING about to finish...";
diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h
index 3595fe1129..5c157b46b2 100644
--- a/assignment-client/src/octree/OctreeServer.h
+++ b/assignment-client/src/octree/OctreeServer.h
@@ -33,12 +33,6 @@ Q_DECLARE_LOGGING_CATEGORY(octree_server)
 
 const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total
 
-enum class OctreeServerState {
-    WaitingForDomainSettings,
-    WaitingForOctreeDataNegotation,
-    Running
-};
-
 /// Handles assignments of type OctreeServer - sending octrees to various clients.
 class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler {
     Q_OBJECT
@@ -46,8 +40,6 @@ public:
     OctreeServer(ReceivedMessage& message);
     ~OctreeServer();
 
-    OctreeServerState _state { OctreeServerState::WaitingForDomainSettings };
-
     /// allows setting of run arguments
     void setArguments(int argc, char** argv);
 
@@ -68,12 +60,12 @@ public:
     static void clientConnected() { _clientCount++; }
     static void clientDisconnected() { _clientCount--; }
 
-    bool isInitialLoadComplete() const { return (_persistThread) ? _persistThread->isInitialLoadComplete() : true; }
-    bool isPersistEnabled() const { return (_persistThread) ? true : false; }
-    quint64 getLoadElapsedTime() const { return (_persistThread) ? _persistThread->getLoadElapsedTime() : 0; }
-    QString getPersistFilename() const { return (_persistThread) ? _persistThread->getPersistFilename() : ""; }
-    QString getPersistFileMimeType() const { return (_persistThread) ? _persistThread->getPersistFileMimeType() : "text/plain"; }
-    QByteArray getPersistFileContents() const { return (_persistThread) ? _persistThread->getPersistFileContents() : QByteArray(); }
+    bool isInitialLoadComplete() const { return (_persistManager) ? _persistManager->isInitialLoadComplete() : true; }
+    bool isPersistEnabled() const { return (_persistManager) ? true : false; }
+    quint64 getLoadElapsedTime() const { return (_persistManager) ? _persistManager->getLoadElapsedTime() : 0; }
+    QString getPersistFilename() const { return (_persistManager) ? _persistManager->getPersistFilename() : ""; }
+    QString getPersistFileMimeType() const { return (_persistManager) ? _persistManager->getPersistFileMimeType() : "text/plain"; }
+    QByteArray getPersistFileContents() const { return (_persistManager) ? _persistManager->getPersistFileContents() : QByteArray(); }
 
     // Subclasses must implement these methods
     virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() = 0;
@@ -149,7 +141,6 @@ private slots:
     void domainSettingsRequestComplete();
     void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
     void removeSendThread();
 
 protected:
@@ -171,7 +162,7 @@ protected:
     QString getConfiguration();
     QString getStatusLink();
 
-    void beginRunning(QByteArray replaceData);
+    void beginRunning();
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
     virtual UniqueSendThread newSendThread(const SharedNodePointer& node) = 0;
@@ -199,7 +190,8 @@ protected:
     bool _debugTimestampNow;
     bool _verboseDebug;
     OctreeInboundPacketProcessor* _octreeInboundPacketProcessor;
-    OctreePersistThread* _persistThread;
+    OctreePersistThread* _persistManager;
+    QThread _persistThread;
 
     int _persistInterval;
     bool _persistFileDownload;
diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 5f05c3cc85..94d76b26ec 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -38,12 +38,11 @@
 const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
 
 OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, int persistInterval,
-                                         bool debugTimestampNow, QString persistAsFileType, const QByteArray& replacementData) :
+                                         bool debugTimestampNow, QString persistAsFileType) :
     _tree(tree),
     _filename(filename),
     _persistInterval(persistInterval),
     _initialLoadComplete(false),
-    _replacementData(replacementData),
     _loadTimeUSecs(0),
     _lastCheck(0),
     _debugTimestampNow(debugTimestampNow),
@@ -55,6 +54,140 @@ OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& file
     _filename = sansExt + "." + _persistAsFileType;
 }
 
+void OctreePersistThread::start() {
+    cleanupOldReplacementBackups();
+
+    QFile tempFile { getTempFilename() };
+    if (tempFile.exists()) {
+        qWarning(octree) << "Found temporary octree file at" << tempFile.fileName();
+        qDebug(octree) << "Attempting to recover from temporary octree file";
+        QFile::remove(_filename);
+        if (tempFile.rename(_filename)) {
+            qDebug(octree) << "Successfully recovered from temporary octree file";
+        } else {
+            qWarning(octree) << "Failed to recover from temporary octree file";
+            tempFile.remove();
+        }
+    }
+
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    const DomainHandler& domainHandler = nodeList->getDomainHandler();
+
+    auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
+
+    OctreeUtils::RawOctreeData data;
+    qCDebug(octree) << "Reading octree data from" << _filename;
+    if (data.readOctreeDataInfoFromFile(_filename)) {
+        qCDebug(octree) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
+        packet->writePrimitive(true);
+        auto id = data.id.toRfc4122();
+        packet->write(id);
+        packet->writePrimitive(data.version);
+    } else {
+        qCWarning(octree) << "No octree data found";
+        packet->writePrimitive(false);
+    }
+
+    qCDebug(octree) << "Sending request for octree data to DS";
+    nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
+}
+
+void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
+    if (_initialLoadComplete) {
+        qCWarning(octree) << "Received OctreeDataFileReply after initial load had completed";
+        return;
+    }
+
+    bool includesNewData;
+    message->readPrimitive(&includesNewData);
+    QByteArray replacementData;
+    if (includesNewData) {
+        replacementData = message->readAll();
+        replaceData(replacementData);
+        qDebug() << "Got reply to octree data file request, new data sent";
+    } else {
+        qDebug() << "Got reply to octree data file request, current entity data is sufficient";
+        
+        OctreeUtils::RawEntityData data;
+        qCDebug(octree) << "Reading octree data from" << _filename;
+        if (data.readOctreeDataInfoFromFile(_filename)) {
+            if (data.id.isNull()) {
+                qCDebug(octree) << "Current octree data has a null id, updating";
+                data.resetIdAndVersion();
+
+                QFile file(_filename);
+                if (file.open(QIODevice::WriteOnly)) {
+                    auto entityData = data.toGzippedByteArray();
+                    file.write(entityData);
+                    file.close();
+                } else {
+                    qCDebug(octree) << "Failed to update octree data";
+                }
+            }
+        }
+    }
+
+    quint64 loadStarted = usecTimestampNow();
+    qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
+
+    OctreeUtils::RawOctreeData data;
+    if (data.readOctreeDataInfoFromFile(_filename)) {
+        qDebug() << "Setting entity version info to: " << data.id << data.version;
+        _tree->setOctreeVersionInfo(data.id, data.version);
+    }
+
+    bool persistentFileRead;
+
+    _tree->withWriteLock([&] {
+        PerformanceWarning warn(true, "Loading Octree File", true);
+
+        persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
+        _tree->pruneTree();
+    });
+
+    quint64 loadDone = usecTimestampNow();
+    _loadTimeUSecs = loadDone - loadStarted;
+
+    _tree->clearDirtyBit(); // the tree is clean since we just loaded it
+    qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
+
+    unsigned long nodeCount = OctreeElement::getNodeCount();
+    unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
+    unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
+    qCDebug(octree, "Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
+
+    bool wantDebug = false;
+    if (wantDebug) {
+        double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime() 
+                                / (double)OctreeElement::getGetChildAtIndexCalls();
+        qCDebug(octree) << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
+                << " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
+
+        double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime() 
+                                / (double)OctreeElement::getSetChildAtIndexCalls();
+        qCDebug(octree) << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
+                << " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
+    }
+
+    _initialLoadComplete = true;
+
+    // Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance.
+    _lastCheck = usecTimestampNow(); // we just loaded, no need to save again
+
+    if (replacementData.isNull()) {
+        sendLatestEntityDataToDS();
+    }
+
+    qDebug() << "Starting timer";
+    QTimer::singleShot(_persistInterval, this, &OctreePersistThread::process);
+
+    emit loadCompleted();
+}
+
+
 QString OctreePersistThread::getPersistFileMimeType() const {
     if (_persistAsFileType == "json") {
         return "application/json";
@@ -96,68 +229,9 @@ bool OctreePersistThread::backupCurrentFile() {
 }
 
 bool OctreePersistThread::process() {
+    qDebug() << "Processing...";
 
-    if (!_initialLoadComplete) {
-        quint64 loadStarted = usecTimestampNow();
-        qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
-
-        if (!_replacementData.isNull()) {
-            replaceData(_replacementData);
-        }
-
-        OctreeUtils::RawOctreeData data;
-        if (data.readOctreeDataInfoFromFile(_filename)) {
-            qDebug() << "Setting entity version info to: " << data.id << data.dataVersion;
-            _tree->setOctreeVersionInfo(data.id, data.dataVersion);
-        }
-
-        bool persistentFileRead;
-
-        _tree->withWriteLock([&] {
-            PerformanceWarning warn(true, "Loading Octree File", true);
-
-            persistentFileRead = _tree->readFromFile(_filename.toLocal8Bit().constData());
-            _tree->pruneTree();
-        });
-
-        quint64 loadDone = usecTimestampNow();
-        _loadTimeUSecs = loadDone - loadStarted;
-
-        _tree->clearDirtyBit(); // the tree is clean since we just loaded it
-        qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
-
-        unsigned long nodeCount = OctreeElement::getNodeCount();
-        unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
-        unsigned long leafNodeCount = OctreeElement::getLeafNodeCount();
-        qCDebug(octree, "Nodes after loading scene %lu nodes %lu internal %lu leaves", nodeCount, internalNodeCount, leafNodeCount);
-
-        bool wantDebug = false;
-        if (wantDebug) {
-            double usecPerGet = (double)OctreeElement::getGetChildAtIndexTime() 
-                                    / (double)OctreeElement::getGetChildAtIndexCalls();
-            qCDebug(octree) << "getChildAtIndexCalls=" << OctreeElement::getGetChildAtIndexCalls()
-                    << " getChildAtIndexTime=" << OctreeElement::getGetChildAtIndexTime() << " perGet=" << usecPerGet;
-
-            double usecPerSet = (double)OctreeElement::getSetChildAtIndexTime() 
-                                    / (double)OctreeElement::getSetChildAtIndexCalls();
-            qCDebug(octree) << "setChildAtIndexCalls=" << OctreeElement::getSetChildAtIndexCalls()
-                    << " setChildAtIndexTime=" << OctreeElement::getSetChildAtIndexTime() << " perSet=" << usecPerSet;
-        }
-
-        _initialLoadComplete = true;
-
-        // Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance.
-        _lastCheck = usecTimestampNow(); // we just loaded, no need to save again
-
-        if (_replacementData.isNull()) {
-            sendLatestEntityDataToDS();
-        }
-        _replacementData.clear();
-
-        emit loadCompleted();
-    }
-
-    if (isStillRunning()) {
+    if (true) { //isStillRunning()) {
         quint64 MSECS_TO_USECS = 1000;
         quint64 USECS_TO_SLEEP = 10 * MSECS_TO_USECS; // every 10ms
         std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
@@ -186,14 +260,15 @@ bool OctreePersistThread::process() {
         }
         
     }
-    return isStillRunning();  // keep running till they terminate us
+    //return isStillRunning();  // keep running till they terminate us
+    QTimer::singleShot(1000, this, &OctreePersistThread::process);
+    return true;
 }
 
 void OctreePersistThread::aboutToFinish() {
     qCDebug(octree) << "Persist thread about to finish...";
     persist();
     qCDebug(octree) << "Persist thread done with about to finish...";
-    _stopThread = true;
 }
 
 QByteArray OctreePersistThread::getPersistFileContents() const {
diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h
index 990a3b1fad..5f161c100f 100644
--- a/libraries/octree/src/OctreePersistThread.h
+++ b/libraries/octree/src/OctreePersistThread.h
@@ -18,7 +18,7 @@
 #include <GenericThread.h>
 #include "Octree.h"
 
-class OctreePersistThread : public GenericThread {
+class OctreePersistThread : public QObject {
     Q_OBJECT
 public:
     class BackupRule {
@@ -36,8 +36,7 @@ public:
                         const QString& filename,
                         int persistInterval = DEFAULT_PERSIST_INTERVAL,
                         bool debugTimestampNow = false,
-                        QString persistAsFileType = "json.gz",
-                        const QByteArray& replacementData = QByteArray());
+                        QString persistAsFileType = "json.gz");
 
     bool isInitialLoadComplete() const { return _initialLoadComplete; }
     quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
@@ -48,25 +47,31 @@ public:
 
     void aboutToFinish(); /// call this to inform the persist thread that the owner is about to finish to support final persist
 
+public slots:
+    void start();
+
 signals:
     void loadCompleted();
 
-protected:
-    /// Implements generic processing behavior for this thread.
-    virtual bool process() override;
+protected slots:
+    bool process();
+    void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
 
+protected:
     void persist();
     bool backupCurrentFile();
+    void cleanupOldReplacementBackups();
 
     void replaceData(QByteArray data);
     void sendLatestEntityDataToDS();
 
+    QString getTempFilename() const { return _filename + ".temp"; }
+
 private:
     OctreePointer _tree;
     QString _filename;
     int _persistInterval;
     bool _initialLoadComplete;
-    QByteArray _replacementData;
 
     quint64 _loadTimeUSecs;
 

From 8f4f3db9ce673ed53e3f8d388836dde317f4295b Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Thu, 10 May 2018 10:48:14 -0700
Subject: [PATCH 04/18] Add octree backup file cleanup

---
 libraries/octree/src/OctreePersistThread.cpp | 64 +++++++++++++++-----
 1 file changed, 50 insertions(+), 14 deletions(-)

diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 94d76b26ec..93d20ab12a 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -26,6 +26,7 @@
 #include <QJsonArray>
 #include <QJsonObject>
 #include <QJsonDocument>
+#include <QRegExp>
 
 #include <NumericalConstants.h>
 #include <PerfStat.h>
@@ -35,7 +36,11 @@
 #include "OctreeUtils.h"
 #include "OctreeDataUtils.h"
 
-const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
+const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL { 1000 * 30 }; // every 30 seconds
+
+constexpr int MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT { 20 };
+//constexpr int MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 50 * 1000 * 1000 };
+constexpr size_t MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 70000 };
 
 OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, int persistInterval,
                                          bool debugTimestampNow, QString persistAsFileType) :
@@ -280,6 +285,37 @@ QByteArray OctreePersistThread::getPersistFileContents() const {
     return fileContents;
 }
 
+void OctreePersistThread::cleanupOldReplacementBackups() {
+    QRegExp filenameRegex { ".*\\.backup\\.\\d{8}-\\d{6}$" };
+    QFileInfo persistFile { _filename };
+    QDir backupDir { persistFile.absolutePath() };
+    backupDir.setSorting(QDir::SortFlag::Time);
+    backupDir.setFilter(QDir::Filter::Files);
+    qDebug() << "Scanning backups for cleanup:" << backupDir.absolutePath();
+
+    auto count = 0;
+    size_t totalSize = 0;
+    for (auto fileInfo : backupDir.entryInfoList()) {
+        auto absPath = fileInfo.absoluteFilePath();
+        qDebug() << "Found:" << absPath;
+        if (filenameRegex.exactMatch(absPath)) {
+            totalSize += fileInfo.size();
+            if (count >= MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT || totalSize > MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES) {
+                qDebug() << "Removing:" << absPath;
+                QFile backup(absPath);
+                if (backup.remove()) {
+                    qDebug() << "Removed backup:" << absPath;
+                } else {
+                    qWarning() << "Failed to remove backup:" << absPath;
+
+                }
+            }
+            count++;
+        }
+    }
+    qDebug() << "Found" << count << "backups";
+}
+
 void OctreePersistThread::persist() {
     if (_tree->isDirty() && _initialLoadComplete) {
 
@@ -291,20 +327,20 @@ void OctreePersistThread::persist() {
 
         _tree->incrementPersistDataVersion();
 
-        // create our "lock" file to indicate we're saving.
-        QString lockFileName = _filename + ".lock";
-        std::ofstream lockFile(lockFileName.toLocal8Bit().constData(), std::ios::out|std::ios::binary);
-        if(lockFile.is_open()) {
-            qCDebug(octree) << "saving Octree lock file created at:" << lockFileName;
-
-            _tree->writeToFile(_filename.toLocal8Bit().constData(), NULL, _persistAsFileType);
+        QString tempFilename = getTempFilename();
+        qCDebug(octree) << "Saving temporary Octree file to:" << tempFilename;
+        if (_tree->writeToFile(tempFilename.toLocal8Bit().constData(), nullptr, _persistAsFileType)) {
+            QFile tempFile { tempFilename };
             _tree->clearDirtyBit(); // tree is clean after saving
-            qCDebug(octree) << "DONE saving Octree to file...";
-
-            lockFile.close();
-            qCDebug(octree) << "saving Octree lock file closed:" << lockFileName;
-            remove(lockFileName.toLocal8Bit().constData());
-            qCDebug(octree) << "saving Octree lock file removed:" << lockFileName;
+            QFile::remove(_filename);
+            if (tempFile.rename(_filename)) {
+                qCDebug(octree) << "DONE moving temporary Octree file to" << _filename;
+            } else {
+                qCWarning(octree) << "Failed to move temporary Octree file to " << _filename;
+                tempFile.remove();
+            }
+        } else {
+            qCWarning(octree) << "Failed to open temp Octree file at" << tempFilename;
         }
 
         sendLatestEntityDataToDS();

From d0af06939f4b51486578cc5e11314b313e93b73f Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Thu, 10 May 2018 15:51:55 -0700
Subject: [PATCH 05/18] Update persistInterval to use std::chrono in
 OctreePersistThread

---
 assignment-client/src/octree/OctreeServer.cpp |  9 +++++++--
 assignment-client/src/octree/OctreeServer.h   |  2 +-
 libraries/octree/src/OctreePersistThread.cpp  | 12 ++++++------
 libraries/octree/src/OctreePersistThread.h    |  6 +++---
 4 files changed, 17 insertions(+), 12 deletions(-)

diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp
index b8329b4d3e..798f539211 100644
--- a/assignment-client/src/octree/OctreeServer.cpp
+++ b/assignment-client/src/octree/OctreeServer.cpp
@@ -1050,8 +1050,13 @@ void OctreeServer::readConfiguration() {
         _persistAsFileType = "json.gz";
 
         _persistInterval = OctreePersistThread::DEFAULT_PERSIST_INTERVAL;
-        readOptionInt(QString("persistInterval"), settingsSectionObject, _persistInterval);
-        qDebug() << "persistInterval=" << _persistInterval;
+        int result { -1 };
+        readOptionInt(QString("persistInterval"), settingsSectionObject, result);
+        if (result != -1) {
+            _persistInterval = std::chrono::seconds(result);
+        }
+
+        qDebug() << "persistInterval=" << _persistInterval.count();
 
         readOptionBool(QString("persistFileDownload"), settingsSectionObject, _persistFileDownload);
         qDebug() << "persistFileDownload=" << _persistFileDownload;
diff --git a/assignment-client/src/octree/OctreeServer.h b/assignment-client/src/octree/OctreeServer.h
index 5c157b46b2..07b1e334b1 100644
--- a/assignment-client/src/octree/OctreeServer.h
+++ b/assignment-client/src/octree/OctreeServer.h
@@ -193,7 +193,7 @@ protected:
     OctreePersistThread* _persistManager;
     QThread _persistThread;
 
-    int _persistInterval;
+    std::chrono::milliseconds _persistInterval;
     bool _persistFileDownload;
     int _maxBackupVersions;
 
diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 93d20ab12a..ae03ff2463 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -36,13 +36,12 @@
 #include "OctreeUtils.h"
 #include "OctreeDataUtils.h"
 
-const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL { 1000 * 30 }; // every 30 seconds
+const std::chrono::seconds OctreePersistThread::DEFAULT_PERSIST_INTERVAL { 30 };
 
 constexpr int MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT { 20 };
-//constexpr int MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 50 * 1000 * 1000 };
-constexpr size_t MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 70000 };
+constexpr size_t MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 50 * 1000 * 1000 };
 
-OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, int persistInterval,
+OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, std::chrono::milliseconds persistInterval,
                                          bool debugTimestampNow, QString persistAsFileType) :
     _tree(tree),
     _filename(filename),
@@ -187,7 +186,8 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
     }
 
     qDebug() << "Starting timer";
-    QTimer::singleShot(_persistInterval, this, &OctreePersistThread::process);
+    QTimer::singleShot(std::chrono::milliseconds(_persistInterval).count(), this, &OctreePersistThread::process);
+    QTimer::singleShot(std::chrono::milliseconds(1000).count(), this, &OctreePersistThread::process);
 
     emit loadCompleted();
 }
@@ -246,7 +246,7 @@ bool OctreePersistThread::process() {
 
         quint64 now = usecTimestampNow();
         quint64 sinceLastSave = now - _lastCheck;
-        quint64 intervalToCheck = _persistInterval * MSECS_TO_USECS;
+        quint64 intervalToCheck = std::chrono::microseconds(_persistInterval).count();
 
         if (sinceLastSave > intervalToCheck) {
             _lastCheck = now;
diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h
index 5f161c100f..bd114cec06 100644
--- a/libraries/octree/src/OctreePersistThread.h
+++ b/libraries/octree/src/OctreePersistThread.h
@@ -30,11 +30,11 @@ public:
         quint64 lastBackup;
     };
 
-    static const int DEFAULT_PERSIST_INTERVAL;
+    static const std::chrono::seconds DEFAULT_PERSIST_INTERVAL;
 
     OctreePersistThread(OctreePointer tree,
                         const QString& filename,
-                        int persistInterval = DEFAULT_PERSIST_INTERVAL,
+                        std::chrono::milliseconds persistInterval = DEFAULT_PERSIST_INTERVAL,
                         bool debugTimestampNow = false,
                         QString persistAsFileType = "json.gz");
 
@@ -70,7 +70,7 @@ protected:
 private:
     OctreePointer _tree;
     QString _filename;
-    int _persistInterval;
+    std::chrono::milliseconds _persistInterval;
     bool _initialLoadComplete;
 
     quint64 _loadTimeUSecs;

From 9afc9d19c5bab4e2845ca51f3117d423e32f93bf Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Fri, 8 Jun 2018 16:00:31 -0700
Subject: [PATCH 06/18] Cleanup comments and process structure in
 OctreePersistThread

---
 assignment-client/src/octree/OctreeServer.cpp |  2 +-
 libraries/octree/src/OctreePersistThread.cpp  | 59 ++++++-------------
 libraries/octree/src/OctreePersistThread.h    |  5 +-
 3 files changed, 22 insertions(+), 44 deletions(-)

diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp
index 798f539211..c2fbc8ab37 100644
--- a/assignment-client/src/octree/OctreeServer.cpp
+++ b/assignment-client/src/octree/OctreeServer.cpp
@@ -1053,7 +1053,7 @@ void OctreeServer::readConfiguration() {
         int result { -1 };
         readOptionInt(QString("persistInterval"), settingsSectionObject, result);
         if (result != -1) {
-            _persistInterval = std::chrono::seconds(result);
+            _persistInterval = std::chrono::milliseconds(result);
         }
 
         qDebug() << "persistInterval=" << _persistInterval.count();
diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index ae03ff2463..af0bf55b46 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -36,7 +36,8 @@
 #include "OctreeUtils.h"
 #include "OctreeDataUtils.h"
 
-const std::chrono::seconds OctreePersistThread::DEFAULT_PERSIST_INTERVAL { 30 };
+constexpr std::chrono::seconds OctreePersistThread::DEFAULT_PERSIST_INTERVAL { 30 };
+constexpr std::chrono::milliseconds TIME_BETWEEN_PROCESSING { 10 };
 
 constexpr int MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT { 20 };
 constexpr size_t MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 50 * 1000 * 1000 };
@@ -48,7 +49,7 @@ OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& file
     _persistInterval(persistInterval),
     _initialLoadComplete(false),
     _loadTimeUSecs(0),
-    _lastCheck(0),
+    _lastPersistCheck(std::chrono::steady_clock::now()),
     _debugTimestampNow(debugTimestampNow),
     _lastTimeDebug(0),
     _persistAsFileType(persistAsFileType)
@@ -95,7 +96,7 @@ void OctreePersistThread::start() {
         packet->writePrimitive(false);
     }
 
-    qCDebug(octree) << "Sending request for octree data to DS";
+    qCDebug(octree) << "Sending OctreeDataFileRequest to DS";
     nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
 }
 
@@ -111,9 +112,9 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
     if (includesNewData) {
         replacementData = message->readAll();
         replaceData(replacementData);
-        qDebug() << "Got reply to octree data file request, new data sent";
+        qDebug() << "Got OctreeDataFileReply, new data sent";
     } else {
-        qDebug() << "Got reply to octree data file request, current entity data is sufficient";
+        qDebug() << "Got OctreeDataFileReply, current entity data is sufficient";
         
         OctreeUtils::RawEntityData data;
         qCDebug(octree) << "Reading octree data from" << _filename;
@@ -178,16 +179,14 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
 
     _initialLoadComplete = true;
 
-    // Since we just loaded the persistent file, we can consider ourselves as having "just checked" for persistance.
-    _lastCheck = usecTimestampNow(); // we just loaded, no need to save again
+    // Since we just loaded the persistent file, we can consider ourselves as having just persist
+    _lastPersistCheck = std::chrono::steady_clock::now();
 
     if (replacementData.isNull()) {
         sendLatestEntityDataToDS();
     }
 
-    qDebug() << "Starting timer";
-    QTimer::singleShot(std::chrono::milliseconds(_persistInterval).count(), this, &OctreePersistThread::process);
-    QTimer::singleShot(std::chrono::milliseconds(1000).count(), this, &OctreePersistThread::process);
+    QTimer::singleShot(TIME_BETWEEN_PROCESSING.count(), this, &OctreePersistThread::process);
 
     emit loadCompleted();
 }
@@ -233,41 +232,21 @@ bool OctreePersistThread::backupCurrentFile() {
     return true;
 }
 
-bool OctreePersistThread::process() {
-    qDebug() << "Processing...";
+void OctreePersistThread::process() {
+    auto startedProcessingAt = std::chrono::steady_clock::now();
 
-    if (true) { //isStillRunning()) {
-        quint64 MSECS_TO_USECS = 1000;
-        quint64 USECS_TO_SLEEP = 10 * MSECS_TO_USECS; // every 10ms
-        std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
+    _tree->update();
 
-        // do our updates then check to save...
-        _tree->update();
+    auto now = std::chrono::steady_clock::now();
+    auto timeSinceLastPersist = now - _lastPersistCheck;
 
-        quint64 now = usecTimestampNow();
-        quint64 sinceLastSave = now - _lastCheck;
-        quint64 intervalToCheck = std::chrono::microseconds(_persistInterval).count();
-
-        if (sinceLastSave > intervalToCheck) {
-            _lastCheck = now;
-            persist();
-        }
+    qDebug() << "Seconds since last persist: " << std::chrono::duration_cast<std::chrono::seconds>(timeSinceLastPersist).count();
+    if (timeSinceLastPersist > _persistInterval) {
+        _lastPersistCheck = now;
+        persist();
     }
-    
-    // if we were asked to debugTimestampNow do that now...
-    if (_debugTimestampNow) {
-        quint64 now = usecTimestampNow();
-        quint64 sinceLastDebug = now - _lastTimeDebug;
-        quint64 DEBUG_TIMESTAMP_INTERVAL = 600000000; // every 10 minutes
 
-        if (sinceLastDebug > DEBUG_TIMESTAMP_INTERVAL) {
-            _lastTimeDebug = usecTimestampNow(true); // ask for debug output
-        }
-        
-    }
-    //return isStillRunning();  // keep running till they terminate us
-    QTimer::singleShot(1000, this, &OctreePersistThread::process);
-    return true;
+    QTimer::singleShot(TIME_BETWEEN_PROCESSING.count(), this, &OctreePersistThread::process);
 }
 
 void OctreePersistThread::aboutToFinish() {
diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h
index bd114cec06..5bb6e51e5f 100644
--- a/libraries/octree/src/OctreePersistThread.h
+++ b/libraries/octree/src/OctreePersistThread.h
@@ -54,7 +54,7 @@ signals:
     void loadCompleted();
 
 protected slots:
-    bool process();
+    void process();
     void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
 
 protected:
@@ -71,12 +71,11 @@ private:
     OctreePointer _tree;
     QString _filename;
     std::chrono::milliseconds _persistInterval;
+    std::chrono::steady_clock::time_point _lastPersistCheck;
     bool _initialLoadComplete;
 
     quint64 _loadTimeUSecs;
 
-    quint64 _lastCheck;
-
     bool _debugTimestampNow;
     quint64 _lastTimeDebug;
 

From 94069c964d77b608a3888e479f414f66bbf8ff52 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Mon, 11 Jun 2018 08:36:13 -0700
Subject: [PATCH 07/18] Remove unused variable and log message in
 OctreePersistThread

---
 libraries/octree/src/OctreePersistThread.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index af0bf55b46..3ceed6cb0f 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -233,14 +233,11 @@ bool OctreePersistThread::backupCurrentFile() {
 }
 
 void OctreePersistThread::process() {
-    auto startedProcessingAt = std::chrono::steady_clock::now();
-
     _tree->update();
 
     auto now = std::chrono::steady_clock::now();
     auto timeSinceLastPersist = now - _lastPersistCheck;
 
-    qDebug() << "Seconds since last persist: " << std::chrono::duration_cast<std::chrono::seconds>(timeSinceLastPersist).count();
     if (timeSinceLastPersist > _persistInterval) {
         _lastPersistCheck = now;
         persist();

From 11a48915e26dd237dedee187011d7d0ac7090c2b Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Mon, 11 Jun 2018 09:47:50 -0700
Subject: [PATCH 08/18] Fix order of variable initialization in
 OctreePersistThread

---
 libraries/octree/src/OctreePersistThread.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 3ceed6cb0f..8d22e896d1 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -47,9 +47,9 @@ OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& file
     _tree(tree),
     _filename(filename),
     _persistInterval(persistInterval),
+    _lastPersistCheck(std::chrono::steady_clock::now()),
     _initialLoadComplete(false),
     _loadTimeUSecs(0),
-    _lastPersistCheck(std::chrono::steady_clock::now()),
     _debugTimestampNow(debugTimestampNow),
     _lastTimeDebug(0),
     _persistAsFileType(persistAsFileType)
@@ -283,7 +283,6 @@ void OctreePersistThread::cleanupOldReplacementBackups() {
                     qDebug() << "Removed backup:" << absPath;
                 } else {
                     qWarning() << "Failed to remove backup:" << absPath;
-
                 }
             }
             count++;

From fa22ea76301a4ef6ebcbdf20d482884ddb82bc76 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Mon, 11 Jun 2018 11:06:11 -0700
Subject: [PATCH 09/18] Update ES to keep the backup that pushes file size over
 the limit

---
 libraries/octree/src/OctreePersistThread.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 8d22e896d1..8f55953360 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -40,7 +40,7 @@ constexpr std::chrono::seconds OctreePersistThread::DEFAULT_PERSIST_INTERVAL { 3
 constexpr std::chrono::milliseconds TIME_BETWEEN_PROCESSING { 10 };
 
 constexpr int MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT { 20 };
-constexpr size_t MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 50 * 1000 * 1000 };
+constexpr int64_t MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES { 50 * 1000 * 1000 };
 
 OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, std::chrono::milliseconds persistInterval,
                                          bool debugTimestampNow, QString persistAsFileType) :
@@ -269,22 +269,22 @@ void OctreePersistThread::cleanupOldReplacementBackups() {
     backupDir.setFilter(QDir::Filter::Files);
     qDebug() << "Scanning backups for cleanup:" << backupDir.absolutePath();
 
-    auto count = 0;
-    size_t totalSize = 0;
+    int count = 0;
+    int64_t totalSize = 0;
     for (auto fileInfo : backupDir.entryInfoList()) {
         auto absPath = fileInfo.absoluteFilePath();
-        qDebug() << "Found:" << absPath;
+        qDebug() << "  Found:" << absPath;
         if (filenameRegex.exactMatch(absPath)) {
-            totalSize += fileInfo.size();
             if (count >= MAX_OCTREE_REPLACEMENT_BACKUP_FILES_COUNT || totalSize > MAX_OCTREE_REPLACEMENT_BACKUP_FILES_SIZE_BYTES) {
-                qDebug() << "Removing:" << absPath;
+                qDebug() << "  Removing:" << absPath;
                 QFile backup(absPath);
                 if (backup.remove()) {
-                    qDebug() << "Removed backup:" << absPath;
+                    qDebug() << "  Removed backup:" << absPath;
                 } else {
-                    qWarning() << "Failed to remove backup:" << absPath;
+                    qWarning() << "  Failed to remove backup:" << absPath;
                 }
             }
+            totalSize += fileInfo.size();
             count++;
         }
     }

From a558f71fcd752ed4d41164c345f0b831878f605f Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Tue, 12 Jun 2018 15:11:57 -0700
Subject: [PATCH 10/18] Fix resoruces path URLs not working with
 Script.resolvePath

Depending on the context, "/~/" means different things. The ScriptEngine
resolves this path to `/scripts`, but everywhere else it resolves to
`/resources`. This change forces us to expand those paths instead of
storing the "/~/" path, which will be incorrectly resolved once the
ScriptEnging is using it.
---
 .../src/EntityTreeRenderer.cpp                 | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/libraries/entities-renderer/src/EntityTreeRenderer.cpp b/libraries/entities-renderer/src/EntityTreeRenderer.cpp
index 600d1c32a8..52312e64c3 100644
--- a/libraries/entities-renderer/src/EntityTreeRenderer.cpp
+++ b/libraries/entities-renderer/src/EntityTreeRenderer.cpp
@@ -44,6 +44,19 @@ size_t std::hash<EntityItemID>::operator()(const EntityItemID& id) const { retur
 std::function<bool()> EntityTreeRenderer::_entitiesShouldFadeFunction;
 std::function<bool()> EntityTreeRenderer::_renderDebugHullsOperator = [] { return false; };
 
+QString resolveScriptURL(const QString& scriptUrl) {
+    auto normalizedScriptUrl = DependencyManager::get<ResourceManager>()->normalizeURL(scriptUrl);
+    QUrl url { normalizedScriptUrl };
+    if (url.isLocalFile()) {
+        // Outside of the ScriptEngine, /~/ resolves to the /resources directory.
+        // Inside of the ScriptEngine, /~/ resolves to the /scripts directory.
+        // Here we expand local paths in case they are /~/ paths, so they aren't
+        // incorrectly recognized as being located in /scripts when utilized in ScriptEngine.
+        return PathUtils::expandToLocalDataAbsolutePath(url).toString();
+    }
+    return normalizedScriptUrl;
+}
+
 EntityTreeRenderer::EntityTreeRenderer(bool wantScripts, AbstractViewStateInterface* viewState,
                                             AbstractScriptingServicesInterface* scriptingServices) :
     _wantScripts(wantScripts),
@@ -221,7 +234,7 @@ void EntityTreeRenderer::reloadEntityScripts() {
         const auto& renderer = entry.second;
         const auto& entity = renderer->getEntity();
         if (!entity->getScript().isEmpty()) {
-            _entitiesScriptEngine->loadEntityScript(entity->getEntityItemID(), entity->getScript(), true);
+            _entitiesScriptEngine->loadEntityScript(entity->getEntityItemID(), resolveScriptURL(entity->getScript()), true);
         }
     }
 }
@@ -914,8 +927,7 @@ void EntityTreeRenderer::checkAndCallPreload(const EntityItemID& entityID, bool
             entity->scriptHasUnloaded();
         }
         if (shouldLoad) {
-            scriptUrl = DependencyManager::get<ResourceManager>()->normalizeURL(scriptUrl);
-            _entitiesScriptEngine->loadEntityScript(entityID, scriptUrl, reload);
+            _entitiesScriptEngine->loadEntityScript(entityID, resolveScriptURL(scriptUrl), reload);
             entity->scriptHasPreloaded();
         }
     }

From 4e230c9e5dc04d3fa152a62a50c5fa5c6999dd56 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Wed, 13 Jun 2018 11:23:37 -0700
Subject: [PATCH 11/18] Remove redundant setvbuf and update signal/slot syntax
 in OctreeServer

---
 assignment-client/src/octree/OctreeServer.cpp | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp
index c2fbc8ab37..2b2aaeb3e9 100644
--- a/assignment-client/src/octree/OctreeServer.cpp
+++ b/assignment-client/src/octree/OctreeServer.cpp
@@ -1207,12 +1207,8 @@ void OctreeServer::beginRunning() {
 
     beforeRun(); // after payload has been processed
 
-    connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
-    connect(nodeList.data(), SIGNAL(nodeKilled(SharedNodePointer)), SLOT(nodeKilled(SharedNodePointer)));
-
-#ifndef WIN32
-    setvbuf(stdout, nullptr, _IOLBF, 0);
-#endif
+    connect(nodeList.data(), &NodeList::nodeAdded, &OctreeServer::nodeAdded);
+    connect(nodeList.data(), &NodeList::nodeKilled, &OctreeServer::nodeKilled);
 
     nodeList->linkedDataCreateCallback = [this](Node* node) {
         auto queryNodeData = createOctreeQueryNode();

From e49df4a0ab4d69b977a6152c452e111a9ac3a706 Mon Sep 17 00:00:00 2001
From: Zach Fox <fox@highfidelity.io>
Date: Wed, 13 Jun 2018 11:42:18 -0700
Subject: [PATCH 12/18] Fix MS15858: Prevent Interface deadlock during 360 Snap
 capture

---
 .../marketplace/spectator-camera/SpectatorCamera.qml            | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unpublishedScripts/marketplace/spectator-camera/SpectatorCamera.qml b/unpublishedScripts/marketplace/spectator-camera/SpectatorCamera.qml
index 1b3698acd8..e0a503fcc8 100644
--- a/unpublishedScripts/marketplace/spectator-camera/SpectatorCamera.qml
+++ b/unpublishedScripts/marketplace/spectator-camera/SpectatorCamera.qml
@@ -205,7 +205,7 @@ Rectangle {
             // Spectator Camera Preview
             Hifi.ResourceImageItem {
                 id: spectatorCameraPreview;
-                visible: masterSwitch.checked;
+                visible: masterSwitch.checked && !root.processing360Snapshot;
                 url: showCameraView.checked || !HMD.active ? "resource://spectatorCameraFrame" : "resource://hmdPreviewFrame";
                 ready: masterSwitch.checked;
                 mirrorVertically: true;

From 6b159d6124b4c127f4b19ee352264f568b842ca9 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Wed, 13 Jun 2018 11:54:20 -0700
Subject: [PATCH 13/18] Remove redundant read of octree data on startup

---
 libraries/octree/src/OctreePersistThread.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index 8f55953360..d783a1effd 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -109,9 +109,12 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
     bool includesNewData;
     message->readPrimitive(&includesNewData);
     QByteArray replacementData;
+    OctreeUtils::RawOctreeData data;
+    bool hasValidOctreeData { false };
     if (includesNewData) {
         replacementData = message->readAll();
         replaceData(replacementData);
+        hasValidOctreeData = data.readOctreeDataInfoFromFile(_filename);
         qDebug() << "Got OctreeDataFileReply, new data sent";
     } else {
         qDebug() << "Got OctreeDataFileReply, current entity data is sufficient";
@@ -119,6 +122,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
         OctreeUtils::RawEntityData data;
         qCDebug(octree) << "Reading octree data from" << _filename;
         if (data.readOctreeDataInfoFromFile(_filename)) {
+            hasValidOctreeData = true;
             if (data.id.isNull()) {
                 qCDebug(octree) << "Current octree data has a null id, updating";
                 data.resetIdAndVersion();
@@ -138,8 +142,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
     quint64 loadStarted = usecTimestampNow();
     qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
 
-    OctreeUtils::RawOctreeData data;
-    if (data.readOctreeDataInfoFromFile(_filename)) {
+    if (hasValidOctreeData) {
         qDebug() << "Setting entity version info to: " << data.id << data.version;
         _tree->setOctreeVersionInfo(data.id, data.version);
     }

From c2f08dfdba2a8b71fc84f154b08b173670fcccf0 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Wed, 13 Jun 2018 12:07:20 -0700
Subject: [PATCH 14/18] Update Octree to atomically persist data

---
 assignment-client/src/octree/OctreeServer.cpp |  4 +--
 libraries/octree/src/Octree.cpp               |  7 +++--
 libraries/octree/src/OctreePersistThread.cpp  | 31 +++----------------
 libraries/octree/src/OctreePersistThread.h    |  2 --
 4 files changed, 12 insertions(+), 32 deletions(-)

diff --git a/assignment-client/src/octree/OctreeServer.cpp b/assignment-client/src/octree/OctreeServer.cpp
index 2b2aaeb3e9..e993bea358 100644
--- a/assignment-client/src/octree/OctreeServer.cpp
+++ b/assignment-client/src/octree/OctreeServer.cpp
@@ -1207,8 +1207,8 @@ void OctreeServer::beginRunning() {
 
     beforeRun(); // after payload has been processed
 
-    connect(nodeList.data(), &NodeList::nodeAdded, &OctreeServer::nodeAdded);
-    connect(nodeList.data(), &NodeList::nodeKilled, &OctreeServer::nodeKilled);
+    connect(nodeList.data(), &NodeList::nodeAdded, this, &OctreeServer::nodeAdded);
+    connect(nodeList.data(), &NodeList::nodeKilled, this, &OctreeServer::nodeKilled);
 
     nodeList->linkedDataCreateCallback = [this](Node* node) {
         auto queryNodeData = createOctreeQueryNode();
diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index 5f943fabf2..d9da06221d 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -30,6 +30,7 @@
 #include <QJsonObject>
 #include <QJsonArray>
 #include <QFileInfo>
+#include <QSaveFile>
 #include <QString>
 #include <QRegularExpression>
 #include <QRegularExpressionMatch>
@@ -972,10 +973,12 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
         return false;
     }
 
-    QFile persistFile(fileName);
+    QSaveFile persistFile(fileName);
     bool success = false;
     if (persistFile.open(QIODevice::WriteOnly)) {
-        success = persistFile.write(jsonDataForFile) != -1;
+        if (persistFile.write(jsonDataForFile) != -1) {
+            success = persistFile.commit();
+        }
     } else {
         qCritical("Could not write to JSON description of entities.");
     }
diff --git a/libraries/octree/src/OctreePersistThread.cpp b/libraries/octree/src/OctreePersistThread.cpp
index d783a1effd..8b1d766418 100644
--- a/libraries/octree/src/OctreePersistThread.cpp
+++ b/libraries/octree/src/OctreePersistThread.cpp
@@ -62,19 +62,6 @@ OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& file
 void OctreePersistThread::start() {
     cleanupOldReplacementBackups();
 
-    QFile tempFile { getTempFilename() };
-    if (tempFile.exists()) {
-        qWarning(octree) << "Found temporary octree file at" << tempFile.fileName();
-        qDebug(octree) << "Attempting to recover from temporary octree file";
-        QFile::remove(_filename);
-        if (tempFile.rename(_filename)) {
-            qDebug(octree) << "Successfully recovered from temporary octree file";
-        } else {
-            qWarning(octree) << "Failed to recover from temporary octree file";
-            tempFile.remove();
-        }
-    }
-
     auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
     packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
 
@@ -182,7 +169,7 @@ void OctreePersistThread::handleOctreeDataFileReply(QSharedPointer<ReceivedMessa
 
     _initialLoadComplete = true;
 
-    // Since we just loaded the persistent file, we can consider ourselves as having just persist
+    // Since we just loaded the persistent file, we can consider ourselves as having just persisted
     _lastPersistCheck = std::chrono::steady_clock::now();
 
     if (replacementData.isNull()) {
@@ -305,20 +292,12 @@ void OctreePersistThread::persist() {
 
         _tree->incrementPersistDataVersion();
 
-        QString tempFilename = getTempFilename();
-        qCDebug(octree) << "Saving temporary Octree file to:" << tempFilename;
-        if (_tree->writeToFile(tempFilename.toLocal8Bit().constData(), nullptr, _persistAsFileType)) {
-            QFile tempFile { tempFilename };
+        qCDebug(octree) << "Saving Octree data to:" << _filename;
+        if (_tree->writeToFile(_filename.toLocal8Bit().constData(), nullptr, _persistAsFileType)) {
             _tree->clearDirtyBit(); // tree is clean after saving
-            QFile::remove(_filename);
-            if (tempFile.rename(_filename)) {
-                qCDebug(octree) << "DONE moving temporary Octree file to" << _filename;
-            } else {
-                qCWarning(octree) << "Failed to move temporary Octree file to " << _filename;
-                tempFile.remove();
-            }
+            qCDebug(octree) << "DONE persisting Octree data to" << _filename;
         } else {
-            qCWarning(octree) << "Failed to open temp Octree file at" << tempFilename;
+            qCWarning(octree) << "Failed to persist Octree data to" << _filename;
         }
 
         sendLatestEntityDataToDS();
diff --git a/libraries/octree/src/OctreePersistThread.h b/libraries/octree/src/OctreePersistThread.h
index 5bb6e51e5f..0044a8fa5a 100644
--- a/libraries/octree/src/OctreePersistThread.h
+++ b/libraries/octree/src/OctreePersistThread.h
@@ -65,8 +65,6 @@ protected:
     void replaceData(QByteArray data);
     void sendLatestEntityDataToDS();
 
-    QString getTempFilename() const { return _filename + ".temp"; }
-
 private:
     OctreePointer _tree;
     QString _filename;

From 2b49d84a229c3bb091cc7c9b63e0a0adf2d7e7b0 Mon Sep 17 00:00:00 2001
From: Ryan Huffman <ryanhuffman@gmail.com>
Date: Wed, 13 Jun 2018 12:57:06 -0700
Subject: [PATCH 15/18] Add better error logging in Octree::writeToJSONFile

---
 libraries/octree/src/Octree.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/libraries/octree/src/Octree.cpp b/libraries/octree/src/Octree.cpp
index d9da06221d..f0e41d59ea 100644
--- a/libraries/octree/src/Octree.cpp
+++ b/libraries/octree/src/Octree.cpp
@@ -978,9 +978,14 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
     if (persistFile.open(QIODevice::WriteOnly)) {
         if (persistFile.write(jsonDataForFile) != -1) {
             success = persistFile.commit();
+            if (!success) {
+                qCritical() << "Failed to commit to JSON save file:" << persistFile.errorString();
+            }
+        } else {
+            qCritical("Failed to write to JSON file.");
         }
     } else {
-        qCritical("Could not write to JSON description of entities.");
+        qCritical("Failed to open JSON file for writing.");
     }
 
     return success;

From 75a75ecd1a109ec90cc076c7326de1283ba8aa84 Mon Sep 17 00:00:00 2001
From: Clement <clement.brisset@gmail.com>
Date: Tue, 29 May 2018 13:48:23 -0700
Subject: [PATCH 16/18] Migrate old logs

---
 server-console/src/main.js | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/server-console/src/main.js b/server-console/src/main.js
index e145a2b76b..ce2ae34ec3 100644
--- a/server-console/src/main.js
+++ b/server-console/src/main.js
@@ -115,13 +115,28 @@ const UPDATER_LOCK_FULL_PATH = getRootHifiDataDirectory() + "/" + UPDATER_LOCK_F
 
 // Configure log
 global.log = require('electron-log');
-const logFile = getApplicationDataDirectory(true) + '/log.txt';
+const oldLogPath = path.join(getApplicationDataDirectory(), '/logs');
+const logPath = path.join(getApplicationDataDirectory(true), '/logs');
+const oldLogFile = path.join(getApplicationDataDirectory(), '/log.txt');
+const logFile = path.join(getApplicationDataDirectory(true), '/log.txt');
+
+if (oldLogFile != logFile && fs.pathExistsSync(oldLogFile)) {
+    fs.moveSync(oldLogFile, logFile);
+    fs.removeSync(oldLogFile);
+}
 fs.ensureFileSync(logFile); // Ensure file exists
 log.transports.file.maxSize = 5 * 1024 * 1024;
 log.transports.file.file = logFile;
 
+if (oldLogPath != logPath && fs.pathExistsSync(oldLogPath)) {
+    fs.ensureDirSync(logPath);
+    fs.moveSync(oldLogPath, logPath);
+    fs.removeSync(oldLogPath);
+}
+log.debug("Log directory:", logPath);
+
 log.debug("build info", buildInfo);
-log.debug("Root hifi directory is: ", getRootHifiDataDirectory());
+log.debug("Root hifi data directory is: ", getRootHifiDataDirectory());
 
 const ipcMain = electron.ipcMain;
 

From 3ed72569a15ad11fe68c1c361d3984592ebf8383 Mon Sep 17 00:00:00 2001
From: Atlante45 <clement.brisset@gmail.com>
Date: Tue, 29 May 2018 15:53:48 -0700
Subject: [PATCH 17/18] Update fs-extra version

---
 server-console/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/server-console/package.json b/server-console/package.json
index 6dd39ea6f8..565658702b 100644
--- a/server-console/package.json
+++ b/server-console/package.json
@@ -27,7 +27,7 @@
     "cheerio": "^0.19.0",
     "electron-log": "1.1.1",
     "extend": "^3.0.0",
-    "fs-extra": "^1.0.0",
+    "fs-extra": "^6.0.0",
     "node-notifier": "^5.2.1",
     "os-homedir": "^1.0.1",
     "request": "^2.85.0",

From 3ea265f2332d805e46cb1a1d163a811ef1c1c963 Mon Sep 17 00:00:00 2001
From: Atlante45 <clement.brisset@gmail.com>
Date: Tue, 5 Jun 2018 17:00:52 -0700
Subject: [PATCH 18/18] Ensure hifi roaming directory is created

---
 server-console/src/main.js | 58 ++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 34 deletions(-)

diff --git a/server-console/src/main.js b/server-console/src/main.js
index ce2ae34ec3..ac4b0411e7 100644
--- a/server-console/src/main.js
+++ b/server-console/src/main.js
@@ -115,32 +115,43 @@ const UPDATER_LOCK_FULL_PATH = getRootHifiDataDirectory() + "/" + UPDATER_LOCK_F
 
 // Configure log
 global.log = require('electron-log');
-const oldLogPath = path.join(getApplicationDataDirectory(), '/logs');
-const logPath = path.join(getApplicationDataDirectory(true), '/logs');
 const oldLogFile = path.join(getApplicationDataDirectory(), '/log.txt');
 const logFile = path.join(getApplicationDataDirectory(true), '/log.txt');
-
-if (oldLogFile != logFile && fs.pathExistsSync(oldLogFile)) {
-    fs.moveSync(oldLogFile, logFile);
-    fs.removeSync(oldLogFile);
+if (oldLogFile != logFile && fs.existsSync(oldLogFile)) {
+    if (!fs.existsSync(oldLogFile)) {
+        fs.moveSync(oldLogFile, logFile);
+    } else {
+        fs.remove(oldLogFile);
+    }
 }
 fs.ensureFileSync(logFile); // Ensure file exists
 log.transports.file.maxSize = 5 * 1024 * 1024;
 log.transports.file.file = logFile;
 
-if (oldLogPath != logPath && fs.pathExistsSync(oldLogPath)) {
-    fs.ensureDirSync(logPath);
-    fs.moveSync(oldLogPath, logPath);
-    fs.removeSync(oldLogPath);
+log.debug("build info", buildInfo);
+log.debug("Root hifi directory is: ", getRootHifiDataDirectory());
+log.debug("App Data directory:", getApplicationDataDirectory());
+fs.ensureDirSync(getApplicationDataDirectory());
+
+var oldLogPath = path.join(getApplicationDataDirectory(), '/logs');
+var logPath = path.join(getApplicationDataDirectory(true), '/logs');
+if (oldLogPath != logPath && fs.existsSync(oldLogPath)) {
+    if (!fs.existsSync(oldLogPath)) {
+        fs.moveSync(oldLogPath, logPath);
+    } else {
+        fs.remove(oldLogPath);
+    }
 }
+fs.ensureDirSync(logPath);
 log.debug("Log directory:", logPath);
 
-log.debug("build info", buildInfo);
-log.debug("Root hifi data directory is: ", getRootHifiDataDirectory());
+const configPath = path.join(getApplicationDataDirectory(), 'config.json');
+var userConfig = new Config();
+userConfig.load(configPath);
+
 
 const ipcMain = electron.ipcMain;
 
-
 var isShuttingDown = false;
 function shutdown() {
     log.debug("Normal shutdown (isShuttingDown: " + isShuttingDown +  ")");
@@ -247,27 +258,6 @@ function deleteOldFiles(directoryPath, maxAgeInSeconds, filenameRegex) {
     }
 }
 
-var oldLogPath = path.join(getApplicationDataDirectory(), '/logs');
-var logPath = path.join(getApplicationDataDirectory(true), '/logs');
-
-if (oldLogPath != logPath) {
-    console.log("Migrating old logs from " + oldLogPath + " to " + logPath);
-    fs.copy(oldLogPath, logPath, err => {
-        if (err) {
-            console.error(err);
-        } else {
-            console.log('success!');
-        }
-    })
-}
-
-log.debug("Log directory:", logPath);
-log.debug("Data directory:", getRootHifiDataDirectory());
-
-const configPath = path.join(getApplicationDataDirectory(), 'config.json');
-var userConfig = new Config();
-userConfig.load(configPath);
-
 // print out uncaught exceptions in the console
 process.on('uncaughtException', function(err) {
     log.error(err);