Add entity file sync and domain content backups

.clang-format

@@ -1,12 +1,12 @@
 Language: Cpp
 Standard: Cpp11
-BasedOnStyle: "Chromium" 
+BasedOnStyle: "Chromium"
 ColumnLimit: 128
 IndentWidth: 4
 UseTab: Never
 
 BreakBeforeBraces: Custom
-BraceWrapping:   
+BraceWrapping:
   AfterEnum: true
   AfterClass: false
   AfterControlStatement: false
@@ -21,11 +21,11 @@ BraceWrapping:
 
 
 AccessModifierOffset: -4
-AllowShortFunctionsOnASingleLine: InlineOnly  
+AllowShortFunctionsOnASingleLine: InlineOnly
-BreakConstructorInitializers: BeforeColon 
+BreakConstructorInitializers: BeforeColon
 BreakConstructorInitializersBeforeComma: true
 IndentCaseLabels: true
-ReflowComments: false 
+ReflowComments: false
 Cpp11BracedListStyle: false
 ContinuationIndentWidth: 4
 ConstructorInitializerAllOnOneLineOrOnePerLine: false

assignment-client/CMakeLists.txt

@@ -6,6 +6,7 @@ setup_hifi_project(Core Gui Network Script Quick WebSockets)
 if (APPLE)
   set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "@executable_path/../Frameworks")
 endif ()
+set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "/testing/")
 
 setup_memory_debugger()
 

assignment-client/src/Agent.cpp

@@ -340,7 +340,6 @@ void Agent::scriptRequestFinished() {
     request->deleteLater();
 }
 
-
 void Agent::executeScript() {
     _scriptEngine = scriptEngineFactory(ScriptEngine::AGENT_SCRIPT, _scriptContents, _payload);
 

assignment-client/src/entities/EntityServer.cpp

@@ -116,7 +116,6 @@ void EntityServer::beforeRun() {
 void EntityServer::entityCreated(const EntityItem& newEntity, const SharedNodePointer& senderNode) {
 }
 
-
 // EntityServer will use the "special packets" to send list of recently deleted entities
 bool EntityServer::hasSpecialPacketsToSend(const SharedNodePointer& node) {
     bool shouldSendDeletedEntities = false;
@@ -277,7 +276,6 @@ int EntityServer::sendSpecialPackets(const SharedNodePointer& node, OctreeQueryN
     return totalBytes;
 }
 
-
 void EntityServer::pruneDeletedEntities() {
     EntityTreePointer tree = std::static_pointer_cast<EntityTree>(_tree);
     if (tree->hasAnyDeletedEntities()) {

assignment-client/src/entities/EntityServer.h

@@ -30,7 +30,6 @@ struct ViewerSendingStats {
 class SimpleEntitySimulation;
 using SimpleEntitySimulationPointer = std::shared_ptr<SimpleEntitySimulation>;
 
-
 class EntityServer : public OctreeServer, public NewlyCreatedEntityHook {
     Q_OBJECT
 public:
@@ -38,7 +37,7 @@ public:
     ~EntityServer();
 
     // Subclasses must implement these methods
-    virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override ;
+    virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override;
     virtual char getMyNodeType() const override { return NodeType::EntityServer; }
     virtual PacketType getMyQueryMessageType() const override { return PacketType::EntityQuery; }
     virtual const char* getMyServerName() const override { return MODEL_SERVER_NAME; }
@@ -82,12 +81,12 @@ private:
     QReadWriteLock _viewerSendingStatsLock;
     QMap<QUuid, QMap<QUuid, ViewerSendingStats>> _viewerSendingStats;
 
-    static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000; // 45m
+    static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000;                    // 45m
-    static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000; // 1h
+    static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000;                    // 1h
-    int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 45m
+    int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS;  // 45m
-    int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 1h
+    int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS;  // 1h
     QTimer _dynamicDomainVerificationTimer;
     void startDynamicDomainVerification();
 };
 
-#endif // hifi_EntityServer_h
+#endif  // hifi_EntityServer_h

assignment-client/src/octree/OctreeServer.cpp

@@ -33,6 +33,10 @@
 #include <PathUtils.h>
 #include <QtCore/QDir>
 
+#include <OctreeUtils.h>
+
+Q_LOGGING_CATEGORY(octree_server, "hifi.octree-server")
+
 int OctreeServer::_clientCount = 0;
 const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000;
 
@@ -84,6 +88,8 @@ int OctreeServer::_longProcessWait = 0;
 int OctreeServer::_shortProcessWait = 0;
 int OctreeServer::_noProcessWait = 0;
 
+static const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
+
 
 void OctreeServer::resetSendingStats() {
     _averageLoopTime.reset();
@@ -202,7 +208,6 @@ void OctreeServer::trackPacketSendingTime(float time) {
     }
 }
 
-
 void OctreeServer::trackProcessWaitTime(float time) {
     const float MAX_SHORT_TIME = 10.0f;
     const float MAX_LONG_TIME = 100.0f;
@@ -283,8 +288,6 @@ void OctreeServer::initHTTPManager(int port) {
     _httpManager = new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this, this);
 }
 
-const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
-
 bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) {
 
 #ifdef FORCE_CRASH
@@ -922,87 +925,6 @@ void OctreeServer::handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> me
     }
 }
 
-void OctreeServer::handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message) {
-    if (!_isFinished && !_isShuttingDown) {
-        // these messages are only allowed to come from the domain server, so make sure that is the case
-        auto nodeList = DependencyManager::get<NodeList>();
-        if (message->getSenderSockAddr() == nodeList->getDomainHandler().getSockAddr()) {
-            // it's far cleaner to load up the new content upon server startup
-            // so here we just store a special file at our persist path
-            // and then force a stop of the server so that it can pick it up when it relaunches
-            if (!_persistAbsoluteFilePath.isEmpty()) {
-                replaceContentFromMessageData(message->getMessage());
-            } else {
-                qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
-            }
-        } else {
-            qDebug() << "Received an octree file replacement that was not from our domain server - refusing to process";
-        }
-    }
-}
-
-// Message->getMessage() contains a QByteArray representation of the URL to download from
-void OctreeServer::handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message) {
-    qInfo() << "Received request to replace content from a url";
-    if (!_isFinished && !_isShuttingDown) {
-        // This call comes from Interface, so we skip our domain server check
-        // but confirm that we have permissions to replace content sets
-        if (DependencyManager::get<NodeList>()->getThisNodeCanReplaceContent()) {
-            if (!_persistAbsoluteFilePath.isEmpty()) {
-                // Convert message data into our URL
-                QString url(message->getMessage());
-                QUrl modelsURL = QUrl(url, QUrl::StrictMode);
-                QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
-                QNetworkRequest request(modelsURL);
-                QNetworkReply* reply = networkAccessManager.get(request);
-                connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
-                    QNetworkReply::NetworkError networkError = reply->error();
-                    if (networkError == QNetworkReply::NoError) {
-                        QByteArray contents = reply->readAll();
-                        replaceContentFromMessageData(contents);
-                    } else {
-                        qDebug() << "Error downloading JSON from specified file";
-                    }
-                });
-            } else {
-                qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
-            }
-        }
-    }
-}
-
-void OctreeServer::replaceContentFromMessageData(QByteArray content) {
-    //Assume we have compressed data
-    auto compressedOctree = content;
-    QByteArray jsonOctree;
-
-    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
-    if (!wasCompressed) {
-        // the source was not compressed, assume we were sent regular JSON data
-        jsonOctree = compressedOctree;
-    }
-    // check the JSON data to verify it is an object
-    if (QJsonDocument::fromJson(jsonOctree).isObject()) {
-        if (!wasCompressed) {
-            // source was not compressed, we compress it before we write it locally
-            gzip(jsonOctree, compressedOctree);
-        }
-        // write the compressed octree data to a special file
-        auto replacementFilePath = _persistAbsoluteFilePath.append(OctreePersistThread::REPLACEMENT_FILE_EXTENSION);
-        QFile replacementFile(replacementFilePath);
-        if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
-            // we've now written our replacement file, time to take the server down so it can
-            // process it when it comes back up
-            qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
-            setFinished(true);
-        } else {
-            qWarning() << "Could not write replacement octree data to file - refusing to process";
-        }
-    } else {
-        qDebug() << "Received replacement octree file that is invalid - refusing to process";
-    }
-}
-
 bool OctreeServer::readOptionBool(const QString& optionName, const QJsonObject& settingsSectionObject, bool& result) {
     result = false; // assume it doesn't exist
     bool optionAvailable = false;
@@ -1119,7 +1041,19 @@ void OctreeServer::readConfiguration() {
             _persistFilePath = getMyDefaultPersistFilename();
         }
 
+        // If persist filename does not exist, let's see if there is one beside the application binary
+        // If there is, let's copy it over to our target persist directory
+        QDir persistPath { _persistFilePath };
+        _persistAbsoluteFilePath = persistPath.absolutePath();
+
+        if (persistPath.isRelative()) {
+            // if the domain settings passed us a relative path, make an absolute path that is relative to the
+            // default data directory
+            _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
+        }
+
         qDebug() << "persistFilePath=" << _persistFilePath;
+        qDebug() << "persisAbsoluteFilePath=" << _persistAbsoluteFilePath;
 
         _persistAsFileType = "json.gz";
 
@@ -1200,20 +1134,90 @@ void OctreeServer::run() {
 }
 
 void OctreeServer::domainSettingsRequestComplete() {
+    if (_state != OctreeServerState::WaitingForDomainSettings) {
+        qCWarning(octree_server) << "Received domain settings after they have already been received";
+        return;
+    }
+
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
+    packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
+
+    packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
+
+    qDebug(octree_server) << "Received domain settings";
+
+    readConfiguration();
+
+    _state = OctreeServerState::WaitingForOctreeDataNegotation;
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    const DomainHandler& domainHandler = nodeList->getDomainHandler();
+
+    auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
+
+    OctreeUtils::RawOctreeData data;
+    qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
+    if (OctreeUtils::readOctreeDataInfoFromFile(_persistAbsoluteFilePath, &data)) {
+        qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
+        packet->writePrimitive(true);
+        auto id = data.id.toRfc4122();
+        packet->write(id);
+        packet->writePrimitive(data.version);
+    } else {
+        qCWarning(octree_server) << "No octree data found";
+        packet->writePrimitive(false);
+    }
+
+    qCDebug(octree_server) << "Sending request for octree data to DS";
+    nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
+} 
+
+void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
+    bool includesNewData;
+    message->readPrimitive(&includesNewData);
+    QByteArray replaceData;
+    if (includesNewData) {
+        replaceData = message->readAll();
+        qDebug() << "Got reply to octree data file request, new data sent";
+    } else {
+        qDebug() << "Got reply to octree data file request, current entity data is sufficient";
+        
+        OctreeUtils::RawOctreeData data;
+        qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
+        if (OctreeUtils::readOctreeDataInfoFromFile(_persistAbsoluteFilePath, &data)) {
+            if (data.id.isNull()) {
+                qCDebug(octree_server) << "Current octree data has a null id, updating";
+                data.id = QUuid::createUuid();
+                data.version = 0;
+
+                QFile file(_persistAbsoluteFilePath);
+                if (file.open(QIODevice::WriteOnly)) {
+                    auto entityData = data.toByteArray();
+                    file.write(entityData);
+                    file.close();
+                } else {
+                    qCDebug(octree_server) << "Failed to update octree data";
+                }
+            }
+        }
+    }
+    beginRunning(replaceData);
+}
+
+void OctreeServer::beginRunning(QByteArray replaceData) {
+    if (_state == OctreeServerState::Running) {
+        qCWarning(octree_server) << "Server is already running";
+        return;
+    }
+
+    _state = OctreeServerState::Running;
 
     auto nodeList = DependencyManager::get<NodeList>();
 
     // we need to ask the DS about agents so we can ping/reply with them
     nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
 
-    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
-    packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
-    packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacement");
-    packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURL");
-
-    readConfiguration();
-
     beforeRun(); // after payload has been processed
 
     connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
@@ -1233,17 +1237,6 @@ void OctreeServer::domainSettingsRequestComplete() {
 
     // if we want Persistence, set up the local file and persist thread
     if (_wantPersist) {
-        // If persist filename does not exist, let's see if there is one beside the application binary
-        // If there is, let's copy it over to our target persist directory
-        QDir persistPath { _persistFilePath };
-        _persistAbsoluteFilePath = persistPath.absolutePath();
-
-        if (persistPath.isRelative()) {
-            // if the domain settings passed us a relative path, make an absolute path that is relative to the
-            // default data directory
-            _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
-        }
-
         static const QString ENTITY_PERSIST_EXTENSION = ".json.gz";
 
         // force the persist file to end with .json.gz
@@ -1328,7 +1321,7 @@ void OctreeServer::domainSettingsRequestComplete() {
 
         // now set up PersistThread
         _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval,
-                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType);
+                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData);
         _persistThread->initialize(true);
     }
 

assignment-client/src/octree/OctreeServer.h

@@ -27,8 +27,18 @@
 #include "OctreeServerConsts.h"
 #include "OctreeInboundPacketProcessor.h"
 
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(octree_server)
+
 const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total
 
+enum class OctreeServerState {
+    WaitingForDomainSettings,
+    WaitingForOctreeDataNegotation,
+    Running
+};
+
 /// Handles assignments of type OctreeServer - sending octrees to various clients.
 class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler {
     Q_OBJECT
@@ -36,6 +46,8 @@ public:
     OctreeServer(ReceivedMessage& message);
     ~OctreeServer();
 
+    OctreeServerState _state { OctreeServerState::WaitingForDomainSettings };
+
     /// allows setting of run arguments
     void setArguments(int argc, char** argv);
 
@@ -137,8 +149,9 @@ private slots:
     void domainSettingsRequestComplete();
     void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message);
+    //void handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message);
-    void handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message);
+    //void handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
     void removeSendThread();
 
 protected:
@@ -159,11 +172,13 @@ protected:
     QString getFileLoadTime();
     QString getConfiguration();
     QString getStatusLink();
+
+    void beginRunning(QByteArray replaceData);
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
     virtual UniqueSendThread newSendThread(const SharedNodePointer& node);
 
-    void replaceContentFromMessageData(QByteArray content);
+    //void replaceContentFromMessageData(QByteArray content);
 
     int _argc;
     const char** _argv;

assignment-client/src/scripts/EntityScriptServer.cpp

@@ -178,7 +178,7 @@ void EntityScriptServer::updateEntityPPS() {
     int numRunningScripts = _entitiesScriptEngine->getNumRunningEntityScripts();
     int pps;
     if (std::numeric_limits<int>::max() / _entityPPSPerScript < numRunningScripts) {
-        qWarning() << QString("Integer multiplaction would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
+        qWarning() << QString("Integer multiplication would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
         pps = std::numeric_limits<int>::max();
         pps = std::min(_maxEntityPPS, pps);
     } else {

domain-server/CMakeLists.txt

@@ -22,7 +22,17 @@ setup_memory_debugger()
 symlink_or_copy_directory_beside_target(${_SHOULD_SYMLINK_RESOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/resources" "resources")
 
 # link the shared hifi libraries
-link_hifi_libraries(embedded-webserver networking shared avatars)
+link_hifi_libraries(embedded-webserver networking shared avatars octree)
+
+add_dependency_external_projects(quazip)
+
+find_package(QuaZip REQUIRED)
+target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${QUAZIP_INCLUDE_DIRS})
+target_link_libraries(${TARGET_NAME} ${QUAZIP_LIBRARIES})
+
+if (WIN32)
+  add_paths_to_fixup_libs(${QUAZIP_DLL_PATH})
+endif ()
 
 # find OpenSSL
 find_package(OpenSSL REQUIRED)

domain-server/src/DomainContentBackupManager.cpp

@@ -0,0 +1,303 @@
+//
+//  DomainContentBackupManager.cpp
+//  libraries/octree/src
+//
+//  Created by Brad Hefta-Gaub on 8/21/13.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include <chrono>
+#include <thread>
+
+#include <cstdio>
+#include <fstream>
+#include <time.h>
+
+#include <QDateTime>
+#include <QDebug>
+#include <QDir>
+#include <QDirIterator>
+#include <QFile>
+#include <QJsonArray>
+#include <QJsonObject>
+#include <QJsonDocument>
+
+#include <NumericalConstants.h>
+#include <PerfStat.h>
+#include <PathUtils.h>
+
+#include "DomainServer.h"
+#include "DomainContentBackupManager.h"
+const int DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL = 1000 * 30;  // every 30 seconds
+
+// Backup format looks like: daily_backup-TIMESTAMP.zip
+const static QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
+const static QString DATETIME_FORMAT_RE("\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}");
+
+void DomainContentBackupManager::addCreateBackupHandler(CreateBackupHandler handler) {
+    _backupHandlers.push_back(handler);
+}
+
+DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory,
+                                                       const QJsonObject& settings,
+                                                       int persistInterval,
+                                                       bool debugTimestampNow)
+    : _backupDirectory(backupDirectory),
+    _persistInterval(persistInterval),
+    _initialLoadComplete(false),
+    _loadTimeUSecs(0),
+    _lastCheck(0),
+    _debugTimestampNow(debugTimestampNow),
+    _lastTimeDebug(0) {
+    parseSettings(settings);
+}
+
+void DomainContentBackupManager::parseSettings(const QJsonObject& settings) {
+    qDebug() << settings << settings["backups"] << settings["backups"].isArray();
+    if (settings["backups"].isArray()) {
+        const QJsonArray& backupRules = settings["backups"].toArray();
+        qCDebug(domain_server) << "BACKUP RULES:";
+
+        for (const QJsonValue& value : backupRules) {
+            QJsonObject obj = value.toObject();
+
+            int interval = 0;
+            int count = 0;
+
+            QJsonValue intervalVal = obj["backupInterval"];
+            if (intervalVal.isString()) {
+                interval = intervalVal.toString().toInt();
+            } else {
+                interval = intervalVal.toInt();
+            }
+
+            QJsonValue countVal = obj["maxBackupVersions"];
+            if (countVal.isString()) {
+                count = countVal.toString().toInt();
+            } else {
+                count = countVal.toInt();
+            }
+
+            auto name = obj["Name"].toString();
+            auto format = obj["format"].toString();
+            format = name.replace(" ", "_").toLower() + "-";
+
+            qCDebug(domain_server) << "    Name:" << name;
+            qCDebug(domain_server) << "        format:" << format;
+            qCDebug(domain_server) << "        interval:" << interval;
+            qCDebug(domain_server) << "        count:" << count;
+
+            BackupRule newRule = { name, interval, format, count, 0 };
+
+            newRule.lastBackupSeconds = getMostRecentBackupTimeInSecs(format);
+
+            if (newRule.lastBackupSeconds > 0) {
+                auto now = QDateTime::currentSecsSinceEpoch();
+                auto sinceLastBackup = now - newRule.lastBackupSeconds;
+                qCDebug(domain_server).noquote() << "        lastBackup:" <<  formatSecTime(sinceLastBackup) << "ago";
+            } else {
+                qCDebug(domain_server) << "        lastBackup: NEVER";
+            }
+
+            _backupRules << newRule;
+        }
+    } else {
+        qCDebug(domain_server) << "BACKUP RULES: NONE";
+    }
+}
+
+int64_t DomainContentBackupManager::getMostRecentBackupTimeInSecs(const QString& format) {
+    int64_t mostRecentBackupInSecs = 0;
+
+    QString mostRecentBackupFileName;
+    QDateTime mostRecentBackupTime;
+
+    bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
+
+    if (recentBackup) {
+        mostRecentBackupInSecs = mostRecentBackupTime.toSecsSinceEpoch();
+    }
+
+    return mostRecentBackupInSecs;
+}
+
+bool DomainContentBackupManager::process() {
+    if (isStillRunning()) {
+        constexpr int64_t MSECS_TO_USECS = 1000;
+        constexpr int64_t USECS_TO_SLEEP = 10 * MSECS_TO_USECS;  // every 10ms
+        std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
+
+        int64_t now = usecTimestampNow();
+        int64_t sinceLastSave = now - _lastCheck;
+        int64_t intervalToCheck = _persistInterval * MSECS_TO_USECS;
+
+        if (sinceLastSave > intervalToCheck) {
+            _lastCheck = now;
+            persist();
+        }
+    }
+
+    // if we were asked to debugTimestampNow do that now...
+    if (_debugTimestampNow) {
+
+        quint64 now = usecTimestampNow();
+        quint64 sinceLastDebug = now - _lastTimeDebug;
+        quint64 DEBUG_TIMESTAMP_INTERVAL = 600000000;  // every 10 minutes
+
+        if (sinceLastDebug > DEBUG_TIMESTAMP_INTERVAL) {
+            _lastTimeDebug = usecTimestampNow(true);  // ask for debug output
+        }
+    }
+
+    return isStillRunning();
+}
+
+void DomainContentBackupManager::aboutToFinish() {
+    qCDebug(domain_server) << "Persist thread about to finish...";
+    persist();
+}
+
+void DomainContentBackupManager::persist() {
+    QDir backupDir { _backupDirectory };
+    backupDir.mkpath(".");
+
+    // create our "lock" file to indicate we're saving.
+    QString lockFileName = _backupDirectory + "/running.lock";
+
+    std::ofstream lockFile(qPrintable(lockFileName), std::ios::out | std::ios::binary);
+    if (lockFile.is_open()) {
+        backup();
+
+        lockFile.close();
+        remove(qPrintable(lockFileName));
+    }
+}
+
+bool DomainContentBackupManager::getMostRecentBackup(const QString& format,
+                                                     QString& mostRecentBackupFileName,
+                                                     QDateTime& mostRecentBackupTime) {
+    QRegExp formatRE { QRegExp::escape(format) + "(" + DATETIME_FORMAT_RE + ")" + "\\.zip" };
+
+    QStringList filters;
+    filters << format + "*.zip";
+
+    bool bestBackupFound = false;
+    QString bestBackupFile;
+    QDateTime bestBackupFileTime;
+
+    // Iterate over all of the backup files in the persist location
+    QDirIterator dirIterator(_backupDirectory, filters, QDir::Files | QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
+    while (dirIterator.hasNext()) {
+        dirIterator.next();
+        auto fileName = dirIterator.fileInfo().fileName();
+
+        if (formatRE.exactMatch(fileName)) {
+            auto datetime = formatRE.cap(1);
+            auto createdAt = QDateTime::fromString(datetime, DATETIME_FORMAT);
+
+            if (!createdAt.isValid()) {
+                qDebug() << "Skipping backup with invalid timestamp: " << datetime;
+                continue;
+            }
+
+            qDebug() << "Checking " << dirIterator.fileInfo().filePath();
+
+            // Based on last modified date, track the most recently modified file as the best backup
+            if (createdAt > bestBackupFileTime) {
+                bestBackupFound = true;
+                bestBackupFile = dirIterator.filePath();
+                bestBackupFileTime = createdAt;
+            }
+        } else {
+            qDebug() << "NO match: " << fileName << formatRE;
+        }
+    }
+
+    // If we found a backup then return the results
+    if (bestBackupFound) {
+        mostRecentBackupFileName = bestBackupFile;
+        mostRecentBackupTime = bestBackupFileTime;
+    }
+    return bestBackupFound;
+}
+
+void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
+    QDir backupDir { _backupDirectory };
+    if (backupDir.exists() && rule.maxBackupVersions > 0) {
+        qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name << "...";
+
+        auto matchingFiles =
+                backupDir.entryInfoList({ rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
+
+        int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions;
+        for (int i = 0; i < backupsToDelete; ++i) {
+            auto fileInfo = matchingFiles[i].absoluteFilePath();
+            QFile backupFile(fileInfo);
+            if (backupFile.remove()) {
+                qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName();
+            } else {
+                qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName();
+            }
+        }
+
+        qCDebug(domain_server) << "Done rolling old backup versions...";
+    } else {
+        qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "."
+                                << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
+                                << " No need to roll backups...";
+    }
+}
+
+void DomainContentBackupManager::backup() {
+    auto nowDateTime = QDateTime::currentDateTime();
+    auto nowSeconds = nowDateTime.toSecsSinceEpoch();
+
+    for (BackupRule& rule : _backupRules) {
+        auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds;
+
+        qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup
+                                << "] "
+                                << "compared to backup interval [" << rule.intervalSeconds << "]...";
+
+        if (secondsSinceLastBackup > rule.intervalSeconds) {
+            qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name
+                                    << "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now...";
+
+            auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT);
+            auto fileName = "backup-" + rule.extensionFormat + timestamp + ".zip";
+            auto zip = new QuaZip(_backupDirectory + "/" + fileName);
+            zip->open(QuaZip::mdAdd);
+
+            for (auto& handler : _backupHandlers) {
+                handler(zip);
+            }
+
+            zip->close();
+
+            qDebug() << "Created backup: " << fileName;
+
+            removeOldBackupVersions(rule);
+
+            if (rule.maxBackupVersions > 0) {
+                // Execute backup
+                auto result = true;
+                if (result) {
+                    qCDebug(domain_server) << "DONE backing up persist file...";
+                    rule.lastBackupSeconds = nowSeconds;
+                } else {
+                    qCDebug(domain_server) << "ERROR in backing up persist file...";
+                    perror("ERROR in backing up persist file");
+                }
+            } else {
+                qCDebug(domain_server) << "This backup rule" << rule.name << " has Max Rolled Backup Versions less than 1 ["
+                                        << rule.maxBackupVersions << "]."
+                                        << " There are no backups to be done...";
+            }
+        } else {
+            qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]...";
+        }
+    }
+}

domain-server/src/DomainContentBackupManager.h

@@ -0,0 +1,88 @@
+//
+//  DomainContentBackupManager.h
+//  libraries/octree/src
+//
+//  Created by Brad Hefta-Gaub on 8/21/13.
+//  Copyright 2013 High Fidelity, Inc.
+//
+//
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_DomainContentBackupManager_h
+#define hifi_DomainContentBackupManager_h
+
+#include <QString>
+#include <GenericThread.h>
+#include <QVector>
+
+#include <quazip5/quazip.h>
+#include <quazip5/quazipfile.h>
+
+#include <functional>
+
+using BackupResult = std::vector<QString>;
+using CreateBackupHandler = std::function<void(QuaZip* quazip)>;
+using RecoverBackupHandler = std::function<void()>;
+
+class DomainContentBackupManager : public GenericThread {
+    Q_OBJECT
+public:
+    class BackupRule {
+    public:
+        QString name;
+        int intervalSeconds;
+        QString extensionFormat;
+        int maxBackupVersions;
+        qint64 lastBackupSeconds;
+    };
+
+    static const int DEFAULT_PERSIST_INTERVAL;
+
+    DomainContentBackupManager(const QString& rootBackupDirectory,
+                               const QJsonObject& settings,
+                               int persistInterval = DEFAULT_PERSIST_INTERVAL,
+                               bool debugTimestampNow = false);
+
+    void addCreateBackupHandler(CreateBackupHandler handler);
+    bool isInitialLoadComplete() const { return _initialLoadComplete; }
+    int64_t getLoadElapsedTime() const { return _loadTimeUSecs; }
+
+    void aboutToFinish();  /// call this to inform the persist thread that the owner is about to finish to support final persist
+
+    void replaceData(QByteArray data);
+
+signals:
+    void loadCompleted();
+
+protected:
+    /// Implements generic processing behavior for this thread.
+    bool process() override;
+
+    void persist();
+    void backup();
+    void removeOldBackupVersions(const BackupRule& rule);
+    bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
+    int64_t getMostRecentBackupTimeInSecs(const QString& format);
+    void parseSettings(const QJsonObject& settings);
+
+private:
+    QString _backupDirectory;
+    std::vector<CreateBackupHandler> _backupHandlers;
+    int _persistInterval;
+    bool _initialLoadComplete;
+
+    int64_t _loadTimeUSecs;
+
+    time_t _lastPersistTime;
+    int64_t _lastCheck;
+    bool _wantBackup{ true };
+    QVector<BackupRule> _backupRules;
+
+    bool _debugTimestampNow;
+    int64_t _lastTimeDebug;
+};
+
+#endif  // hifi_DomainContentBackupManager_h

domain-server/src/DomainServer.cpp

@@ -24,6 +24,7 @@
 #include <QTimer>
 #include <QUrlQuery>
 #include <QCommandLineParser>
+#include <QUuid>
 
 #include <AccountManager.h>
 #include <AssetClient.h>
@@ -47,7 +48,14 @@
 #include "DomainServerNodeData.h"
 #include "NodeConnectionData.h"
 
+#include <Gzip.h>
+
+#include <OctreeUtils.h>
+
+Q_LOGGING_CATEGORY(domain_server, "hifi.domain_server")
+
 const QString ACCESS_TOKEN_KEY_PATH = "metaverse.access_token";
+const QString DomainServer::REPLACEMENT_FILE_EXTENSION = ".replace";
 
 int const DomainServer::EXIT_CODE_REBOOT = 234923;
 
@@ -280,6 +288,30 @@ DomainServer::DomainServer(int argc, char* argv[]) :
             qDebug() << "Ignoring subnet in whitelist, invalid ip portion: " << subnet;
         }
     }
+
+    qDebug() << "Starting persist thread";
+    if (QDir(getEntitiesDirPath()).mkpath(".")) {
+        qCDebug(domain_server) << "Created entities data directory";
+    }
+    maybeHandleReplacementEntityFile();
+    auto entitiesFilePath = getEntitiesFilePath();
+    _contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), _settingsManager.responseObjectForType("6")["entity_server_settings"].toObject()));
+    _contentManager->addCreateBackupHandler([entitiesFilePath](QuaZip* zip) {
+        qDebug() << "Creating a backup from handler";
+
+        QFile entitiesFile { entitiesFilePath };
+
+        if (entitiesFile.open(QIODevice::ReadOnly)) {
+            QuaZipFile zipFile { zip };
+            zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo("models.json.gz", entitiesFilePath));
+            zipFile.write(entitiesFile.readAll());
+            zipFile.close();
+            if (zipFile.getZipError() != UNZ_OK) {
+                qDebug() << "Failed to write entities file to backup:" << zipFile.getZipError();
+            }
+        }
+    });
+    _contentManager->initialize(true);
 }
 
 void DomainServer::parseCommandLine() {
@@ -352,6 +384,11 @@ DomainServer::~DomainServer() {
 
     // destroy the LimitedNodeList before the DomainServer QCoreApplication is down
     DependencyManager::destroy<LimitedNodeList>();
+
+    if (_contentManager) {
+        _contentManager->aboutToFinish();
+        _contentManager->terminating();
+    }
 }
 
 void DomainServer::queuedQuit(QString quitMessage, int exitCode) {
@@ -691,6 +728,12 @@ void DomainServer::setupNodeListAndAssignments() {
     packetReceiver.registerListener(PacketType::ICEServerHeartbeatDenied, this, "processICEServerHeartbeatDenialPacket");
     packetReceiver.registerListener(PacketType::ICEServerHeartbeatACK, this, "processICEServerHeartbeatACK");
 
+    packetReceiver.registerListener(PacketType::OctreeDataFileRequest, this, "processOctreeDataRequestMessage");
+    packetReceiver.registerListener(PacketType::OctreeDataPersist, this, "processOctreeDataPersistMessage");
+
+    packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacementRequest");
+    packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURLRequest");
+
     // set a custom packetVersionMatch as the verify packet operator for the udt::Socket
     nodeList->setPacketFilterOperator(&DomainServer::isPacketVerified);
 
@@ -1605,6 +1648,7 @@ void DomainServer::sendHeartbeatToIceServer() {
             qWarning() << "Waiting for keypair generation to complete before sending ICE heartbeat.";
 
             if (!limitedNodeList->getSessionUUID().isNull()) {
+                qDebug() << "generating keypair";
                 accountManager->generateNewDomainKeypair(limitedNodeList->getSessionUUID());
             } else {
                 qWarning() << "Attempting to send ICE server heartbeat with no domain ID. This is not supported";
@@ -1695,10 +1739,88 @@ void DomainServer::sendHeartbeatToIceServer() {
     } else {
         qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server.";
         qDebug() << "Waiting for" << _iceServerAddr << "host lookup response";
-
     }
 }
 
+void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message) {
+    qDebug() << "Received octree data persist message";
+    auto data = message->readAll();
+    auto filePath = getEntitiesFilePath();
+
+    QFile f(filePath);
+    if (f.open(QIODevice::WriteOnly)) {
+        f.write(data);
+        OctreeUtils::RawOctreeData octreeData;
+        if (OctreeUtils::readOctreeDataInfoFromData(data, &octreeData)) {
+            qCDebug(domain_server) << "Wrote new entiteis file" << octreeData.id << octreeData.version;
+        } else {
+            qCDebug(domain_server) << "Failed to read new octree data info";
+        }
+    } else {
+        qCDebug(domain_server) << "Failed to write new entities file";
+    }
+}
+
+QString DomainServer::getContentBackupDir() {
+    return PathUtils::getAppDataFilePath("backup");
+}
+
+QString DomainServer::getEntitiesDirPath() {
+    return PathUtils::getAppDataFilePath("entities");
+}
+
+QString DomainServer::getEntitiesFilePath() {
+    return PathUtils::getAppDataFilePath("entities/models.json.gz");
+}
+
+QString DomainServer::getEntitiesReplacementFilePath() {
+    return getEntitiesFilePath().append(REPLACEMENT_FILE_EXTENSION);
+}
+
+void DomainServer::processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message) {
+    qDebug() << "Got request for octree data from " << message->getSenderSockAddr();
+
+    bool remoteHasExistingData { false };
+    QUuid id;
+    int version;
+    message->readPrimitive(&remoteHasExistingData);
+    if (remoteHasExistingData) {
+        auto idData = message->read(16);
+        id = QUuid::fromRfc4122(idData);
+        message->readPrimitive(&version);
+        qCDebug(domain_server) << "Entity server does have existing data: ID(" << id << ") DataVersion(" << version << ")";
+    } else {
+        qCDebug(domain_server) << "Entity server does not have existing data";
+    }
+    auto entityFilePath = getEntitiesFilePath();
+
+    //QFile file(entityFilePath);
+    auto reply = NLPacketList::create(PacketType::OctreeDataFileReply, QByteArray(), true, true);
+    OctreeUtils::RawOctreeData data;
+    if (OctreeUtils::readOctreeDataInfoFromFile(entityFilePath, &data)) {
+        if (data.id == id && data.version <= version) {
+            qCDebug(domain_server) << "ES has sufficient octree data, not sending data";
+            reply->writePrimitive(false);
+        } else {
+            qCDebug(domain_server) << "Sending newer octree data to ES";
+            QFile file(entityFilePath);
+            if (file.open(QIODevice::ReadOnly)) {
+                reply->writePrimitive(true);
+                reply->write(file.readAll());
+            } else {
+                qCDebug(domain_server) << "Unable to load entity file";
+                reply->writePrimitive(false);
+            }
+        }
+    } else {
+        qCDebug(domain_server) << "Domain server does not have valid octree data";
+        reply->writePrimitive(false);
+    }
+
+    auto nodeList = DependencyManager::get<LimitedNodeList>();
+    nodeList->sendPacketList(std::move(reply), message->getSenderSockAddr());
+}
+
 void DomainServer::processNodeJSONStatsPacket(QSharedPointer<ReceivedMessage> packetList, SharedNodePointer sendingNode) {
     auto nodeData = static_cast<DomainServerNodeData*>(sendingNode->getLinkedData());
     if (nodeData) {
@@ -3105,9 +3227,64 @@ void DomainServer::setupGroupCacheRefresh() {
     }
 }
 
+void DomainServer::maybeHandleReplacementEntityFile() {
+    QFile replacementFile(getEntitiesReplacementFilePath());
+    if (replacementFile.exists()) {
+        qCDebug(domain_server) << "Replacing existing entity date with replacement file";
+        QFile currentFile(getEntitiesFilePath());
+        if (currentFile.exists()) {
+            if (currentFile.remove()) {
+                qCDebug(domain_server) << "Removed existing entity file";
+            } else {
+                qCWarning(domain_server) << "Failled to remove existing entity file";
+            }
+        }
+        if (replacementFile.rename(getEntitiesFilePath())) {
+            qCDebug(domain_server) << "Successfully updated entities data file with replacement file";
+        } else {
+            qCWarning(domain_server) << "Failed to update entities data file with replacement file";
+        }
+    }
+}
+
 void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
     // enumerate the nodes and find any octree type servers with active sockets
 
+    //Assume we have compressed data
+    auto compressedOctree = octreeFile;
+    QByteArray jsonOctree;
+
+    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
+    if (!wasCompressed) {
+        // the source was not compressed, assume we were sent regular JSON data
+        jsonOctree = compressedOctree;
+    }
+
+    OctreeUtils::RawOctreeData data;
+    if (OctreeUtils::readOctreeDataInfoFromData(jsonOctree, &data)) {
+        data.id = QUuid::createUuid();
+        data.version = 0;
+
+        gzip(data.toByteArray(), compressedOctree);
+
+        // write the compressed octree data to a special file
+        auto replacementFilePath = getEntitiesReplacementFilePath();
+        QFile replacementFile(replacementFilePath);
+        if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
+            // we've now written our replacement file, time to take the server down so it can
+            // process it when it comes back up
+            qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
+
+            QMetaObject::invokeMethod(this, "restart", Qt::QueuedConnection);
+        } else {
+            qWarning() << "Could not write replacement octree data to file - refusing to process";
+        }
+    } else {
+        qDebug() << "Received replacement octree file that is invalid - refusing to process";
+    }
+
+
+    return;
     auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
     limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
         return node->getType() == NodeType::EntityServer && node->getActiveSocket();
@@ -3121,3 +3298,37 @@ void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
         limitedNodeList->sendPacketList(std::move(octreeFilePacketList), *octreeNode);
     });
 }
+
+void DomainServer::handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message) {
+    qInfo() << "Received request to replace content from a url";
+    auto node = DependencyManager::get<LimitedNodeList>()->findNodeWithAddr(message->getSenderSockAddr());
+    if (node) {
+        qDebug() << "Found node: " << node->getCanReplaceContent();
+    }
+    if (node->getCanReplaceContent()) {
+        // Convert message data into our URL
+        QString url(message->getMessage());
+        QUrl modelsURL = QUrl(url, QUrl::StrictMode);
+        QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+        QNetworkRequest request(modelsURL);
+        QNetworkReply* reply = networkAccessManager.get(request);
+        connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
+            QNetworkReply::NetworkError networkError = reply->error();
+            if (networkError == QNetworkReply::NoError) {
+                handleOctreeFileReplacement(reply->readAll());
+            } else {
+                qDebug() << "Error downloading JSON from specified file: " << modelsURL;
+            }
+        });
+    }
+}
+
+
+
+
+void DomainServer::handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message) {
+    auto node = DependencyManager::get<NodeList>()->nodeWithUUID(message->getSourceID());
+    if (node->getCanReplaceContent()) {
+        handleOctreeFileReplacement(message->readAll());
+    }
+}

domain-server/src/DomainServer.h

@@ -32,9 +32,14 @@
 #include "DomainServerSettingsManager.h"
 #include "DomainServerWebSessionData.h"
 #include "WalletTransaction.h"
+#include "DomainContentBackupManager.h"
 
 #include "PendingAssignedNodeData.h"
 
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(domain_server)
+
 typedef QSharedPointer<Assignment> SharedAssignmentPointer;
 typedef QMultiHash<QUuid, WalletTransaction*> TransactionHash;
 
@@ -65,6 +70,8 @@ public:
     bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
     bool handleHTTPSRequest(HTTPSConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
 
+    static const QString REPLACEMENT_FILE_EXTENSION;
+
 public slots:
     /// Called by NodeList to inform us a node has been added
     void nodeAdded(SharedNodePointer node);
@@ -84,6 +91,13 @@ private slots:
     void processICEServerHeartbeatDenialPacket(QSharedPointer<ReceivedMessage> message);
     void processICEServerHeartbeatACK(QSharedPointer<ReceivedMessage> message);
 
+    void handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeFileReplacement(QByteArray octreeFile);
+
+    void processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message);
+    void processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message);
+
     void setupPendingAssignmentCredits();
     void sendPendingTransactionsToServer();
 
@@ -91,8 +105,7 @@ private slots:
     void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); }
     void sendHeartbeatToIceServer();
 
-    void handleConnectedNode(SharedNodePointer newNode);
+    void handleConnectedNode(SharedNodePointer newNode); 
-
     void handleTempDomainSuccess(QNetworkReply& requestReply);
     void handleTempDomainError(QNetworkReply& requestReply);
 
@@ -109,8 +122,6 @@ private slots:
     void handleSuccessfulICEServerAddressUpdate(QNetworkReply& requestReply);
     void handleFailedICEServerAddressUpdate(QNetworkReply& requestReply);
 
-    void handleOctreeFileReplacement(QByteArray octreeFile);
-
     void updateReplicatedNodes();
     void updateDownstreamNodes();
     void updateUpstreamNodes();
@@ -127,6 +138,13 @@ private:
     const QUuid& getID();
     void parseCommandLine();
 
+    QString getContentBackupDir();
+    QString getEntitiesDirPath();
+    QString getEntitiesFilePath();
+    QString getEntitiesReplacementFilePath();
+
+    void maybeHandleReplacementEntityFile();
+
     void setupNodeListAndAssignments();
     bool optionallySetupOAuth();
     bool optionallyReadX509KeyAndCertificate();
@@ -252,6 +270,8 @@ private:
     bool _sendICEServerAddressToMetaverseAPIInProgress { false };
     bool _sendICEServerAddressToMetaverseAPIRedo { false };
 
+    std::unique_ptr<DomainContentBackupManager> _contentManager { nullptr };
+
     QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
 
     QThread _assetClientThread;

interface/src/Application.cpp

@@ -6240,13 +6240,15 @@ bool Application::askToReplaceDomainContent(const QString& url) {
                     // Given confirmation, send request to domain server to replace content
                     qCDebug(interfaceapp) << "Attempting to replace domain content: " << url;
                     QByteArray urlData(url.toUtf8());
-                    auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
+                    auto limitedNodeList = DependencyManager::get<NodeList>();
+                    const auto& domainHandler = limitedNodeList->getDomainHandler();
                     limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
                             return node->getType() == NodeType::EntityServer && node->getActiveSocket();
-                        }, [&urlData, limitedNodeList](const SharedNodePointer& octreeNode) {
+                        }, [&urlData, limitedNodeList, &domainHandler](const SharedNodePointer& octreeNode) {
                             auto octreeFilePacket = NLPacket::create(PacketType::OctreeFileReplacementFromUrl, urlData.size(), true);
                             octreeFilePacket->write(urlData);
-                            limitedNodeList->sendPacket(std::move(octreeFilePacket), *octreeNode);
+                            qDebug() << "WRiting url data: " << urlData;
+                            limitedNodeList->sendPacket(std::move(octreeFilePacket), domainHandler.getSockAddr());
                         });
                     auto addressManager = DependencyManager::get<AddressManager>();
                     addressManager->handleLookupString(DOMAIN_SPAWNING_POINT);

interface/src/ui/DomainConnectionModel.cpp

@@ -98,4 +98,4 @@ void DomainConnectionModel::refresh() {
     //inform view that we want refresh data
     beginResetModel();
     endResetModel();
-}
+}

libraries/entities/src/EntityTree.cpp

@@ -2244,6 +2244,8 @@ bool EntityTree::writeToMap(QVariantMap& entityDescription, OctreeElementPointer
     if (! entityDescription.contains("Entities")) {
         entityDescription["Entities"] = QVariantList();
     }
+    entityDescription["DataVersion"] = ++_persistDataVersion;
+    entityDescription["Id"] = _persistID;
     QScriptEngine scriptEngine;
     RecurseOctreeToMapOperator theOperator(entityDescription, element, &scriptEngine, skipDefaultValues,
                                             skipThoseWithBadParents, _myAvatar);
@@ -2256,6 +2258,14 @@ bool EntityTree::readFromMap(QVariantMap& map) {
     int contentVersion = map["Version"].toInt();
     bool needsConversion = (contentVersion < (int)EntityVersion::ZoneLightInheritModes);
 
+    if (map.contains("Id")) {
+        _persistID = map["Id"].toUuid();
+    }
+
+    if (map.contains("DataVersion")) {
+        _persistDataVersion = map["DataVersion"].toInt();
+    }
+
     // map will have a top-level list keyed as "Entities".  This will be extracted
     // and iterated over.  Each member of this list is converted to a QVariantMap, then
     // to a QScriptValue, and then to EntityItemProperties.  These properties are used

libraries/image/CMakeLists.txt

@@ -5,7 +5,8 @@ link_hifi_libraries(shared gpu)
 if (NOT ANDROID)
     add_dependency_external_projects(nvtt)
     find_package(NVTT REQUIRED)
+
     target_include_directories(${TARGET_NAME} PRIVATE ${NVTT_INCLUDE_DIRS})
     target_link_libraries(${TARGET_NAME} ${NVTT_LIBRARIES})
     add_paths_to_fixup_libs(${NVTT_DLL_PATH})
-endif()
+endif()

libraries/networking/src/LimitedNodeList.cpp

@@ -326,6 +326,7 @@ bool LimitedNodeList::packetSourceAndHashMatchAndTrackBandwidth(const udt::Packe
                     static QMultiMap<QUuid, PacketType> hashDebugSuppressMap;
 
                     if (!hashDebugSuppressMap.contains(sourceID, headerType)) {
+                        qCDebug(networking) << packetHeaderHash << expectedHash;
                         qCDebug(networking) << "Packet hash mismatch on" << headerType << "- Sender" << sourceID;
 
                         hashDebugSuppressMap.insert(sourceID, headerType);

libraries/networking/src/ThreadedAssignment.h

@@ -18,8 +18,6 @@
 
 #include "Assignment.h"
 
-using DownstreamNodeFoundCallback = std::function<void(Node& downstreamNode)>;
-
 class ThreadedAssignment : public Assignment {
     Q_OBJECT
 public:
@@ -47,10 +45,10 @@ protected:
     QTimer _domainServerTimer;
     QTimer _statsTimer;
     int _numQueuedCheckIns { 0 };
-    
+
 protected slots:
     void domainSettingsRequestFailed();
-    
+
 private slots:
     void checkInWithDomainServerOrExit();
 };

libraries/networking/src/udt/PacketHeaders.h

@@ -126,6 +126,11 @@ public:
         EntityScriptCallMethod,
         ChallengeOwnershipRequest,
         ChallengeOwnershipReply,
+
+        OctreeDataFileRequest,
+        OctreeDataFileReply,
+        OctreeDataPersist,
+
         NUM_PACKET_TYPE
     };
 
@@ -165,6 +170,8 @@ public:
             << PacketTypeEnum::Value::DomainConnectionDenied << PacketTypeEnum::Value::DomainServerPathQuery
             << PacketTypeEnum::Value::DomainServerPathResponse << PacketTypeEnum::Value::DomainServerAddedNode
             << PacketTypeEnum::Value::DomainServerConnectionToken << PacketTypeEnum::Value::DomainSettingsRequest
+            << PacketTypeEnum::Value::OctreeDataFileRequest << PacketTypeEnum::Value::OctreeDataFileReply
+            << PacketTypeEnum::Value::OctreeDataPersist << PacketTypeEnum::Value::OctreeFileReplacementFromUrl
             << PacketTypeEnum::Value::DomainSettings << PacketTypeEnum::Value::ICEServerPeerInformation
             << PacketTypeEnum::Value::ICEServerQuery << PacketTypeEnum::Value::ICEServerHeartbeat
             << PacketTypeEnum::Value::ICEServerHeartbeatACK << PacketTypeEnum::Value::ICEPing

libraries/networking/src/udt/Socket.cpp

@@ -328,7 +328,7 @@ void Socket::checkForReadyReadBackup() {
 void Socket::readPendingDatagrams() {
     int packetSizeWithHeader = -1;
 
-    while ((packetSizeWithHeader = _udpSocket.pendingDatagramSize()) != -1) {
+    while ((packetSizeWithHeader = _udpSocket.pendingDatagramSize()) > 0) {
 
         // we're reading a packet so re-start the readyRead backup timer
         _readyReadBackupTimer->start();
@@ -517,7 +517,7 @@ void Socket::handleSocketError(QAbstractSocket::SocketError socketError) {
     static QString repeatedMessage
         = LogHandler::getInstance().addRepeatedMessageRegex(SOCKET_REGEX);
 
-    qCDebug(networking) << "udt::Socket error - " << socketError;
+    qCDebug(networking) << "udt::Socket error - " << socketError << _udpSocket.errorString();
 }
 
 void Socket::handleStateChanged(QAbstractSocket::SocketState socketState) {

libraries/octree/CMakeLists.txt

@@ -1,3 +1,4 @@
 set(TARGET_NAME octree)
+include_directories(system /usr/local/Cellar/qt5/5.9.1/include)
 setup_hifi_library()
 link_hifi_libraries(shared networking)

libraries/octree/src/Octree.cpp

@@ -1757,6 +1757,19 @@ bool Octree::readJSONFromStream(uint64_t streamLength, QDataStream& inputStream,
     QVariant asVariant = asDocument.toVariant();
     QVariantMap asMap = asVariant.toMap();
     bool success = readFromMap(asMap);
+    /*
+    if (success) {
+        if (asMap.contains("DataVersion") && asMap.contains("Id")) {
+            bool versionOk;
+            auto dataVersion = asMap["DataVersion"].toLongLong(&versionOk);
+            if (versionOk) {
+                auto id = asMap["Id"].toUuid();
+                _persistDataVersion = dataVersion;
+                _persistID = id;
+            }
+        }
+    }
+    */
     delete[] rawData;
     return success;
 }
@@ -1778,11 +1791,9 @@ bool Octree::writeToFile(const char* fileName, const OctreeElementPointer& eleme
     return success;
 }
 
-bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& element, bool doGzip) {
+bool Octree::toJSON(QJsonDocument* doc, const OctreeElementPointer& element) {
     QVariantMap entityDescription;
 
-    qCDebug(octree, "Saving JSON SVO to file %s...", fileName);
-
     OctreeElementPointer top;
     if (element) {
         top = element;
@@ -1802,17 +1813,35 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
         return false;
     }
 
-    // convert the QVariantMap to JSON
+    *doc = QJsonDocument::fromVariant(entityDescription);
-    QByteArray jsonData = QJsonDocument::fromVariant(entityDescription).toJson();
+    return true;
-    QByteArray jsonDataForFile;
+}
 
-    if (doGzip) {
+bool Octree::toGzippedJSON(QByteArray* data, const OctreeElementPointer& element) {
-        if (!gzip(jsonData, jsonDataForFile, -1)) {
+    QJsonDocument doc;
-            qCritical("unable to gzip data while saving to json.");
+    if (!toJSON(&doc, element)) {
-            return false;
+        qCritical("Failed to convert Entities to QVariantMap while converting to json.");
-        }
+        return false;
+    }
+
+    QByteArray jsonData = doc.toJson();
+
+    if (!gzip(jsonData, *data, -1)) {
+        qCritical("Unable to gzip data while saving to json.");
+        return false;
     } else {
-        jsonDataForFile = jsonData;
+        qDebug() <<"Did gzip!";
+    }
+
+    return true;
+}
+
+bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& element, bool doGzip) {
+    qCDebug(octree, "Saving JSON SVO to file %s...", fileName);
+
+    QByteArray jsonDataForFile;
+    if (!toGzippedJSON(&jsonDataForFile)) {
+        return false;
     }
 
     QFile persistFile(fileName);
@@ -1823,6 +1852,7 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
         qCritical("Could not write to JSON description of entities.");
     }
 
+
     return success;
 }
 

libraries/octree/src/Octree.h

@@ -283,8 +283,10 @@ public:
     void loadOctreeFile(const char* fileName);
 
     // Octree exporters
-    bool writeToFile(const char* filename, const OctreeElementPointer& element = NULL, QString persistAsFileType = "json.gz");
+    bool toJSON(QJsonDocument* doc, const OctreeElementPointer& element = nullptr);
-    bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = NULL, bool doGzip = false);
+    bool toGzippedJSON(QByteArray* data, const OctreeElementPointer& element = nullptr);
+    bool writeToFile(const char* filename, const OctreeElementPointer& element = nullptr, QString persistAsFileType = "json.gz");
+    bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = nullptr, bool doGzip = false);
     virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues,
                             bool skipThoseWithBadParents) = 0;
 
@@ -326,6 +328,11 @@ public:
     virtual void dumpTree() { }
     virtual void pruneTree() { }
 
+    void setEntityVersionInfo(QUuid id, int64_t dataVersion) {
+        _persistID = id;
+        _persistDataVersion = dataVersion;
+    }
+
     virtual void resetEditStats() { }
     virtual quint64 getAverageDecodeTime() const { return 0; }
     virtual quint64 getAverageLookupTime() const { return 0;  }
@@ -359,6 +366,9 @@ protected:
 
     OctreeElementPointer _rootElement = nullptr;
 
+    QUuid _persistID { QUuid::createUuid() };
+    int _persistDataVersion { 0 };
+
     bool _isDirty;
     bool _shouldReaverage;
     bool _stopImport;

libraries/octree/src/OctreePersistThread.cpp

@@ -31,18 +31,19 @@
 
 #include "OctreeLogging.h"
 #include "OctreePersistThread.h"
+#include "OctreeUtils.h"
 
 const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
-const QString OctreePersistThread::REPLACEMENT_FILE_EXTENSION = ".replace";
 
 OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval,
                                          bool wantBackup, const QJsonObject& settings, bool debugTimestampNow,
-                                         QString persistAsFileType) :
+                                         QString persistAsFileType, const QByteArray& replacementData) :
     _tree(tree),
     _filename(filename),
     _backupDirectory(backupDirectory),
     _persistInterval(persistInterval),
     _initialLoadComplete(false),
+    _replacementData(replacementData),
     _loadTimeUSecs(0),
     _lastCheck(0),
     _wantBackup(wantBackup),
@@ -52,6 +53,7 @@ OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& file
 {
     parseSettings(settings);
 
+
     // in case the persist filename has an extension that doesn't match the file type
     QString sansExt = fileNameWithoutExtension(_filename, PERSIST_EXTENSIONS);
     _filename = sansExt + "." + _persistAsFileType;
@@ -132,51 +134,56 @@ quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& forma
     return mostRecentBackupInUsecs;
 }
 
-void OctreePersistThread::possiblyReplaceContent() {
+void OctreePersistThread::replaceData(QByteArray data) {
-    // before we load the normal file, check if there's a pending replacement file
+    backupCurrentFile();
-    auto replacementFileName = _filename + REPLACEMENT_FILE_EXTENSION;
 
-    QFile replacementFile { replacementFileName };
+    QFile currentFile { _filename };
-    if (replacementFile.exists()) {
+    if (currentFile.open(QIODevice::WriteOnly)) {
-        // we have a replacement file to process
+        currentFile.write(data);
-        qDebug() << "Replacing models file with" << replacementFileName;
+        qDebug() << "Wrote replacement data";
-
+    } else {
-        // first take the current models file and move it to a different filename, appended with the timestamp
+        qWarning() << "Failed to write replacement data";
-        QFile currentFile { _filename };
-        if (currentFile.exists()) {
-            static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss";
-            auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT);
-
-            if (currentFile.rename(backupFileName)) {
-                qDebug() << "Moved previous models file to" << backupFileName;
-            } else {
-                qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file";
-
-                if (!replacementFile.remove()) {
-                    qWarning() << "Could not remove replacement models file from" << replacementFileName
-                        << "- replacement will be re-attempted on next server restart";
-                    return;
-                }
-            }
-        }
-
-        // rename the replacement file to match what the persist thread is just about to read
-        if (!replacementFile.rename(_filename)) {
-            qWarning() << "Could not replace models file with" << replacementFileName << "- starting with empty models file";
-        }
     }
 }
 
+// Return true if current file is backed up successfully or doesn't exist.
+bool OctreePersistThread::backupCurrentFile() {
+    // first take the current models file and move it to a different filename, appended with the timestamp
+    QFile currentFile { _filename };
+    if (currentFile.exists()) {
+        static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss";
+        auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT);
+
+        if (currentFile.rename(backupFileName)) {
+            qDebug() << "Moved previous models file to" << backupFileName;
+            return true;
+        } else {
+            qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file";
+            return false;
+        }
+    }
+    return true;
+}
 
 bool OctreePersistThread::process() {
 
     if (!_initialLoadComplete) {
-        possiblyReplaceContent();
-
         quint64 loadStarted = usecTimestampNow();
         qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
 
-        bool persistantFileRead;
+        if (_replacementData.isNull()) {
+            sendLatestEntityDataToDS();
+        } else {
+            replaceData(_replacementData);
+            _replacementData.clear();
+        }
+
+        OctreeUtils::RawOctreeData data;
+        if (OctreeUtils::readOctreeDataInfoFromFile(_filename, &data)) {
+            _tree->setEntityVersionInfo(data.id, data.version);
+        }
+
+        bool persistentFileRead;
 
         _tree->withWriteLock([&] {
             PerformanceWarning warn(true, "Loading Octree File", true);
@@ -199,7 +206,7 @@ bool OctreePersistThread::process() {
                 qCDebug(octree) << "Loading Octree... lock file removed:" << lockFileName;
             }
 
-            persistantFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
+            persistentFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
             _tree->pruneTree();
         });
 
@@ -207,7 +214,7 @@ bool OctreePersistThread::process() {
         _loadTimeUSecs = loadDone - loadStarted;
 
         _tree->clearDirtyBit(); // the tree is clean since we just loaded it
-        qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistantFileRead));
+        qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
 
         unsigned long nodeCount = OctreeElement::getNodeCount();
         unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
@@ -272,7 +279,6 @@ bool OctreePersistThread::process() {
     return isStillRunning();  // keep running till they terminate us
 }
 
-
 void OctreePersistThread::aboutToFinish() {
     qCDebug(octree) << "Persist thread about to finish...";
     persist();
@@ -319,6 +325,23 @@ void OctreePersistThread::persist() {
             remove(qPrintable(lockFileName));
             qCDebug(octree) << "saving Octree lock file removed:" << lockFileName;
         }
+
+        sendLatestEntityDataToDS();
+    }
+}
+
+void OctreePersistThread::sendLatestEntityDataToDS() {
+    qDebug() << "Sending latest entity data to DS";
+    auto nodeList = DependencyManager::get<NodeList>();
+    const DomainHandler& domainHandler = nodeList->getDomainHandler();
+
+    QByteArray data;
+    if (_tree->toGzippedJSON(&data)) {
+        auto message = NLPacketList::create(PacketType::OctreeDataPersist, QByteArray(), true, true);
+        message->write(data);
+        nodeList->sendPacketList(std::move(message), domainHandler.getSockAddr());
+    } else {
+        qCWarning(octree) << "Failed to persist octree to DS";
     }
 }
 
@@ -453,7 +476,6 @@ void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) {
     }
 }
 
-
 void OctreePersistThread::backup() {
     qCDebug(octree) << "backup operation wantBackup:" << _wantBackup;
     if (_wantBackup) {

libraries/octree/src/OctreePersistThread.h

@@ -18,7 +18,6 @@
 #include <GenericThread.h>
 #include "Octree.h"
 
-/// Generalized threaded processor for handling received inbound packets.
 class OctreePersistThread : public GenericThread {
     Q_OBJECT
 public:
@@ -32,11 +31,11 @@ public:
     };
 
     static const int DEFAULT_PERSIST_INTERVAL;
-    static const QString REPLACEMENT_FILE_EXTENSION;
 
     OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory,
                         int persistInterval = DEFAULT_PERSIST_INTERVAL, bool wantBackup = false,
-                        const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false, QString persistAsFileType="json.gz");
+                        const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false,
+                        QString persistAsFileType="json.gz", const QByteArray& replacementData = QByteArray());
 
     bool isInitialLoadComplete() const { return _initialLoadComplete; }
     quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
@@ -61,7 +60,10 @@ protected:
     bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
     quint64 getMostRecentBackupTimeInUsecs(const QString& format);
     void parseSettings(const QJsonObject& settings);
-    void possiblyReplaceContent();
+    bool backupCurrentFile();
+
+    void replaceData(QByteArray data);
+    void sendLatestEntityDataToDS();
 
 private:
     OctreePointer _tree;
@@ -69,6 +71,7 @@ private:
     QString _backupDirectory;
     int _persistInterval;
     bool _initialLoadComplete;
+    QByteArray _replacementData;
 
     quint64 _loadTimeUSecs;
 

libraries/octree/src/OctreeUtils.cpp

@@ -16,7 +16,11 @@
 #include <glm/glm.hpp>
 
 #include <AABox.h>
+#include <Gzip.h>
 
+#include <QJsonObject>
+#include <QJsonDocument>
+#include <QFile>
 
 float calculateRenderAccuracy(const glm::vec3& position,
         const AABox& bounds,
@@ -75,3 +79,76 @@ float getOrthographicAccuracySize(float octreeSizeScale, int boundaryLevelAdjust
     const float smallestSize = 0.01f;
     return (smallestSize * MAX_VISIBILITY_DISTANCE_FOR_UNIT_ELEMENT) / boundaryDistanceForRenderLevel(boundaryLevelAdjust, octreeSizeScale);
 }
+
+bool OctreeUtils::readOctreeFile(QString path, QJsonDocument* doc) {
+    QFile file(path);
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCritical() << "Cannot open json file for reading: " << path;
+        return false;
+    }
+
+    QByteArray data = file.readAll();
+    QByteArray jsonData;
+
+    if (path.endsWith(".json.gz")) {
+        if (!gunzip(data, jsonData)) {
+            qCritical() << "json File not in gzip format: " << path;
+            return false;
+        }
+    } else {
+        jsonData = data;
+    }
+
+    *doc = QJsonDocument::fromJson(jsonData);
+    return !doc->isNull();
+}
+
+bool readOctreeDataInfoFromJSON(QJsonObject root, OctreeUtils::RawOctreeData* octreeData) {
+    if (root.contains("Id") && root.contains("DataVersion")) {
+        octreeData->id = root["Id"].toVariant().toUuid();
+        octreeData->version = root["DataVersion"].toInt();
+    }
+    if (root.contains("Entities")) {
+        octreeData->octreeData = root["Entities"].toArray();
+    }
+    return true;
+}
+
+bool OctreeUtils::readOctreeDataInfoFromData(QByteArray data, OctreeUtils::RawOctreeData* octreeData) {
+    QByteArray jsonData;
+    if (gunzip(data, jsonData)) {
+        data = jsonData;
+    }
+
+    auto doc = QJsonDocument::fromJson(data);
+    if (doc.isNull()) {
+        return false;
+    }
+
+    auto root = doc.object();
+    return readOctreeDataInfoFromJSON(root, octreeData);
+}
+
+bool OctreeUtils::readOctreeDataInfoFromFile(QString path, OctreeUtils::RawOctreeData* octreeData) {
+    QJsonDocument doc;
+    if (!OctreeUtils::readOctreeFile(path, &doc)) {
+        return false;
+    }
+
+    auto root = doc.object();
+    return readOctreeDataInfoFromJSON(root, octreeData);
+}
+
+QByteArray OctreeUtils::RawOctreeData::toByteArray() {
+    QJsonObject obj {
+        { "DataVersion", QJsonValue((qint64)version) },
+        { "Id", QJsonValue(id.toString()) },
+        { "Version", QJsonValue(5) },
+        { "Entities", octreeData }
+    };
+
+    QJsonDocument doc;
+    doc.setObject(obj);
+
+    return doc.toJson();
+}

libraries/octree/src/OctreeUtils.h

@@ -14,7 +14,30 @@
 
 #include "OctreeConstants.h"
 
+#include <QUuid>
+#include <QJsonArray>
+
 class AABox;
+class QJsonDocument;
+
+namespace OctreeUtils {
+
+// RawOctreeData is an intermediate format between JSON and a fully deserialized Octree.
+class RawOctreeData {
+public:
+    QUuid id { QUuid() };
+    int64_t version { -1 };
+
+    QJsonArray octreeData;
+
+    QByteArray toByteArray();
+};
+
+bool readOctreeFile(QString path, QJsonDocument* doc);
+bool readOctreeDataInfoFromData(QByteArray data, RawOctreeData* octreeData);
+bool readOctreeDataInfoFromFile(QString path, RawOctreeData* octreeData);
+
+}
 
 /// renderAccuracy represents a floating point "visibility" of an object based on it's view from the camera. At a simple
 /// level it returns 0.0f for things that are so small for the current settings that they could not be visible.

libraries/shared/src/SharedUtil.cpp

@@ -105,7 +105,7 @@ void usecTimestampNowForceClockSkew(qint64 clockSkew) {
     ::usecTimestampNowAdjust = clockSkew;
 }
 
-static qint64 TIME_REFERENCE = 0; // in usec
+static std::atomic<qint64> TIME_REFERENCE { 0 }; // in usec
 static std::once_flag usecTimestampNowIsInitialized;
 static QElapsedTimer timestampTimer;
 
@@ -771,6 +771,10 @@ QString formatUsecTime(double usecs) {
     return formatUsecTime<double>(usecs);
 }
 
+QString formatSecTime(qint64 secs) {
+    return formatUsecTime(secs * 1000000);
+}
+
 
 QString formatSecondsElapsed(float seconds) {
     QString result;

libraries/shared/src/SharedUtil.h

@@ -216,6 +216,7 @@ QString formatUsecTime(float usecs);
 QString formatUsecTime(double usecs);
 QString formatUsecTime(quint64 usecs);
 QString formatUsecTime(qint64 usecs);
+QString formatSecTime(qint64 secs);
 
 QString formatSecondsElapsed(float seconds);
 bool similarStrings(const QString& stringA, const QString& stringB);