Merge pull request #12432 from highfidelity/feat/content-settings

Add content archives to domain server web interface

.clang-format

@@ -1,12 +1,12 @@
 Language: Cpp
 Standard: Cpp11
-BasedOnStyle: "Chromium" 
+BasedOnStyle: "Chromium"
 ColumnLimit: 128
 IndentWidth: 4
 UseTab: Never
 
 BreakBeforeBraces: Custom
-BraceWrapping:   
+BraceWrapping:
   AfterEnum: true
   AfterClass: false
   AfterControlStatement: false
@@ -21,11 +21,11 @@ BraceWrapping:
 
 
 AccessModifierOffset: -4
-AllowShortFunctionsOnASingleLine: InlineOnly  
-BreakConstructorInitializers: BeforeColon 
-BreakConstructorInitializersBeforeComma: true
+AllowShortFunctionsOnASingleLine: InlineOnly
+BreakConstructorInitializers: AfterColon
+BreakConstructorInitializersBeforeComma: false
 IndentCaseLabels: true
-ReflowComments: false 
+ReflowComments: false
 Cpp11BracedListStyle: false
 ContinuationIndentWidth: 4
 ConstructorInitializerAllOnOneLineOrOnePerLine: false

assignment-client/src/Agent.cpp

@@ -340,7 +340,6 @@ void Agent::scriptRequestFinished() {
     request->deleteLater();
 }
 
-
 void Agent::executeScript() {
     _scriptEngine = scriptEngineFactory(ScriptEngine::AGENT_SCRIPT, _scriptContents, _payload);
 

assignment-client/src/assets/AssetServer.cpp

@@ -257,12 +257,10 @@ AssetServer::AssetServer(ReceivedMessage& message) :
     _transferTaskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT);
     _bakingTaskPool.setMaxThreadCount(1);
 
+    // Queue all requests until the Asset Server is fully setup
     auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
-    packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo");
-    packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload");
-    packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOperation");
-    
+    packetReceiver.registerListenerForTypes({ PacketType::AssetGet, PacketType::AssetGetInfo, PacketType::AssetUpload, PacketType::AssetMappingOperation }, this, "queueRequests");
+
 #ifdef Q_OS_WIN
     updateConsumedCores();
     QTimer* timer = new QTimer(this);
@@ -291,6 +289,7 @@ void AssetServer::aboutToFinish() {
         if (pendingRunnable) {
             it = _pendingBakes.erase(it);
         } else {
+            qDebug() << "Aborting bake for" << it.key();
             it.value()->abort();
             ++it;
         }
@@ -396,6 +395,7 @@ void AssetServer::completeSetup() {
 
         if (_fileMappings.size() > 0) {
             cleanupUnmappedFiles();
+            cleanupBakedFilesForDeletedAssets();
         }
 
         nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
@@ -417,10 +417,65 @@ void AssetServer::completeSetup() {
 
     PathUtils::removeTemporaryApplicationDirs();
     PathUtils::removeTemporaryApplicationDirs("Oven");
+
+    qCDebug(asset_server) << "Overriding temporary queuing packet handler.";
+    // We're fully setup, override the request queueing handler and replay all requests
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
+    packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo");
+    packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload");
+    packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOperation");
+
+    replayRequests();
+}
+
+void AssetServer::queueRequests(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode) {
+    qCDebug(asset_server) << "Queuing requests until fully setup";
+
+    QMutexLocker lock { &_queuedRequestsMutex };
+    _queuedRequests.push_back({ packet, senderNode });
+
+    // If we've stopped queueing but the callback was already in flight,
+    // then replay it immediately.
+    if (!_isQueueingRequests) {
+        lock.unlock();
+        replayRequests();
+    }
+}
+
+void AssetServer::replayRequests() {
+    RequestQueue queue;
+    {
+        QMutexLocker lock { &_queuedRequestsMutex };
+        std::swap(queue, _queuedRequests);
+        _isQueueingRequests = false;
+    }
+
+    qCDebug(asset_server) << "Replaying" << queue.size() << "requests.";
+
+    for (const auto& request : queue) {
+        switch (request.first->getType()) {
+            case PacketType::AssetGet:
+                handleAssetGet(request.first, request.second);
+                break;
+            case PacketType::AssetGetInfo:
+                handleAssetGetInfo(request.first, request.second);
+                break;
+            case PacketType::AssetUpload:
+                handleAssetUpload(request.first, request.second);
+                break;
+            case PacketType::AssetMappingOperation:
+                handleAssetMappingOperation(request.first, request.second);
+                break;
+            default:
+                qCWarning(asset_server) << "Unknown queued request type:" << request.first->getType();
+                break;
+        }
+    }
 }
 
 void AssetServer::cleanupUnmappedFiles() {
-    QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(AssetUtils::SHA256_HASH_HEX_LENGTH) + "}" };
+    QRegExp hashFileRegex { AssetUtils::ASSET_HASH_REGEX_STRING };
 
     auto files = _filesDirectory.entryInfoList(QDir::Files);
 
@@ -452,6 +507,38 @@ void AssetServer::cleanupUnmappedFiles() {
     }
 }
 
+void AssetServer::cleanupBakedFilesForDeletedAssets() {
+    qCInfo(asset_server) << "Performing baked asset cleanup for deleted assets";
+
+    std::set<AssetUtils::AssetHash> bakedHashes;
+
+    for (const auto& it : _fileMappings) {
+        // check if this is a mapping to baked content
+        if (it.first.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
+            // extract the hash from the baked mapping
+            AssetUtils::AssetHash hash = it.first.mid(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER.length(),
+                                                      AssetUtils::SHA256_HASH_HEX_LENGTH);
+
+            // add the hash to our set of hashes for which we have baked content
+            bakedHashes.insert(hash);
+        }
+    }
+
+    // enumerate the hashes for which we have baked content
+    for (const auto& hash : bakedHashes) {
+        // check if we have a mapping that points to this hash
+        auto matchingMapping = std::find_if(std::begin(_fileMappings), std::end(_fileMappings),
+                                            [&hash](const std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash> mappingPair) {
+            return mappingPair.second == hash;
+        });
+
+        if (matchingMapping == std::end(_fileMappings)) {
+            // we didn't find a mapping for this hash, remove any baked content we still have for it
+            removeBakedPathsForDeletedAsset(hash);
+        }
+    }
+}
+
 void AssetServer::handleAssetMappingOperation(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
     using AssetMappingOperationType = AssetUtils::AssetMappingOperationType;
 
@@ -1301,6 +1388,8 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina
 }
 
 void AssetServer::handleAbortedBake(QString originalAssetHash, QString assetPath) {
+    qDebug() << "Aborted bake:" << originalAssetHash;
+
     // for an aborted bake we don't do anything but remove the BakeAssetTask from our pending bakes
     _pendingBakes.remove(originalAssetHash);
 }

assignment-client/src/assets/AssetServer.h

@@ -44,6 +44,7 @@ public slots:
 private slots:
     void completeSetup();
 
+    void queueRequests(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetGetInfo(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetGet(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetUpload(QSharedPointer<ReceivedMessage> packetList, SharedNodePointer senderNode);
@@ -52,6 +53,8 @@ private slots:
     void sendStatsPacket() override;
 
 private:
+    void replayRequests();
+
     void handleGetMappingOperation(ReceivedMessage& message, NLPacketList& replyPacket);
     void handleGetAllMappingOperation(NLPacketList& replyPacket);
     void handleSetMappingOperation(ReceivedMessage& message, bool hasWriteAccess, NLPacketList& replyPacket);
@@ -80,6 +83,9 @@ private:
     /// Delete any unmapped files from the local asset directory
     void cleanupUnmappedFiles();
 
+    /// Delete any baked files for assets removed from the local asset directory
+    void cleanupBakedFilesForDeletedAssets();
+
     QString getPathToAssetHash(const AssetUtils::AssetHash& assetHash);
 
     std::pair<AssetUtils::BakingStatus, QString> getAssetStatus(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash);
@@ -115,6 +121,11 @@ private:
     QHash<AssetUtils::AssetHash, std::shared_ptr<BakeAssetTask>> _pendingBakes;
     QThreadPool _bakingTaskPool;
 
+    QMutex _queuedRequestsMutex;
+    bool _isQueueingRequests { true };
+    using RequestQueue = QVector<QPair<QSharedPointer<ReceivedMessage>, SharedNodePointer>>;
+    RequestQueue _queuedRequests;
+
     bool _wasColorTextureCompressionEnabled { false };
     bool _wasGrayscaleTextureCompressionEnabled { false  };
     bool _wasNormalTextureCompressionEnabled { false };

assignment-client/src/assets/BakeAssetTask.cpp

@@ -69,8 +69,10 @@ void BakeAssetTask::run() {
 
     _ovenProcess.reset(new QProcess());
 
+    QEventLoop loop;
+
     connect(_ovenProcess.get(), static_cast<void(QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished),
-            this, [this, tempOutputDir](int exitCode, QProcess::ExitStatus exitStatus) {
+            this, [&loop, this, tempOutputDir](int exitCode, QProcess::ExitStatus exitStatus) {
         qDebug() << "Baking process finished: " << exitCode << exitStatus;
 
         if (exitStatus == QProcess::CrashExit) {
@@ -108,6 +110,7 @@ void BakeAssetTask::run() {
             emit bakeFailed(_assetHash, _assetPath, errors);
         }
 
+        loop.quit();
     });
 
     qDebug() << "Starting oven for " << _assetPath;
@@ -117,11 +120,21 @@ void BakeAssetTask::run() {
         emit bakeFailed(_assetHash, _assetPath, errors);
         return;
     }
-    _ovenProcess->waitForFinished();
+
+    _isBaking = true;
+
+    loop.exec();
 }
 
 void BakeAssetTask::abort() {
-    if (!_wasAborted.exchange(true)) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "abort");
+        return;
+    }
+    qDebug() << "Aborting BakeAssetTask for" << _assetHash;
+    if (_ovenProcess->state() != QProcess::NotRunning) {
+        qDebug() << "Teminating oven process for" << _assetHash;
+        _wasAborted = true;
         _ovenProcess->terminate();
     }
 }

assignment-client/src/assets/BakeAssetTask.h

@@ -27,12 +27,14 @@ class BakeAssetTask : public QObject, public QRunnable {
 public:
     BakeAssetTask(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath);
 
+    // Thread-safe inspection methods
     bool isBaking() { return _isBaking.load(); }
+    bool wasAborted() const { return _wasAborted.load(); }
 
     void run() override;
 
+public slots:
     void abort();
-    bool wasAborted() const { return _wasAborted.load(); }
 
 signals:
     void bakeComplete(QString assetHash, QString assetPath, QString tempOutputDir, QVector<QString> outputFiles);

assignment-client/src/entities/EntityServer.cpp

@@ -116,7 +116,6 @@ void EntityServer::beforeRun() {
 void EntityServer::entityCreated(const EntityItem& newEntity, const SharedNodePointer& senderNode) {
 }
 
-
 // EntityServer will use the "special packets" to send list of recently deleted entities
 bool EntityServer::hasSpecialPacketsToSend(const SharedNodePointer& node) {
     bool shouldSendDeletedEntities = false;
@@ -277,7 +276,6 @@ int EntityServer::sendSpecialPackets(const SharedNodePointer& node, OctreeQueryN
     return totalBytes;
 }
 
-
 void EntityServer::pruneDeletedEntities() {
     EntityTreePointer tree = std::static_pointer_cast<EntityTree>(_tree);
     if (tree->hasAnyDeletedEntities()) {

assignment-client/src/entities/EntityServer.h

@@ -30,7 +30,6 @@ struct ViewerSendingStats {
 class SimpleEntitySimulation;
 using SimpleEntitySimulationPointer = std::shared_ptr<SimpleEntitySimulation>;
 
-
 class EntityServer : public OctreeServer, public NewlyCreatedEntityHook {
     Q_OBJECT
 public:
@@ -38,7 +37,7 @@ public:
     ~EntityServer();
 
     // Subclasses must implement these methods
-    virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override ;
+    virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override;
     virtual char getMyNodeType() const override { return NodeType::EntityServer; }
     virtual PacketType getMyQueryMessageType() const override { return PacketType::EntityQuery; }
     virtual const char* getMyServerName() const override { return MODEL_SERVER_NAME; }
@@ -82,12 +81,12 @@ private:
     QReadWriteLock _viewerSendingStatsLock;
     QMap<QUuid, QMap<QUuid, ViewerSendingStats>> _viewerSendingStats;
 
-    static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000; // 45m
-    static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000; // 1h
-    int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 45m
-    int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 1h
+    static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000;                    // 45m
+    static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000;                    // 1h
+    int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS;  // 45m
+    int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS;  // 1h
     QTimer _dynamicDomainVerificationTimer;
     void startDynamicDomainVerification();
 };
 
-#endif // hifi_EntityServer_h
+#endif  // hifi_EntityServer_h

assignment-client/src/octree/OctreeServer.cpp

@@ -33,6 +33,10 @@
 #include <PathUtils.h>
 #include <QtCore/QDir>
 
+#include <OctreeDataUtils.h>
+
+Q_LOGGING_CATEGORY(octree_server, "hifi.octree-server")
+
 int OctreeServer::_clientCount = 0;
 const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000;
 
@@ -84,6 +88,8 @@ int OctreeServer::_longProcessWait = 0;
 int OctreeServer::_shortProcessWait = 0;
 int OctreeServer::_noProcessWait = 0;
 
+static const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
+
 
 void OctreeServer::resetSendingStats() {
     _averageLoopTime.reset();
@@ -202,7 +208,6 @@ void OctreeServer::trackPacketSendingTime(float time) {
     }
 }
 
-
 void OctreeServer::trackProcessWaitTime(float time) {
     const float MAX_SHORT_TIME = 10.0f;
     const float MAX_LONG_TIME = 100.0f;
@@ -283,8 +288,6 @@ void OctreeServer::initHTTPManager(int port) {
     _httpManager = new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this, this);
 }
 
-const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
-
 bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) {
 
 #ifdef FORCE_CRASH
@@ -922,87 +925,6 @@ void OctreeServer::handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> me
     }
 }
 
-void OctreeServer::handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message) {
-    if (!_isFinished && !_isShuttingDown) {
-        // these messages are only allowed to come from the domain server, so make sure that is the case
-        auto nodeList = DependencyManager::get<NodeList>();
-        if (message->getSenderSockAddr() == nodeList->getDomainHandler().getSockAddr()) {
-            // it's far cleaner to load up the new content upon server startup
-            // so here we just store a special file at our persist path
-            // and then force a stop of the server so that it can pick it up when it relaunches
-            if (!_persistAbsoluteFilePath.isEmpty()) {
-                replaceContentFromMessageData(message->getMessage());
-            } else {
-                qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
-            }
-        } else {
-            qDebug() << "Received an octree file replacement that was not from our domain server - refusing to process";
-        }
-    }
-}
-
-// Message->getMessage() contains a QByteArray representation of the URL to download from
-void OctreeServer::handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message) {
-    qInfo() << "Received request to replace content from a url";
-    if (!_isFinished && !_isShuttingDown) {
-        // This call comes from Interface, so we skip our domain server check
-        // but confirm that we have permissions to replace content sets
-        if (DependencyManager::get<NodeList>()->getThisNodeCanReplaceContent()) {
-            if (!_persistAbsoluteFilePath.isEmpty()) {
-                // Convert message data into our URL
-                QString url(message->getMessage());
-                QUrl modelsURL = QUrl(url, QUrl::StrictMode);
-                QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
-                QNetworkRequest request(modelsURL);
-                QNetworkReply* reply = networkAccessManager.get(request);
-                connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
-                    QNetworkReply::NetworkError networkError = reply->error();
-                    if (networkError == QNetworkReply::NoError) {
-                        QByteArray contents = reply->readAll();
-                        replaceContentFromMessageData(contents);
-                    } else {
-                        qDebug() << "Error downloading JSON from specified file";
-                    }
-                });
-            } else {
-                qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
-            }
-        }
-    }
-}
-
-void OctreeServer::replaceContentFromMessageData(QByteArray content) {
-    //Assume we have compressed data
-    auto compressedOctree = content;
-    QByteArray jsonOctree;
-
-    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
-    if (!wasCompressed) {
-        // the source was not compressed, assume we were sent regular JSON data
-        jsonOctree = compressedOctree;
-    }
-    // check the JSON data to verify it is an object
-    if (QJsonDocument::fromJson(jsonOctree).isObject()) {
-        if (!wasCompressed) {
-            // source was not compressed, we compress it before we write it locally
-            gzip(jsonOctree, compressedOctree);
-        }
-        // write the compressed octree data to a special file
-        auto replacementFilePath = _persistAbsoluteFilePath.append(OctreePersistThread::REPLACEMENT_FILE_EXTENSION);
-        QFile replacementFile(replacementFilePath);
-        if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
-            // we've now written our replacement file, time to take the server down so it can
-            // process it when it comes back up
-            qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
-            setFinished(true);
-        } else {
-            qWarning() << "Could not write replacement octree data to file - refusing to process";
-        }
-    } else {
-        qDebug() << "Received replacement octree file that is invalid - refusing to process";
-    }
-}
-
 bool OctreeServer::readOptionBool(const QString& optionName, const QJsonObject& settingsSectionObject, bool& result) {
     result = false; // assume it doesn't exist
     bool optionAvailable = false;
@@ -1119,7 +1041,18 @@ void OctreeServer::readConfiguration() {
             _persistFilePath = getMyDefaultPersistFilename();
         }
 
+        QDir persistPath { _persistFilePath };
+
+        if (persistPath.isRelative()) {
+            // if the domain settings passed us a relative path, make an absolute path that is relative to the
+            // default data directory
+            _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
+        } else {
+            _persistAbsoluteFilePath = persistPath.absolutePath();
+        }
+
         qDebug() << "persistFilePath=" << _persistFilePath;
+        qDebug() << "persisAbsoluteFilePath=" << _persistAbsoluteFilePath;
 
         _persistAsFileType = "json.gz";
 
@@ -1200,20 +1133,94 @@ void OctreeServer::run() {
 }
 
 void OctreeServer::domainSettingsRequestComplete() {
+    if (_state != OctreeServerState::WaitingForDomainSettings) {
+        qCWarning(octree_server) << "Received domain settings after they have already been received";
+        return;
+    }
+
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
+    packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
+
+    packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
+
+    qDebug(octree_server) << "Received domain settings";
+
+    readConfiguration();
+
+    _state = OctreeServerState::WaitingForOctreeDataNegotation;
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    const DomainHandler& domainHandler = nodeList->getDomainHandler();
+
+    auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
+
+    OctreeUtils::RawOctreeData data;
+    qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
+    if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
+        qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
+        packet->writePrimitive(true);
+        auto id = data.id.toRfc4122();
+        packet->write(id);
+        packet->writePrimitive(data.version);
+    } else {
+        qCWarning(octree_server) << "No octree data found";
+        packet->writePrimitive(false);
+    }
+
+    qCDebug(octree_server) << "Sending request for octree data to DS";
+    nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
+} 
+
+void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
+    if (_state != OctreeServerState::WaitingForOctreeDataNegotation) {
+        qCWarning(octree_server) << "Server received ocree data file reply but is not currently negotiating.";
+        return;
+    }
+
+    bool includesNewData;
+    message->readPrimitive(&includesNewData);
+    QByteArray replaceData;
+    if (includesNewData) {
+        replaceData = message->readAll();
+        qDebug() << "Got reply to octree data file request, new data sent";
+    } else {
+        qDebug() << "Got reply to octree data file request, current entity data is sufficient";
+        
+        OctreeUtils::RawEntityData data;
+        qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
+        if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
+            if (data.id.isNull()) {
+                qCDebug(octree_server) << "Current octree data has a null id, updating";
+                data.resetIdAndVersion();
+
+                QFile file(_persistAbsoluteFilePath);
+                if (file.open(QIODevice::WriteOnly)) {
+                    auto entityData = data.toGzippedByteArray();
+                    file.write(entityData);
+                    file.close();
+                } else {
+                    qCDebug(octree_server) << "Failed to update octree data";
+                }
+            }
+        }
+    }
+
+    _state = OctreeServerState::Running;
+    beginRunning(replaceData);
+}
+
+void OctreeServer::beginRunning(QByteArray replaceData) {
+    if (_state != OctreeServerState::Running) {
+        qCWarning(octree_server) << "Server is not running";
+        return;
+    }
 
     auto nodeList = DependencyManager::get<NodeList>();
 
     // we need to ask the DS about agents so we can ping/reply with them
     nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
 
-    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
-    packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
-    packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacement");
-    packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURL");
-
-    readConfiguration();
-
     beforeRun(); // after payload has been processed
 
     connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
@@ -1233,17 +1240,6 @@ void OctreeServer::domainSettingsRequestComplete() {
 
     // if we want Persistence, set up the local file and persist thread
     if (_wantPersist) {
-        // If persist filename does not exist, let's see if there is one beside the application binary
-        // If there is, let's copy it over to our target persist directory
-        QDir persistPath { _persistFilePath };
-        _persistAbsoluteFilePath = persistPath.absolutePath();
-
-        if (persistPath.isRelative()) {
-            // if the domain settings passed us a relative path, make an absolute path that is relative to the
-            // default data directory
-            _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
-        }
-
         static const QString ENTITY_PERSIST_EXTENSION = ".json.gz";
 
         // force the persist file to end with .json.gz
@@ -1328,7 +1324,7 @@ void OctreeServer::domainSettingsRequestComplete() {
 
         // now set up PersistThread
         _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval,
-                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType);
+                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData);
         _persistThread->initialize(true);
     }
 

assignment-client/src/octree/OctreeServer.h

@@ -27,8 +27,18 @@
 #include "OctreeServerConsts.h"
 #include "OctreeInboundPacketProcessor.h"
 
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(octree_server)
+
 const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total
 
+enum class OctreeServerState {
+    WaitingForDomainSettings,
+    WaitingForOctreeDataNegotation,
+    Running
+};
+
 /// Handles assignments of type OctreeServer - sending octrees to various clients.
 class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler {
     Q_OBJECT
@@ -36,6 +46,8 @@ public:
     OctreeServer(ReceivedMessage& message);
     ~OctreeServer();
 
+    OctreeServerState _state { OctreeServerState::WaitingForDomainSettings };
+
     /// allows setting of run arguments
     void setArguments(int argc, char** argv);
 
@@ -137,8 +149,7 @@ private slots:
     void domainSettingsRequestComplete();
     void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message);
-    void handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
     void removeSendThread();
 
 protected:
@@ -159,12 +170,12 @@ protected:
     QString getFileLoadTime();
     QString getConfiguration();
     QString getStatusLink();
+
+    void beginRunning(QByteArray replaceData);
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
     virtual UniqueSendThread newSendThread(const SharedNodePointer& node);
 
-    void replaceContentFromMessageData(QByteArray content);
-
     int _argc;
     const char** _argv;
     char** _parsedArgV;

assignment-client/src/scripts/EntityScriptServer.cpp

@@ -178,7 +178,7 @@ void EntityScriptServer::updateEntityPPS() {
     int numRunningScripts = _entitiesScriptEngine->getNumRunningEntityScripts();
     int pps;
     if (std::numeric_limits<int>::max() / _entityPPSPerScript < numRunningScripts) {
-        qWarning() << QString("Integer multiplaction would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
+        qWarning() << QString("Integer multiplication would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
         pps = std::numeric_limits<int>::max();
         pps = std::min(_maxEntityPPS, pps);
     } else {

domain-server/CMakeLists.txt

@@ -24,7 +24,19 @@ symlink_or_copy_directory_beside_target(${_SHOULD_SYMLINK_RESOURCES} "${CMAKE_CU
 # link the shared hifi libraries
 include_hifi_library_headers(gpu)
 include_hifi_library_headers(graphics)
-link_hifi_libraries(embedded-webserver networking shared avatars)
+link_hifi_libraries(embedded-webserver networking shared avatars octree)
+
+target_zlib()
+
+add_dependency_external_projects(quazip)
+
+find_package(QuaZip REQUIRED)
+target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${QUAZIP_INCLUDE_DIRS})
+target_link_libraries(${TARGET_NAME} ${QUAZIP_LIBRARIES})
+
+if (WIN32)
+  add_paths_to_fixup_libs(${QUAZIP_DLL_PATH})
+endif ()
 
 # find OpenSSL
 find_package(OpenSSL REQUIRED)

domain-server/resources/describe-settings.json

@@ -1,5 +1,5 @@
 {
-  "version": 2.1,
+  "version": 2.2,
   "settings": [
     {
       "name": "metaverse",
@@ -306,7 +306,37 @@
             }
           ],
           "non-deletable-row-key": "permissions_id",
-          "non-deletable-row-values": [ "localhost", "anonymous", "logged-in" ]
+          "non-deletable-row-values": [ "localhost", "anonymous", "logged-in" ],
+          "default": [
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "anonymous"
+            },
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "friends"
+            },
+            {
+                "id_can_adjust_locks": true,
+                "id_can_connect": true,
+                "id_can_connect_past_max_capacity": true,
+                "id_can_kick": true,
+                "id_can_replace_content": true,
+                "id_can_rez": true,
+                "id_can_rez_certified": true,
+                "id_can_rez_tmp": true,
+                "id_can_rez_tmp_certified": true,
+                "id_can_write_to_asset_server": true,
+                "permissions_id": "localhost"
+            },
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "logged-in"
+            }
+          ]
         },
         {
           "name": "group_permissions",
@@ -1321,73 +1351,6 @@
           "default": "30000",
           "advanced": true
         },
-        {
-          "name": "backups",
-          "type": "table",
-          "label": "Backup Rules",
-          "help": "In this table you can define a set of rules for how frequently to backup copies of your entites content file.",
-          "numbered": false,
-          "can_add_new_rows": true,
-          "default": [
-            {
-              "Name": "Half Hourly Rolling",
-              "backupInterval": 1800,
-              "format": ".backup.halfhourly.%N",
-              "maxBackupVersions": 5
-            },
-            {
-              "Name": "Daily Rolling",
-              "backupInterval": 86400,
-              "format": ".backup.daily.%N",
-              "maxBackupVersions": 7
-            },
-            {
-              "Name": "Weekly Rolling",
-              "backupInterval": 604800,
-              "format": ".backup.weekly.%N",
-              "maxBackupVersions": 4
-            },
-            {
-              "Name": "Thirty Day Rolling",
-              "backupInterval": 2592000,
-              "format": ".backup.thirtyday.%N",
-              "maxBackupVersions": 12
-            }
-          ],
-          "columns": [
-            {
-              "name": "Name",
-              "label": "Name",
-              "can_set": true,
-              "placeholder": "Example",
-              "default": "Example"
-            },
-            {
-              "name": "format",
-              "label": "Rule Format",
-              "can_set": true,
-              "help": "Format used to create the extension for the backup of your persisted entities. Use a format with %N to get rolling. Or use date formatting like %Y-%m-%d.%H:%M:%S.%z",
-              "placeholder": ".backup.example.%N",
-              "default": ".backup.example.%N"
-            },
-            {
-              "name": "backupInterval",
-              "label": "Backup Interval in Seconds",
-              "help": "Interval between backup checks in seconds.",
-              "placeholder": 1800,
-              "default": 1800,
-              "can_set": true
-            },
-            {
-              "name": "maxBackupVersions",
-              "label": "Max Rolled Backup Versions",
-              "help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?",
-              "placeholder": 5,
-              "default": 5,
-              "can_set": true
-            }
-          ]
-        },
         {
           "name": "NoPersist",
           "type": "checkbox",
@@ -1649,6 +1612,67 @@
         }
       ]
     },
+    {
+      "name": "automatic_content_archives",
+      "label": "Automatic Content Archives",
+      "settings": [
+        {
+          "name": "backup_rules",
+          "type": "table",
+          "label": "Rolling Backup Rules",
+          "help": "Define how frequently to create automatic content archives",
+          "numbered": false,
+          "can_add_new_rows": true,
+          "default": [
+            {
+              "Name": "Half Hourly Rolling",
+              "backupInterval": 1800,
+              "maxBackupVersions": 5
+            },
+            {
+              "Name": "Daily Rolling",
+              "backupInterval": 86400,
+              "maxBackupVersions": 7
+            },
+            {
+              "Name": "Weekly Rolling",
+              "backupInterval": 604800,
+              "maxBackupVersions": 4
+            },
+            {
+              "Name": "Thirty Day Rolling",
+              "backupInterval": 2592000,
+              "maxBackupVersions": 12
+            }
+          ],
+          "columns": [
+            {
+              "name": "Name",
+              "label": "Name",
+              "can_set": true,
+              "placeholder": "Example",
+              "default": "Example"
+            },
+            {
+              "name": "backupInterval",
+              "label": "Backup Interval in Seconds",
+              "help": "Interval between backup checks in seconds.",
+              "placeholder": 1800,
+              "default": 1800,
+              "can_set": true
+            },
+            {
+              "name": "maxBackupVersions",
+              "label": "Max Rolled Backup Versions",
+              "help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?",
+              "placeholder": 5,
+              "default": 5,
+              "can_set": true
+            }
+          ]
+        }
+      ]
+    },
     {
       "name": "wizard",
       "label": "Setup Wizard",

domain-server/resources/web/base-settings-scripts.html

@@ -3,5 +3,4 @@
 <script src='/js/bootbox.min.js'></script>
 <script src='/js/form2js.min.js'></script>
 <script src='/js/bootstrap-switch.min.js'></script>
-<script src='/js/shared.js'></script>
 <script src='/js/base-settings.js'></script>

domain-server/resources/web/content/index.shtml

@@ -14,6 +14,8 @@
 
 <!--#include virtual="base-settings-scripts.html"-->
 
+<script src="js/moment-locale.min.js"></script>
+<script src="js/bootstrap-sortable.min.js"></script>
 <script src="js/content.js"></script>
 
 <!--#include virtual="page-end.html"-->

domain-server/resources/web/content/js/content.js

@@ -1,37 +1,437 @@
 $(document).ready(function(){
 
-  Settings.afterReloadActions = function() {};
+  var RESTORE_SETTINGS_UPLOAD_ID = 'restore-settings-button';
+  var RESTORE_SETTINGS_FILE_ID = 'restore-settings-file';
+  var UPLOAD_CONTENT_ALLOWED_DIV_ID = 'upload-content-allowed';
+  var UPLOAD_CONTENT_RECOVERING_DIV_ID = 'upload-content-recovering';
 
-  var frm = $('#upload-form');
-  frm.submit(function (ev) {
-    $.ajax({
-      type: frm.attr('method'),
-      url: frm.attr('action'),
-      data: new FormData($(this)[0]),
-      cache: false,
-      contentType: false,
-      processData: false,
-      success: function (data) {
-        swal({
-          title: 'Uploaded',
-          type: 'success',
-          text: 'Your Entity Server is restarting to replace its local content with the uploaded file.',
-          confirmButtonText: 'OK'
-        })
-      },
-      error: function (data) {
-        swal({
-          title: '',
-          type: 'error',
-          text: 'Your entities file could not be transferred to the Entity Server.</br>Verify that the file is a <i>.json</i> or <i>.json.gz</i> entities file and try again.',
-          html: true,
-          confirmButtonText: 'OK',
+  var isRestoring = false;
+
+  function progressBarHTML(extraClass, label) {
+    var html = "<div class='progress'>";
+    html += "<div class='" + extraClass + " progress-bar progress-bar-success progress-bar-striped active' role='progressbar' aria-valuemin='0' aria-valuemax='100'>";
+    html += label + "<span class='sr-only'></span></div></div>";
+    return html;
+  }
+
+  function setupBackupUpload() {
+    // construct the HTML needed for the settings backup panel
+    var html = "<div class='form-group'><div id='" + UPLOAD_CONTENT_ALLOWED_DIV_ID + "'>";
+
+    html += "<span class='help-block'>Upload a content archive (.zip) or entity file (.json, .json.gz) to replace the content of this domain.";
+    html += "<br/>Note: Your domain content will be replaced by the content you upload, but the existing backup files of your domain's content will not immediately be changed.</span>";
+
+    html += "<input id='restore-settings-file' name='restore-settings' type='file'>";
+    html += "<button type='button' id='" + RESTORE_SETTINGS_UPLOAD_ID + "' disabled='true' class='btn btn-primary'>Upload Content</button>";
+
+    html += "</div><div id='" + UPLOAD_CONTENT_RECOVERING_DIV_ID + "'>";
+    html += "<span class='help-block'>Restore in progress</span>";
+    html += progressBarHTML('recovery', 'Restoring');
+    html += "</div></div>";
+
+    $('#' + Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID + ' .panel-body').html(html);
+  }
+
+  // handle content archive or entity file upload
+
+  // when the selected file is changed, enable the button if there's a selected file
+  $('body').on('change', '#' + RESTORE_SETTINGS_FILE_ID, function() {
+    $('#' + RESTORE_SETTINGS_UPLOAD_ID).attr('disabled', $(this).val().length == 0);
+  });
+
+  // when the upload button is clicked, send the file to the DS
+  // and reload the page if restore was successful or
+  // show an error if not
+  $('body').on('click', '#' + RESTORE_SETTINGS_UPLOAD_ID, function(e){
+    e.preventDefault();
+
+    swalAreYouSure(
+      "Your domain content will be replaced by the uploaded content archive or entity file",
+      "Restore content",
+      function() {
+        var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
+
+        var fileFormData = new FormData();
+        fileFormData.append('restore-file', files[0]);
+
+        showSpinnerAlert("Uploading content to restore");
+
+        $.ajax({
+          url: '/content/upload',
+          type: 'POST',
+          cache: false,
+          processData: false,
+          contentType: false,
+          data: fileFormData
+        }).done(function(data, textStatus, jqXHR) {
+          isRestoring = true;
+
+          // immediately reload backup information since one should be restoring now
+          reloadBackupInformation();
+
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain content.\n"
+            + "Please ensure that the content archive or entity file is valid and try again."
+          );
         });
       }
-    });
-
-    ev.preventDefault();
-
-    showSpinnerAlert("Uploading Entities File");
+    );
   });
+
+  var GENERATE_ARCHIVE_BUTTON_ID = 'generate-archive-button';
+  var CONTENT_ARCHIVES_NORMAL_ID = 'content-archives-success';
+  var CONTENT_ARCHIVES_ERROR_ID = 'content-archives-error';
+  var AUTOMATIC_ARCHIVES_TABLE_ID = 'automatic-archives-table';
+  var AUTOMATIC_ARCHIVES_TBODY_ID = 'automatic-archives-tbody';
+  var MANUAL_ARCHIVES_TABLE_ID = 'manual-archives-table';
+  var MANUAL_ARCHIVES_TBODY_ID = 'manual-archives-tbody';
+  var AUTO_ARCHIVES_SETTINGS_LINK_ID = 'auto-archives-settings-link';
+  var ACTION_MENU_CLASS = 'action-menu';
+
+  var automaticBackups = [];
+  var manualBackups = [];
+
+  function setupContentArchives() {
+    // construct the HTML needed for the content archives panel
+    var html = "<div id='" + CONTENT_ARCHIVES_NORMAL_ID + "'><div class='form-group'>";
+    html += "<label class='control-label'>Automatic Content Archives</label>";
+    html += "<span class='help-block'>Your domain server makes regular archives of the content in your domain. In the list below, you can see and download all of your backups of domain content and content settings."
+    html += "<a href='/settings/#automatic_content_archives' id='" + AUTO_ARCHIVES_SETTINGS_LINK_ID + "'>Click here to manage automatic content archive intervals.</a></span>";
+    html += "</div>";
+    html += "<table class='table sortable' id='" + AUTOMATIC_ARCHIVES_TABLE_ID + "'>";
+
+    var backups_table_head = "<thead><tr class='gray-tr'><th>Archive Name</th><th data-defaultsort='desc'>Archive Date</th>"
+      + "<th data-defaultsort='disabled'></th><th class='" + ACTION_MENU_CLASS + "' data-defaultsort='disabled'>Actions</th>"
+      + "</tr></thead>";
+
+    html += backups_table_head;
+    html += "<tbody id='" + AUTOMATIC_ARCHIVES_TBODY_ID + "'></tbody></table>";
+    html += "<div class='form-group'>";
+    html += "<label class='control-label'>Manual Content Archives</label>";
+    html += "<span class='help-block'>You can generate and download an archive of your domain content right now. You can also download, delete and restore any archive listed.</span>";
+    html += "<button type='button' id='" + GENERATE_ARCHIVE_BUTTON_ID + "' class='btn btn-primary'>Generate New Archive</button>";
+    html += "</div>";
+    html += "<table class='table sortable' id='" + MANUAL_ARCHIVES_TABLE_ID + "'>";
+    html += backups_table_head;
+    html += "<tbody id='" + MANUAL_ARCHIVES_TBODY_ID + "'></tbody></table></div>";
+
+    html += "<div class='form-group' id='" + CONTENT_ARCHIVES_ERROR_ID + "' style='display:none;'>"
+      + "<span class='help-block'>There was a problem loading your list of automatic and manual content archives. "
+      + "Please reload the page to try again.</span></div>";
+
+    // put the base HTML in the content archives panel
+    $('#' + Settings.CONTENT_ARCHIVES_PANEL_ID + ' .panel-body').html(html);
+  }
+
+  var BACKUP_RESTORE_LINK_CLASS = 'restore-backup';
+  var BACKUP_DOWNLOAD_LINK_CLASS = 'download-backup';
+  var BACKUP_DELETE_LINK_CLASS = 'delete-backup';
+  var ACTIVE_BACKUP_ROW_CLASS = 'active-backup';
+  var CORRUPTED_ROW_CLASS = 'danger';
+
+  function reloadBackupInformation() {
+    // make a GET request to get backup information to populate the table
+    $.ajax({
+      url: '/api/backups',
+      cache: false
+    }).done(function(data) {
+
+      // split the returned data into manual and automatic manual backups
+      var splitBackups = _.partition(data.backups, function(value, index) {
+        return value.isManualBackup;
+      });
+
+      if (isRestoring && !data.status.isRecovering) {
+        // we were recovering and we finished - the DS is going to restart so show the restart modal
+        showRestartModal();
+        return;
+      }
+
+      isRestoring = data.status.isRecovering;
+
+      manualBackups = splitBackups[0];
+      automaticBackups = splitBackups[1];
+
+      // populate the backups tables with the backups
+      function createBackupTableRow(backup) {
+        return "<tr data-backup-id='" + backup.id + "' data-backup-name='" + backup.name + "'>"
+          + "<td data-value='" + backup.name.toLowerCase() + "'>" + backup.name + "</td><td data-value='" + backup.createdAtMillis + "'>"
+          + moment(backup.createdAtMillis).format('lll')
+          + "</td><td class='backup-status'></td><td class='" + ACTION_MENU_CLASS + "'>"
+          + "<div class='dropdown'><div class='dropdown-toggle' data-toggle='dropdown' aria-expanded='false'><span class='glyphicon glyphicon-option-vertical'></span></div>"
+          + "<ul class='dropdown-menu dropdown-menu-right'>"
+          + "<li><a class='" + BACKUP_RESTORE_LINK_CLASS + "' href='#'>Restore from here</a></li><li class='divider'></li>"
+          + "<li><a class='" + BACKUP_DOWNLOAD_LINK_CLASS + "' href='/api/backups/" + backup.id + "'>Download</a></li><li class='divider'></li>"
+          + "<li><a class='" + BACKUP_DELETE_LINK_CLASS + "' href='#' target='_blank'>Delete</a></li></ul></div></td>";
+      }
+
+      function updateProgressBars($progressBar, value) {
+        $progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%');
+        $progressBar.find('.sr-only').html(value + "% Complete");
+      }
+
+      // before we add any new rows and update existing ones
+      // remove our flag for active rows
+      $('.' + ACTIVE_BACKUP_ROW_CLASS).removeClass(ACTIVE_BACKUP_ROW_CLASS);
+
+      function updateOrAddTableRow(backup, tableBodyID) {
+        // check for a backup with this ID
+        var $backupRow = $("tr[data-backup-id='" + backup.id + "']");
+
+        if ($backupRow.length == 0) {
+          // create a new row and then add it to the table
+          $backupRow = $(createBackupTableRow(backup));
+          $('#' + tableBodyID).append($backupRow);
+        }
+
+        // update the row status column depending on if it is available or recovering
+        if (!backup.isAvailable) {
+          // add a progress bar to the status row for availability
+          $backupRow.find('td.backup-status').html(progressBarHTML('availability', 'Archiving'));
+
+          // set the value of the progress bar based on availability progress
+          updateProgressBars($backupRow.find('.progress-bar'), backup.availabilityProgress * 100);
+        } else if (backup.id == data.status.recoveringBackupId) {
+          // add a progress bar to the status row for recovery
+          $backupRow.find('td.backup-status').html(progressBarHTML('recovery', 'Restoring'));
+        } else if (backup.isCorrupted) {
+          // add text for corrupted status to row
+          $backupRow.find('td.backup-status').html('<span>Corrupted</span>');
+        } else {
+          // no special status for this row, use an empty status column
+          $backupRow.find('td.backup-status').html('');
+        }
+
+        // color the row red if it is corrupted
+        $backupRow.toggleClass(CORRUPTED_ROW_CLASS, backup.isCorrupted);
+
+        // disable restore if the backup is corrupted
+        $backupRow.find('a.' + BACKUP_RESTORE_LINK_CLASS).parent().toggleClass('disabled', backup.isCorrupted);
+
+        // toggle the dropdown menu depending on if the row is available
+        $backupRow.find('td.' + ACTION_MENU_CLASS + ' .dropdown').toggle(backup.isAvailable);
+
+        $backupRow.addClass(ACTIVE_BACKUP_ROW_CLASS);
+      }
+
+      if (automaticBackups.length > 0) {
+        for (var backupIndex in automaticBackups) {
+          updateOrAddTableRow(automaticBackups[backupIndex], AUTOMATIC_ARCHIVES_TBODY_ID);
+        }
+      }
+
+      if (manualBackups.length > 0) {
+        for (var backupIndex in manualBackups) {
+          updateOrAddTableRow(manualBackups[backupIndex], MANUAL_ARCHIVES_TBODY_ID);
+        }
+      }
+
+      // at this point, any rows that no longer have the ACTIVE_BACKUP_ROW_CLASS
+      // are deleted backups, so we remove them from the table
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID + ' tbody tr:not(.' + ACTIVE_BACKUP_ROW_CLASS + ')').remove();
+
+      // check if the restore action on all rows should be enabled or disabled
+      $('tr:not(.' + CORRUPTED_ROW_CLASS + ') .' + BACKUP_RESTORE_LINK_CLASS).parent().toggleClass('disabled', data.status.isRecovering);
+
+      // hide or show the manual content upload file and button depending on our recovering status
+      $('#' + UPLOAD_CONTENT_ALLOWED_DIV_ID).toggle(!data.status.isRecovering);
+      $('#' + UPLOAD_CONTENT_RECOVERING_DIV_ID).toggle(data.status.isRecovering);
+
+      // update the progress bars for current restore status
+      if (data.status.isRecovering) {
+        updateProgressBars($('.recovery.progress-bar'), data.status.recoveryProgress * 100);
+      }
+
+      // tell bootstrap sortable to update for the new rows
+      $.bootstrapSortable({ applyLast: true });
+
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID).toggle(true);
+      $('#' + CONTENT_ARCHIVES_ERROR_ID).toggle(false);
+
+    }).fail(function(){
+      // we've hit the very rare case where we couldn't load the list of backups from the domain server
+
+      // set our backups to empty
+      automaticBackups = [];
+      manualBackups = [];
+
+      // replace the content archives panel with a simple error message
+      // stating that the user should reload the page
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID).toggle(false);
+      $('#' + CONTENT_ARCHIVES_ERROR_ID).toggle(true);
+
+    }).always(function(){
+      // toggle showing or hiding the tables depending on if they have entries
+      $('#' + AUTOMATIC_ARCHIVES_TABLE_ID).toggle(automaticBackups.length > 0);
+      $('#' + MANUAL_ARCHIVES_TABLE_ID).toggle(manualBackups.length > 0);
+    });
+  }
+
+  // handle click in table to restore a given content backup
+  $('body').on('click', '.' + BACKUP_RESTORE_LINK_CLASS, function(e) {
+    // stop the default behaviour
+    e.preventDefault();
+
+    // if this is a disabled link, don't proceed with the restore
+    if ($(this).parent().hasClass('disabled')) {
+      return false;
+    }
+
+    // grab the name of this backup so we can show it in alerts
+    var backupName = $(this).closest('tr').attr('data-backup-name');
+
+    // grab the ID of this backup in case we need to send a POST
+    var backupID = $(this).closest('tr').attr('data-backup-id');
+
+    // make sure the user knows what is about to happen
+    swalAreYouSure(
+      "Your domain content will be replaced by the content archive " + backupName,
+      "Restore content",
+      function() {
+        // show a spinner while we send off our request
+        showSpinnerAlert("Starting restore of " +  backupName);
+
+        // setup an AJAX POST to request content restore
+        $.post('/api/backups/recover/' + backupID).done(function(data, textStatus, jqXHR) {
+          isRestoring = true;
+
+          // immediately reload our backup information since one should be restoring now
+          reloadBackupInformation();
+
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain content.\n"
+            + "If the problem persists, the content archive may be corrupted."
+          );
+        });
+      }
+    )
+  });
+
+  // handle click in table to delete a given content backup
+  $('body').on('click', '.' + BACKUP_DELETE_LINK_CLASS, function(e){
+    // stop the default behaviour
+    e.preventDefault();
+
+    // grab the name of this backup so we can show it in alerts
+    var backupName = $(this).closest('tr').attr('data-backup-name');
+
+    // grab the ID of this backup in case we need to send the DELETE request
+    var backupID = $(this).closest('tr').attr('data-backup-id');
+
+    // make sure the user knows what is about to happen
+    swalAreYouSure(
+      "The content archive " + backupName + " will be deleted and will no longer be available for restore or download from this page.",
+      "Delete content archive",
+      function() {
+        // show a spinner while we send off our request
+        showSpinnerAlert("Deleting content archive " +  backupName);
+
+        // setup an AJAX DELETE to request content archive delete
+        $.ajax({
+          url: '/api/backups/' + backupID,
+          type: 'DELETE'
+        }).done(function(data, textStatus, jqXHR) {
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was an unexpected error deleting the content archive"
+          );
+        }).always(function(){
+          // reload the list of content archives in case we deleted a backup
+          // or it's no longer an available backup for some other reason
+          reloadBackupInformation();
+        });
+      }
+    )
+  });
+
+  // handle click on automatic content archive settings link
+  $('body').on('click', '#' + AUTO_ARCHIVES_SETTINGS_LINK_ID, function(e) {
+    if (Settings.pendingChanges > 0) {
+      // don't follow the link right away, make sure the user knows they are about to leave
+      // the page and lose changes
+      e.preventDefault();
+
+      var settingsLink = $(this).attr('href');
+
+      swalAreYouSure(
+        "You have pending changes to content settings that have not been saved. They will be lost if you leave the page to manage automatic content archive intervals.",
+        "Proceed without Saving",
+        function() {
+          // user wants to drop their changes, switch pages
+          window.location =  settingsLink;
+        }
+      );
+    }
+  });
+
+  // handle click on manual archive creation button
+  $('body').on('click', '#' + GENERATE_ARCHIVE_BUTTON_ID, function(e) {
+    e.preventDefault();
+
+    // show a sweet alert to ask the user to provide a name for their content archive
+    swal({
+      title: "Generate a content archive",
+      type: "input",
+      text: "This will capture the state of all the content in your domain right now, which you can save as a backup and restore from later.",
+      confirmButtonText: "Generate Archive",
+      showCancelButton: true,
+      closeOnConfirm: false,
+      inputPlaceholder: 'Archive Name'
+    }, function(inputValue){
+      if (inputValue === false) {
+        return false;
+      }
+
+      if (inputValue === "") {
+        swal.showInputError("Please give the content archive a name.")
+        return false;
+      }
+
+      var MANUAL_ARCHIVE_NAME_REGEX = /^[a-zA-Z0-9\-_ ]+$/;
+      if (!MANUAL_ARCHIVE_NAME_REGEX.test(inputValue)) {
+        swal.showInputError("Valid characters include A-z, 0-9, ' ', '_', and '-'.");
+        return false;
+      }
+
+      // post the provided archive name to ask the server to kick off a manual backup
+      $.ajax({
+        type: 'POST',
+        url: '/api/backups',
+        data: {
+          'name': inputValue
+        }
+      }).done(function(data) {
+        // since we successfully setup a new content archive, reload the table of archives
+        // which should show that this archive is pending creation
+        swal.close();
+        reloadBackupInformation();
+      }).fail(function(jqXHR, textStatus, errorThrown) {
+        showErrorMessage(
+          "Error",
+          "There was an unexpected error creating the manual content archive"
+        )
+      });
+    });
+  });
+
+  Settings.extraGroupsAtIndex = Settings.extraContentGroupsAtIndex;
+
+  Settings.afterReloadActions = function() {
+    setupBackupUpload();
+    setupContentArchives();
+
+    // load the latest backups immediately
+    reloadBackupInformation();
+
+    // setup a timer to reload them every 5 seconds
+    setInterval(reloadBackupInformation, 5000);
+  };
 });

domain-server/resources/web/css/bootstrap-sortable.css

@@ -0,0 +1,110 @@
+/**
+ * adding sorting ability to HTML tables with Bootstrap styling
+ * @summary HTML tables sorting ability
+ * @version 2.0.0
+ * @requires tinysort, moment.js, jQuery
+ * @license MIT
+ * @author Matus Brlit (drvic10k)
+ * @copyright Matus Brlit (drvic10k), bootstrap-sortable contributors
+ */
+
+table.sortable span.sign {
+    display: block;
+    position: absolute;
+    top: 50%;
+    right: 5px;
+    font-size: 12px;
+    margin-top: -10px;
+    color: #bfbfc1;
+}
+
+table.sortable th:after {
+    display: block;
+    position: absolute;
+    top: 50%;
+    right: 5px;
+    font-size: 12px;
+    margin-top: -10px;
+    color: #bfbfc1;
+}
+
+table.sortable th.arrow:after {
+    content: '';
+}
+
+table.sortable span.arrow, span.reversed, th.arrow.down:after, th.reversedarrow.down:after, th.arrow.up:after, th.reversedarrow.up:after {
+    border-style: solid;
+    border-width: 5px;
+    font-size: 0;
+    border-color: #ccc transparent transparent transparent;
+    line-height: 0;
+    height: 0;
+    width: 0;
+    margin-top: -2px;
+}
+
+    table.sortable span.arrow.up, th.arrow.up:after {
+        border-color: transparent transparent #ccc transparent;
+        margin-top: -7px;
+    }
+
+table.sortable span.reversed, th.reversedarrow.down:after {
+    border-color: transparent transparent #ccc transparent;
+    margin-top: -7px;
+}
+
+    table.sortable span.reversed.up, th.reversedarrow.up:after {
+        border-color: #ccc transparent transparent transparent;
+        margin-top: -2px;
+    }
+
+table.sortable span.az:before, th.az.down:after {
+    content: "a .. z";
+}
+
+table.sortable span.az.up:before, th.az.up:after {
+    content: "z .. a";
+}
+
+table.sortable th.az.nosort:after, th.AZ.nosort:after, th._19.nosort:after, th.month.nosort:after {
+    content: "..";
+}
+
+table.sortable span.AZ:before, th.AZ.down:after {
+    content: "A .. Z";
+}
+
+table.sortable span.AZ.up:before, th.AZ.up:after {
+    content: "Z .. A";
+}
+
+table.sortable span._19:before, th._19.down:after {
+    content: "1 .. 9";
+}
+
+table.sortable span._19.up:before, th._19.up:after {
+    content: "9 .. 1";
+}
+
+table.sortable span.month:before, th.month.down:after {
+    content: "jan .. dec";
+}
+
+table.sortable span.month.up:before, th.month.up:after {
+    content: "dec .. jan";
+}
+
+table.sortable>thead th:not([data-defaultsort=disabled]) {
+    cursor: pointer;
+    position: relative;
+    top: 0;
+    left: 0;
+}
+
+table.sortable>thead th:hover:not([data-defaultsort=disabled]) {
+    background: #efefef;
+}
+
+table.sortable>thead th div.mozilla {
+    position: relative;
+}

domain-server/resources/web/css/style.css

@@ -355,21 +355,31 @@ table .headers + .headers td {
   }
 }
 
-ul.nav li.dropdown ul.dropdown-menu {
+ul.dropdown-menu {
   padding: 0px 0px;
 }
 
-ul.nav li.dropdown li a {
+ul.dropdown-menu li a {
   padding-top: 7px;
   padding-bottom: 7px;
 }
 
-ul.nav li.dropdown li a:hover {
+ul.dropdown-menu li a:hover {
   color: white;
   background-color: #337ab7;
 }
 
-ul.nav li.dropdown ul.dropdown-menu .divider {
+table ul.dropdown-menu li:first-child a:hover {
+  border-top-left-radius: 4px;
+  border-top-right-radius: 4px;
+}
+
+ul.dropdown-menu li:last-child a:hover {
+  border-bottom-left-radius: 4px;
+  border-bottom-right-radius: 4px;
+}
+
+ul.dropdown-menu .divider {
   margin: 0px 0;
 }
 
@@ -434,3 +444,42 @@ ul.nav li.dropdown ul.dropdown-menu .divider {
 .save-button-text {
   pointer-events: none;
 }
+
+#content_archives .panel-body {
+  padding: 0;
+}
+
+#content_archives .panel-body .form-group {
+  padding: 15px;
+}
+
+#content_archives .panel-body th, #content_archives .panel-body td {
+  padding: 8px 15px;
+}
+
+#content_archives table {
+  border-top: 1px solid #ddd;
+  border-bottom: 1px solid #ddd;
+}
+
+tr.gray-tr {
+  background-color: #f5f5f5;
+}
+
+table .action-menu {
+  text-align: right;
+  width: 90px;
+}
+
+.dropdown-toggle span.glyphicon-option-vertical {
+  font-size: 110%;
+  cursor: pointer;
+  border-radius: 50%;
+  background-color: #F5F5F5;
+  padding: 4px 4px 4px 6px;
+}
+
+.dropdown.open span.glyphicon-option-vertical {
+  background-color: #337AB7;
+  color: white;
+}

domain-server/resources/web/footer.html

@@ -1,4 +1,5 @@
 </div>
 <script src='/js/jquery-2.1.4.min.js'></script>
 <script src='/js/bootstrap.min.js'></script>
+<script src='/js/shared.js'></script>
 <script src='/js/domain-server.js'></script>

domain-server/resources/web/header.html

@@ -9,6 +9,7 @@
     <link href="/css/style.css" rel="stylesheet" media="screen">
     <link href="/css/sweetalert.css" rel="stylesheet" media="screen">
     <link href="/css/bootstrap-switch.min.css" rel="stylesheet" media="screen">
+    <link href="/css/bootstrap-sortable.css" rel="stylesheet" media="screen">
 
     <script src='/js/sweetalert.min.js'></script>
   </head>

domain-server/resources/web/js/base-settings.js

@@ -106,8 +106,12 @@ function reloadSettings(callback) {
   $.getJSON(Settings.endpoint, function(data){
     _.extend(data, viewHelpers);
 
-    for (var spliceIndex in Settings.extraGroups) {
-      data.descriptions.splice(spliceIndex, 0, Settings.extraGroups[spliceIndex]);
+    for (var spliceIndex in Settings.extraGroupsAtIndex) {
+      data.descriptions.splice(spliceIndex, 0, Settings.extraGroupsAtIndex[spliceIndex]);
+    }
+
+    for (var endGroupIndex in Settings.extraGroupsAtEnd) {
+      data.descriptions.push(Settings.extraGroupsAtEnd[endGroupIndex]);
     }
 
     $('#panels').html(Settings.panelsTemplate(data));
@@ -122,6 +126,8 @@ function reloadSettings(callback) {
 
     $('[data-toggle="tooltip"]').tooltip();
 
+    Settings.pendingChanges = 0;
+
     // call the callback now that settings are loaded
     callback(true);
   }).fail(function() {
@@ -257,7 +263,7 @@ $(document).ready(function(){
     }
   });
 
-  $('#' + Settings.FORM_ID).on('change keyup paste', '.' + Settings.TRIGGER_CHANGE_CLASS , function(e){
+  $('#' + Settings.FORM_ID).on('change input propertychange', '.' + Settings.TRIGGER_CHANGE_CLASS , function(e){
     // this input was changed, add the changed data attribute to it
     $(this).attr('data-changed', true);
 
@@ -676,11 +682,11 @@ function makeTableHiddenInputs(setting, initialValues, categoryValue) {
     } else {
       html +=
         "<td " + (col.hidden ? "style='display: none;'" : "") + " class='" + Settings.DATA_COL_CLASS + "' " +
-            "name='" + col.name + "'>" +
-          "<input type='text' style='display: none;' class='form-control' placeholder='" + (col.placeholder ? col.placeholder : "") + "' " +
-                 "value='" + (defaultValue || "") + "' data-default='" + (defaultValue || "") + "'" +
-                 (col.readonly ? " readonly" : "") + ">" +
-        "</td>";
+          "name='" + col.name + "'>" +
+          "<input type='text' style='display: none;' class='form-control " + Settings.TRIGGER_CHANGE_CLASS +
+          "' placeholder='" + (col.placeholder ? col.placeholder : "") + "' " +
+          "value='" + (defaultValue || "") + "' data-default='" + (defaultValue || "") + "'" +
+          (col.readonly ? " readonly" : "") + ">" + "</td>";
     }
   })
 
@@ -801,6 +807,8 @@ function badgeForDifferences(changedElement) {
     }
   });
 
+  Settings.pendingChanges = totalChanges;
+
   if (totalChanges == 0) {
     totalChanges = ""
   }
@@ -830,7 +838,7 @@ function addTableRow(row) {
     var keyInput = row.children(".key").children("input");
 
     // whenever the keyInput changes, re-badge for differences
-    keyInput.on('change keyup paste', function(e){
+    keyInput.on('change input propertychange', function(e){
       // update siblings in the row to have the correct name
       var currentKey = $(this).val();
 

domain-server/resources/web/js/domain-server.js

@@ -28,7 +28,7 @@ function settingsGroupAnchor(base, html_id) {
 }
 
 $(document).ready(function(){
-  var url = window.location;
+  var url = location.protocol + '//' + location.host+location.pathname;
 
   // Will only work if string in href matches with location
   $('ul.nav a[href="'+ url +'"]').parent().addClass('active');
@@ -39,22 +39,49 @@ $(document).ready(function(){
   }).parent().addClass('active');
 
   $('body').on('click', '#restart-server', function(e) {
-    swal( {
-      title: "Are you sure?",
-      text: "This will restart your domain server, causing your domain to be briefly offline.",
-      type: "warning",
-      html: true,
-      showCancelButton: true
-    }, function() {
-      $.get("/restart");
-      showRestartModal();
-    });
+    swalAreYouSure(
+      "This will restart your domain server, causing your domain to be briefly offline.",
+      "Restart",
+      function() {
+        swal.close();
+        $.get("/restart");
+        showRestartModal();
+      }
+    )
     return false;
   });
 
   var $contentDropdown = $('#content-settings-nav-dropdown');
   var $settingsDropdown = $('#domain-settings-nav-dropdown');
 
+  // define extra groups to add to setting panels, with their splice index
+  Settings.extraContentGroupsAtIndex = {
+    0: {
+      html_id: Settings.CONTENT_ARCHIVES_PANEL_ID,
+      label: 'Content Archives'
+    },
+    1: {
+      html_id: Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID,
+      label: 'Upload Content'
+    }
+  };
+
+  Settings.extraContentGroupsAtEnd = [];
+
+  Settings.extraDomainGroupsAtIndex = {
+    1: {
+      html_id: 'places',
+      label: 'Places'
+    }
+  }
+
+  Settings.extraDomainGroupsAtEnd = [
+    {
+      html_id: 'settings_backup',
+      label: 'Settings Backup / Restore'
+    }
+  ]
+
   // for pages that have the settings dropdowns
   if ($contentDropdown.length && $settingsDropdown.length) {
     // make a JSON request to get the dropdown menus for content and settings
@@ -65,6 +92,15 @@ $(document).ready(function(){
         return "<li class='setting-group'><a href='" + settingsGroupAnchor(base, html_id) + "'>" + group.label + "<span class='badge'></span></a></li>";
       }
 
+      // add the dummy settings groups that get populated via JS
+      for (var spliceIndex in Settings.extraContentGroupsAtIndex) {
+        data.content_settings.splice(spliceIndex, 0, Settings.extraContentGroupsAtIndex[spliceIndex]);
+      }
+
+      for (var endIndex in Settings.extraContentGroupsAtEnd) {
+        data.content_settings.push(Settings.extraContentGroupsAtEnd[endIndex]);
+      }
+
       $.each(data.content_settings, function(index, group){
         if (index > 0) {
           $contentDropdown.append("<li role='separator' class='divider'></li>");
@@ -73,25 +109,22 @@ $(document).ready(function(){
         $contentDropdown.append(makeGroupDropdownElement(group, "/content/"));
       });
 
+      // add the dummy settings groups that get populated via JS
+      for (var spliceIndex in Settings.extraDomainGroupsAtIndex) {
+        data.domain_settings.splice(spliceIndex, 0, Settings.extraDomainGroupsAtIndex[spliceIndex]);
+      }
+
+      for (var endIndex in Settings.extraDomainGroupsAtEnd) {
+        data.domain_settings.push(Settings.extraDomainGroupsAtEnd[endIndex]);
+      }
+
       $.each(data.domain_settings, function(index, group){
         if (index > 0) {
           $settingsDropdown.append("<li role='separator' class='divider'></li>");
         }
 
         $settingsDropdown.append(makeGroupDropdownElement(group, "/settings/"));
-
-        // for domain settings, we add a dummy "Places" group that we fill
-        // via the API - add it to the dropdown menu in the right spot
-        // which is after "Metaverse / Networking"
-        if (group.name == "metaverse") {
-          $settingsDropdown.append("<li role='separator' class='divider'></li>");
-          $settingsDropdown.append(makeGroupDropdownElement({ html_id: 'places', label: 'Places' }, "/settings/"));
-        }
       });
-
-      // append a link for the "Settings Backup" panel
-      $settingsDropdown.append("<li role='separator' class='divider'></li>");
-      $settingsDropdown.append(makeGroupDropdownElement({ html_id: 'settings_backup', label: 'Settings Backup'}, "/settings"));
     });
   }
 });

domain-server/resources/web/js/shared.js

@@ -42,7 +42,9 @@ Object.assign(Settings, {
   ADD_PLACE_BTN_ID: 'add-place-btn',
   FORM_ID: 'settings-form',
   INVALID_ROW_CLASS: 'invalid-input',
-  DATA_ROW_INDEX: 'data-row-index'
+  DATA_ROW_INDEX: 'data-row-index',
+  CONTENT_ARCHIVES_PANEL_ID: 'content_archives',
+  UPLOAD_CONTENT_BACKUP_PANEL_ID: 'upload_content'
 });
 
 var URLs = {
@@ -96,6 +98,17 @@ var DOMAIN_ID_TYPE_TEMP = 1;
 var DOMAIN_ID_TYPE_FULL = 2;
 var DOMAIN_ID_TYPE_UNKNOWN = 3;
 
+function swalAreYouSure(text, confirmButtonText, callback) {
+  swal({
+    title: "Are you sure?",
+    text: text,
+    type: "warning",
+    showCancelButton: true,
+    confirmButtonText: confirmButtonText,
+    closeOnConfirm: false
+  }, callback);
+}
+
 function domainIDIsSet() {
   if (typeof Settings.data.values.metaverse !== 'undefined' &&
     typeof Settings.data.values.metaverse.id !== 'undefined') {
@@ -164,7 +177,7 @@ function getDomainFromAPI(callback) {
   if (callback === undefined) {
     callback = function() {};
   }
-  
+
   if (!domainIDIsSet()) {
     callback({ status: 'fail' });
     return null;

domain-server/resources/web/settings/js/settings.js

@@ -14,17 +14,8 @@ $(document).ready(function(){
       return b;
   })(window.location.search.substr(1).split('&'));
 
-  // define extra groups to add to description, with their splice index
-  Settings.extraGroups = {
-    1: {
-      html_id: 'places',
-      label: 'Places'
-    },
-    "-1": {
-      html_id: 'settings_backup',
-      label: 'Settings Backup'
-    }
-  }
+  Settings.extraGroupsAtEnd = Settings.extraDomainGroupsAtEnd;
+  Settings.extraGroupsAtIndex = Settings.extraDomainGroupsAtIndex;
 
   Settings.afterReloadActions = function() {
     // append the domain selection modal
@@ -103,20 +94,17 @@ $(document).ready(function(){
       var password = formJSON["security"]["http_password"];
 
       if ((password == sha256_digest("")) && (username == undefined || (username && username.length != 0))) {
-        swal({
-          title: "Are you sure?",
-          text: "You have entered a blank password with a non-blank username. Are you sure you want to require a blank password?",
-          type: "warning",
-          showCancelButton: true,
-          confirmButtonColor: "#5cb85c",
-          confirmButtonText: "Yes!",
-          closeOnConfirm: true
-        },
-        function () {
+        swalAreYouSure(
+          "You have entered a blank password with a non-blank username. Are you sure you want to require a blank password?",
+          "Use blank password",
+          function() {
+            swal.close();
+            
             formJSON["security"]["http_password"] = "";
 
             postSettings(formJSON);
-        });
+          }
+        );
 
         return;
       }
@@ -643,7 +631,6 @@ $(document).ready(function(){
     autoNetworkingEl.after(form);
   }
 
-
   function setupPlacesTable() {
     // create a dummy table using our view helper
     var placesTableSetting = {
@@ -1043,32 +1030,38 @@ $(document).ready(function(){
   $('body').on('click', '#' + RESTORE_SETTINGS_UPLOAD_ID, function(e){
     e.preventDefault();
 
-    var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
+    swalAreYouSure(
+      "Your domain settings will be replaced by the uploaded settings",
+      "Restore settings",
+      function() {
+        var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
 
-    var fileFormData = new FormData();
-    fileFormData.append('restore-file', files[0]);
+        var fileFormData = new FormData();
+        fileFormData.append('restore-file', files[0]);
 
-    showSpinnerAlert("Restoring Settings");
+        showSpinnerAlert("Restoring Settings");
 
-    $.ajax({
-      url: '/settings/restore',
-      type: 'POST',
-      processData: false,
-      contentType: false,
-      dataType: 'json',
-      data: fileFormData
-    }).done(function(data, textStatus, jqXHR) {
-      swal.close();
-      showRestartModal();
-    }).fail(function(jqXHR, textStatus, errorThrown) {
-      showErrorMessage(
-        "Error",
-        "There was a problem restoring domain settings.\n"
-        + "Please ensure that your current domain settings are valid and try again."
-      );
+        $.ajax({
+          url: '/settings/restore',
+          type: 'POST',
+          processData: false,
+          contentType: false,
+          dataType: 'json',
+          data: fileFormData
+        }).done(function(data, textStatus, jqXHR) {
+          swal.close();
+          showRestartModal();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain settings.\n"
+            + "Please ensure that your current domain settings are valid and try again."
+          );
 
-      reloadSettings();
-    });
+          reloadSettings();
+        });
+      }
+    );
   });
 
   $('body').on('change', '#' + RESTORE_SETTINGS_FILE_ID, function() {
@@ -1089,7 +1082,7 @@ $(document).ready(function(){
     html += "<div class='form-group'>";
     html += "<label class='control-label'>Upload a Settings Configuration</label>";
     html += "<span class='help-block'>Upload a settings configuration to quickly configure this domain";
-    html += "<br/>Note: Your domain's settings will be replaced by the settings you upload</span>";
+    html += "<br/>Note: Your domain settings will be replaced by the settings you upload</span>";
 
     html += "<input id='restore-settings-file' name='restore-settings' type='file'>";
     html += "<button type='button' id='" + RESTORE_SETTINGS_UPLOAD_ID + "' disabled='true' class='btn btn-primary'>Upload Domain Settings</button>";
@@ -1097,8 +1090,5 @@ $(document).ready(function(){
     html += "</div>";
 
     $('#settings_backup .panel-body').html(html);
-
-    // add an upload button to the footer to kick off the upload form
-
   }
 });

domain-server/resources/web/wizard/index.shtml

@@ -261,6 +261,5 @@
 <script src='/js/underscore-min.js'></script>
 <script src='/js/bootbox.min.js'></script>
 <script src='/js/sha256.js'></script>
-<script src='/js/shared.js'></script>
 <script src='js/wizard.js'></script>
 <!--#include virtual="page-end.html"-->

domain-server/resources/web/wizard/js/wizard.js

@@ -396,10 +396,12 @@ function savePermissions() {
 
   var admins = $('#admin-usernames').val().split(',');
 
-  var existingAdmins = Settings.data.values.security.permissions.map(function(value) {
-    return value.permissions_id;
-  });
-  admins = admins.concat(existingAdmins);
+  if (Settings.data.values.security.permissions) {
+    var existingAdmins = Settings.data.values.security.permissions.map(function(value) {
+      return value.permissions_id;
+    });
+    admins = admins.concat(existingAdmins);
+  }
 
   // Filter out unique values
   admins = _.uniq(admins.map(function(username) {

domain-server/src/AssetsBackupHandler.cpp

@@ -0,0 +1,605 @@
+//
+//  AssetsBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Clement Brisset on 1/12/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "AssetsBackupHandler.h"
+
+#include <QJsonDocument>
+#include <QDate>
+#include <QtCore/QLoggingCategory>
+
+#include <quazip5/quazipfile.h>
+#include <quazip5/quazipdir.h>
+
+#include <AssetClient.h>
+#include <AssetRequest.h>
+#include <AssetUpload.h>
+#include <MappingRequest.h>
+#include <PathUtils.h>
+
+using namespace std;
+
+static const QString ASSETS_DIR { "/assets/" };
+static const QString MAPPINGS_FILE { "mappings.json" };
+static const QString ZIP_ASSETS_FOLDER { "files" };
+static const chrono::minutes MAX_REFRESH_TIME { 5 };
+
+Q_DECLARE_LOGGING_CATEGORY(asset_backup)
+Q_LOGGING_CATEGORY(asset_backup, "hifi.asset-backup");
+
+AssetsBackupHandler::AssetsBackupHandler(const QString& backupDirectory) :
+    _assetsDirectory(backupDirectory + ASSETS_DIR)
+{
+    // Make sure the asset directory exists.
+    QDir(_assetsDirectory).mkpath(".");
+
+    refreshAssetsOnDisk();
+
+    setupRefreshTimer();
+}
+
+void AssetsBackupHandler::setupRefreshTimer() {
+    _mappingsRefreshTimer.setTimerType(Qt::CoarseTimer);
+    _mappingsRefreshTimer.setSingleShot(true);
+    QObject::connect(&_mappingsRefreshTimer, &QTimer::timeout, this, &AssetsBackupHandler::refreshMappings);
+
+    auto nodeList = DependencyManager::get<LimitedNodeList>();
+    QObject::connect(nodeList.data(), &LimitedNodeList::nodeActivated, this, [this](SharedNodePointer node) {
+        if (node->getType() == NodeType::AssetServer) {
+            // run immediately for the first time.
+            _mappingsRefreshTimer.start(0);
+        }
+    });
+    QObject::connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, [this](SharedNodePointer node) {
+        if (node->getType() == NodeType::AssetServer) {
+            _mappingsRefreshTimer.stop();
+        }
+    });
+}
+
+void AssetsBackupHandler::refreshAssetsOnDisk() {
+    QDir assetsDir { _assetsDirectory };
+    auto assetNames = assetsDir.entryList(QDir::Files);
+
+    // store all valid hashes
+    copy_if(begin(assetNames), end(assetNames),
+            inserter(_assetsOnDisk, begin(_assetsOnDisk)),
+            AssetUtils::isValidHash);
+
+}
+
+void AssetsBackupHandler::refreshAssetsInBackups() {
+    _assetsInBackups.clear();
+    for (const auto& backup : _backups) {
+        for (const auto& mapping : backup.mappings) {
+            _assetsInBackups.insert(mapping.second);
+        }
+    }
+}
+
+void AssetsBackupHandler::checkForMissingAssets() {
+    vector<AssetUtils::AssetHash> missingAssets;
+    set_difference(begin(_assetsInBackups), end(_assetsInBackups),
+                   begin(_assetsOnDisk), end(_assetsOnDisk),
+                   back_inserter(missingAssets));
+    if (missingAssets.size() > 0) {
+        qCWarning(asset_backup) << "Found" << missingAssets.size() << "backup assets missing from disk.";
+    }
+}
+
+void AssetsBackupHandler::checkForAssetsToDelete() {
+    vector<AssetUtils::AssetHash> deprecatedAssets;
+    set_difference(begin(_assetsOnDisk), end(_assetsOnDisk),
+                   begin(_assetsInBackups), end(_assetsInBackups),
+                   back_inserter(deprecatedAssets));
+
+    if (deprecatedAssets.size() > 0) {
+        qCDebug(asset_backup) << "Found" << deprecatedAssets.size() << "backup assets to delete from disk.";
+        const auto noCorruptedBackups = none_of(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+            return backup.corruptedBackup;
+        });
+        if (noCorruptedBackups) {
+            for (const auto& hash : deprecatedAssets) {
+                auto success = QFile::remove(_assetsDirectory + hash);
+                if (success) {
+                    _assetsOnDisk.erase(hash);
+                } else {
+                    qCWarning(asset_backup) << "Could not delete asset:" << hash;
+                }
+            }
+        } else {
+            qCWarning(asset_backup) << "Some backups did not load properly, aborting delete operation for safety.";
+        }
+    }
+}
+
+bool AssetsBackupHandler::isCorruptedBackup(const QString& backupName) {
+    auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& value) {
+        return value.name == backupName;
+    });
+
+    if (it == end(_backups)) {
+        return false;
+    }
+
+    return it->corruptedBackup;
+}
+
+std::pair<bool, float> AssetsBackupHandler::isAvailable(const QString& backupName) {
+    const auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        return { true, 1.0f };
+    }
+
+    int mappingsMissing = 0;
+    for (const auto& mapping : it->mappings) {
+        if (_assetsLeftToRequest.find(mapping.second) != end(_assetsLeftToRequest)) {
+            ++mappingsMissing;
+        }
+    }
+
+    if (mappingsMissing == 0) {
+        return { true, 1.0f };
+    }
+
+    float progress = (float)it->mappings.size();
+    progress -= (float)mappingsMissing;
+    progress /= it->mappings.size();
+
+    return { false, progress };
+}
+
+std::pair<bool, float> AssetsBackupHandler::getRecoveryStatus() {
+    if (_assetsLeftToUpload.empty() &&
+        _mappingsLeftToSet.empty() &&
+        _mappingsLeftToDelete.empty() &&
+        _mappingRequestsInFlight == 0) {
+        return { false, 1.0f };
+    }
+
+    float progress = (float)_numRestoreOperations;
+    progress -= (float)_assetsLeftToUpload.size();
+    progress -= (float)_mappingRequestsInFlight;
+    progress /= (float)_numRestoreOperations;
+
+    return { true, progress };
+}
+
+void AssetsBackupHandler::loadBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    _backups.emplace_back(backupName, AssetUtils::Mappings(), false);
+    auto& backup = _backups.back();
+
+    if (!zip.setCurrentFile(MAPPINGS_FILE)) {
+        qCCritical(asset_backup) << "Failed to find" << MAPPINGS_FILE << "while loading backup";
+        qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QFile::ReadOnly)) {
+        qCCritical(asset_backup) << "Could not unzip backup file for load:" << MAPPINGS_FILE;
+        qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    QJsonParseError error;
+    auto document = QJsonDocument::fromJson(zipFile.readAll(), &error);
+    if (document.isNull() || !document.isObject()) {
+        qCCritical(asset_backup) << "Could not parse backup file to JSON object for load:" << MAPPINGS_FILE;
+        qCCritical(asset_backup) << "    Error:" << error.errorString();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    auto jsonObject = document.object();
+    for (auto it = begin(jsonObject); it != end(jsonObject); ++it) {
+        const auto& assetPath = it.key();
+        const auto& assetHash = it.value().toString();
+
+        if (!AssetUtils::isValidHash(assetHash)) {
+            qCCritical(asset_backup) << "Corrupted mapping in loading backup file" << backupName << ":" << it.key();
+            backup.corruptedBackup = true;
+            continue;
+        }
+
+        backup.mappings[assetPath] = assetHash;
+        _assetsInBackups.insert(assetHash);
+    }
+}
+
+void AssetsBackupHandler::loadingComplete() {
+    checkForMissingAssets();
+    checkForAssetsToDelete();
+}
+
+void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is already an operation in progress.";
+        return;
+    }
+
+    if (_lastMappingsRefresh.time_since_epoch().count() == 0) {
+        qCWarning(asset_backup) << "Current mappings not yet loaded.";
+        return;
+    }
+
+    if ((p_high_resolution_clock::now() - _lastMappingsRefresh) > MAX_REFRESH_TIME) {
+        qCWarning(asset_backup) << "Backing up asset mappings that might be stale.";
+    }
+
+    AssetUtils::Mappings mappings;
+
+    QJsonObject jsonObject;
+    for (const auto& mapping : _currentMappings) {
+        mappings[mapping.first] = mapping.second;
+        _assetsInBackups.insert(mapping.second);
+        jsonObject.insert(mapping.first, mapping.second);
+    }
+    QJsonDocument document(jsonObject);
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(MAPPINGS_FILE))) {
+        qCDebug(asset_backup) << "Could not open zip file:" << zipFile.getZipError();
+        return;
+    }
+    zipFile.write(document.toJson());
+    zipFile.close();
+    if (zipFile.getZipError() != UNZ_OK) {
+        qCDebug(asset_backup) << "Could not close zip file: " << zipFile.getZipError();
+        return;
+    }
+    _backups.emplace_back(backupName, mappings, false);
+    qDebug() << "Created asset backup:" << backupName;
+}
+
+void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is already a backup/restore in progress.";
+        return;
+    }
+
+    if (_lastMappingsRefresh.time_since_epoch().count() == 0) {
+        qCWarning(asset_backup) << "Current mappings not yet loaded.";
+        return;
+    }
+
+    if ((p_high_resolution_clock::now() - _lastMappingsRefresh) > MAX_REFRESH_TIME) {
+        qCWarning(asset_backup) << "Recovering while current asset mappings might be stale.";
+    }
+
+    auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        loadBackup(backupName, zip);
+
+        QuaZipDir zipDir { &zip, ZIP_ASSETS_FOLDER };
+
+        auto assetNames = zipDir.entryList(QDir::Files);
+        for (const auto& asset : assetNames) {
+            if (AssetUtils::isValidHash(asset)) {
+                if (!zip.setCurrentFile(zipDir.filePath(asset))) {
+                    qCCritical(asset_backup) << "Failed to find" << asset << "while recovering backup";
+                    qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+                    continue;
+                }
+
+                QuaZipFile zipFile { &zip };
+                if (!zipFile.open(QFile::ReadOnly)) {
+                    qCCritical(asset_backup) << "Could not unzip asset file:" << asset;
+                    qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+                    continue;
+                }
+
+                writeAssetFile(asset, zipFile.readAll());
+            }
+        }
+
+        // iterator is end() and has been invalidated in the `loadBackup` call
+        // grab the new iterator
+        it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+            return backup.name == backupName;
+        });
+
+        if (it == end(_backups)) {
+            qCCritical(asset_backup) << "Failed to recover backup:" << backupName;
+            return;
+        }
+    }
+
+    const auto& newMappings = it->mappings;
+    computeServerStateDifference(_currentMappings, newMappings);
+
+    restoreAllAssets();
+}
+
+void AssetsBackupHandler::deleteBackup(const QString& backupName) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is a backup/restore in progress.";
+        return;
+    }
+
+    const auto it = remove_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        qCDebug(asset_backup) << "Could not find backup" << backupName << "to delete.";
+        return;
+    }
+
+    _backups.erase(it, end(_backups));
+
+    refreshAssetsInBackups();
+    checkForAssetsToDelete();
+    qDebug() << "Deleted asset backup:" << backupName;
+}
+
+void AssetsBackupHandler::consolidateBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is a backup/restore in progress.";
+        return;
+    }
+
+    const auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        qCDebug(asset_backup) << "Could not find backup" << backupName << "to consolidate.";
+        return;
+    }
+
+    for (const auto& mapping : it->mappings) {
+        const auto& hash = mapping.second;
+
+        QDir assetsDir { _assetsDirectory };
+        QFile file { assetsDir.filePath(hash) };
+        if (!file.open(QFile::ReadOnly)) {
+            qCCritical(asset_backup) << "Could not open asset file" << file.fileName();
+            continue;
+        }
+
+        QuaZipFile zipFile { &zip };
+        if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(ZIP_ASSETS_FOLDER + "/" + hash))) {
+            qCDebug(asset_backup) << "Could not open zip file:" << zipFile.getZipError();
+            continue;
+        }
+        zipFile.write(file.readAll());
+        zipFile.close();
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCDebug(asset_backup) << "Could not close zip file: " << zipFile.getZipError();
+            continue;
+        }
+    }
+
+}
+
+void AssetsBackupHandler::refreshMappings() {
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto request = assetClient->createGetAllMappingsRequest();
+
+    QObject::connect(request, &GetAllMappingsRequest::finished, this, [this](GetAllMappingsRequest* request) {
+        if (request->getError() == MappingRequest::NoError) {
+            const auto& mappings = request->getMappings();
+
+            // Clear existing mappings
+            _currentMappings.clear();
+
+            // Set new mapping, but ignore baked assets
+            for (const auto& mapping : mappings) {
+                if (!mapping.first.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
+                    _currentMappings.insert({ mapping.first, mapping.second.hash });
+                }
+            }
+            _lastMappingsRefresh = p_high_resolution_clock::now();
+
+            downloadMissingFiles(_currentMappings);
+        } else {
+            qCCritical(asset_backup) << "Could not refresh asset server mappings.";
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        request->deleteLater();
+
+        // Launch next mappings request
+        static constexpr int MAPPINGS_REFRESH_INTERVAL = 30 * 1000;
+        _mappingsRefreshTimer.start(MAPPINGS_REFRESH_INTERVAL);
+    });
+
+    request->start();
+}
+
+void AssetsBackupHandler::downloadMissingFiles(const AssetUtils::Mappings& mappings) {
+    auto wasEmpty = _assetsLeftToRequest.empty();
+
+    for (const auto& mapping : mappings) {
+        const auto& hash = mapping.second;
+        if (_assetsOnDisk.find(hash) == end(_assetsOnDisk)) {
+            _assetsLeftToRequest.insert(hash);
+        }
+    }
+
+    // If we were empty, that means no download chain was already going, start one.
+    if (wasEmpty) {
+        downloadNextMissingFile();
+    }
+}
+
+void AssetsBackupHandler::downloadNextMissingFile() {
+    if (_assetsLeftToRequest.empty()) {
+        return;
+    }
+    auto hash = *begin(_assetsLeftToRequest);
+
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto assetRequest = assetClient->createRequest(hash);
+
+    QObject::connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) {
+        if (request->getError() == AssetRequest::NoError) {
+            qCDebug(asset_backup) << "Backing up asset" << request->getHash();
+
+            bool success = writeAssetFile(request->getHash(), request->getData());
+            if (!success) {
+                qCCritical(asset_backup) << "Failed to write asset file" << request->getHash();
+            }
+        } else {
+            qCCritical(asset_backup) << "Failed to backup asset" << request->getHash();
+        }
+
+        _assetsLeftToRequest.erase(request->getHash());
+        downloadNextMissingFile();
+
+        request->deleteLater();
+    });
+
+    assetRequest->start();
+}
+
+bool AssetsBackupHandler::writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data) {
+    QDir assetsDir { _assetsDirectory };
+    QFile file { assetsDir.filePath(hash) };
+    if (!file.open(QFile::WriteOnly)) {
+        qCCritical(asset_backup) << "Could not open asset file for write:" << file.fileName();
+        return false;
+    }
+
+    auto bytesWritten = file.write(data);
+    if (bytesWritten != data.size()) {
+        qCCritical(asset_backup) << "Could not write data to file" << file.fileName();
+        file.remove();
+        return false;
+    }
+
+    _assetsOnDisk.insert(hash);
+
+    return true;
+}
+
+void AssetsBackupHandler::computeServerStateDifference(const AssetUtils::Mappings& currentMappings,
+                                                       const AssetUtils::Mappings& newMappings) {
+    _mappingsLeftToSet.reserve((int)newMappings.size());
+    _assetsLeftToUpload.reserve((int)newMappings.size());
+    _mappingsLeftToDelete.reserve((int)currentMappings.size());
+
+    set<AssetUtils::AssetHash> currentAssets;
+    for (const auto& currentMapping : currentMappings) {
+        const auto& currentPath = currentMapping.first;
+        const auto& currentHash = currentMapping.second;
+
+        if (newMappings.find(currentPath) == end(newMappings)) {
+            _mappingsLeftToDelete.push_back(currentPath);
+        }
+        currentAssets.insert(currentHash);
+    }
+
+    for (const auto& newMapping : newMappings) {
+        const auto& newPath = newMapping.first;
+        const auto& newHash = newMapping.second;
+
+        auto it = currentMappings.find(newPath);
+        if (it == end(currentMappings) || it->second != newHash) {
+            _mappingsLeftToSet.push_back({ newPath, newHash });
+        }
+        if (currentAssets.find(newHash) == end(currentAssets)) {
+            _assetsLeftToUpload.push_back(newHash);
+        }
+    }
+
+    _numRestoreOperations = (int)_assetsLeftToUpload.size() + (int)_mappingsLeftToSet.size();
+    if (!_mappingsLeftToDelete.empty()) {
+        ++_numRestoreOperations;
+    }
+
+    qCDebug(asset_backup) << "Mappings to set:" << _mappingsLeftToSet.size();
+    qCDebug(asset_backup) << "Mappings to del:" << _mappingsLeftToDelete.size();
+    qCDebug(asset_backup) << "Assets to upload:" << _assetsLeftToUpload.size();
+}
+
+void AssetsBackupHandler::restoreAllAssets() {
+    restoreNextAsset();
+}
+
+void AssetsBackupHandler::restoreNextAsset() {
+    if (_assetsLeftToUpload.empty()) {
+        updateMappings();
+        return;
+    }
+
+    auto hash = _assetsLeftToUpload.back();
+    _assetsLeftToUpload.pop_back();
+
+    auto assetFilename = _assetsDirectory + hash;
+
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto request = assetClient->createUpload(assetFilename);
+
+    QObject::connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) {
+        if (request->getError() != AssetUpload::NoError) {
+            qCCritical(asset_backup) << "Failed to restore asset:" << request->getFilename();
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        restoreNextAsset();
+
+        request->deleteLater();
+    });
+
+    request->start();
+}
+
+void AssetsBackupHandler::updateMappings() {
+    auto assetClient = DependencyManager::get<AssetClient>();
+    for (const auto& mapping : _mappingsLeftToSet) {
+        auto request = assetClient->createSetMappingRequest(mapping.first, mapping.second);
+        QObject::connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) {
+            if (request->getError() != MappingRequest::NoError) {
+                qCCritical(asset_backup) << "Failed to set mapping:" << request->getPath();
+                qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+            }
+
+            --_mappingRequestsInFlight;
+
+            request->deleteLater();
+        });
+
+        request->start();
+        ++_mappingRequestsInFlight;
+    }
+    _mappingsLeftToSet.clear();
+
+    auto request = assetClient->createDeleteMappingsRequest(_mappingsLeftToDelete);
+    QObject::connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) {
+        if (request->getError() != MappingRequest::NoError) {
+            qCCritical(asset_backup) << "Failed to delete mappings";
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        --_mappingRequestsInFlight;
+
+        request->deleteLater();
+    });
+    _mappingsLeftToDelete.clear();
+
+    request->start();
+    ++_mappingRequestsInFlight;
+}

domain-server/src/AssetsBackupHandler.h

@@ -0,0 +1,98 @@
+//
+//  AssetsBackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 1/12/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AssetsBackupHandler_h
+#define hifi_AssetsBackupHandler_h
+
+#include <set>
+#include <map>
+
+#include <QObject>
+#include <QTimer>
+#include <QJsonDocument>
+#include <QJsonObject>
+
+#include <AssetUtils.h>
+#include <ReceivedMessage.h>
+#include <PortableHighResolutionClock.h>
+
+#include "BackupHandler.h"
+
+class AssetsBackupHandler : public QObject, public BackupHandlerInterface {
+    Q_OBJECT
+
+public:
+    AssetsBackupHandler(const QString& backupDirectory);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override;
+    std::pair<bool, float> getRecoveryStatus() override;
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override;
+    void loadingComplete() override;
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+    void deleteBackup(const QString& backupName) override;
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override;
+    bool isCorruptedBackup(const QString& backupName) override;
+
+    bool operationInProgress() { return getRecoveryStatus().first; }
+
+private:
+    void setupRefreshTimer();
+    void refreshMappings();
+
+    void refreshAssetsInBackups();
+    void refreshAssetsOnDisk();
+    void checkForMissingAssets();
+    void checkForAssetsToDelete();
+
+    void downloadMissingFiles(const AssetUtils::Mappings& mappings);
+    void downloadNextMissingFile();
+    bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data);
+
+    void computeServerStateDifference(const AssetUtils::Mappings& currentMappings,
+                                      const AssetUtils::Mappings& newMappings);
+    void restoreAllAssets();
+    void restoreNextAsset();
+    void updateMappings();
+
+    QString _assetsDirectory;
+
+    QTimer _mappingsRefreshTimer;
+    p_high_resolution_clock::time_point _lastMappingsRefresh;
+    AssetUtils::Mappings _currentMappings;
+
+    struct AssetServerBackup {
+        AssetServerBackup(const QString& pName, AssetUtils::Mappings pMappings, bool pCorruptedBackup) :
+            name(pName), mappings(pMappings), corruptedBackup(pCorruptedBackup) {}
+
+        QString name;
+        AssetUtils::Mappings mappings;
+        bool corruptedBackup;
+    };
+
+    // Internal storage for backups on disk
+    std::vector<AssetServerBackup> _backups;
+    std::set<AssetUtils::AssetHash> _assetsInBackups;
+    std::set<AssetUtils::AssetHash> _assetsOnDisk;
+
+    // Internal storage for backup in progress
+    std::set<AssetUtils::AssetHash> _assetsLeftToRequest;
+
+    // Internal storage for restore in progress
+    std::vector<AssetUtils::AssetHash> _assetsLeftToUpload;
+    std::vector<std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash>> _mappingsLeftToSet;
+    AssetUtils::AssetPathList _mappingsLeftToDelete;
+    int _mappingRequestsInFlight { 0 };
+    int _numRestoreOperations { 0 }; // Used to compute a restore progress.
+};
+
+#endif /* hifi_AssetsBackupHandler_h */

domain-server/src/BackupHandler.h

@@ -0,0 +1,40 @@
+//
+//  BackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/5/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_BackupHandler_h
+#define hifi_BackupHandler_h
+
+#include <memory>
+
+#include <QString>
+
+class QuaZip;
+
+class BackupHandlerInterface {
+public:
+    virtual ~BackupHandlerInterface() = default;
+
+    virtual std::pair<bool, float> isAvailable(const QString& backupName) = 0;
+
+    // Returns whether a recovery is ongoing and a progress between 0 and 1 if one is.
+    virtual std::pair<bool, float> getRecoveryStatus() = 0;
+
+    virtual void loadBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void loadingComplete() = 0;
+    virtual void createBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void recoverBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void deleteBackup(const QString& backupName) = 0;
+    virtual void consolidateBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual bool isCorruptedBackup(const QString& backupName) = 0;
+};
+using BackupHandlerPointer = std::unique_ptr<BackupHandlerInterface>;
+
+#endif /* hifi_BackupHandler_h */

domain-server/src/BackupSupervisor.cpp

@@ -1,400 +0,0 @@
-//
-//  BackupSupervisor.cpp
-//  domain-server/src
-//
-//  Created by Clement Brisset on 1/12/18.
-//  Copyright 2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#include "BackupSupervisor.h"
-
-#include <QJsonDocument>
-#include <QDate>
-
-#include <AssetClient.h>
-#include <AssetRequest.h>
-#include <AssetUpload.h>
-#include <MappingRequest.h>
-#include <PathUtils.h>
-
-const QString BACKUPS_DIR = "backups/";
-const QString ASSETS_DIR = "files/";
-const QString MAPPINGS_PREFIX = "mappings-";
-
-using namespace std;
-
-BackupSupervisor::BackupSupervisor() {
-    _backupsDirectory = PathUtils::getAppDataPath() + BACKUPS_DIR;
-    QDir backupDir { _backupsDirectory };
-    if (!backupDir.exists()) {
-        backupDir.mkpath(".");
-    }
-
-    _assetsDirectory = PathUtils::getAppDataPath() + BACKUPS_DIR + ASSETS_DIR;
-    QDir assetsDir { _assetsDirectory };
-    if (!assetsDir.exists()) {
-        assetsDir.mkpath(".");
-    }
-
-    loadAllBackups();
-}
-
-void BackupSupervisor::loadAllBackups() {
-    _backups.clear();
-    _assetsInBackups.clear();
-    _assetsOnDisk.clear();
-    _allBackupsLoadedSuccessfully = true;
-
-    QDir assetsDir { _assetsDirectory };
-    auto assetNames = assetsDir.entryList(QDir::Files);
-    qDebug() << "Loading" << assetNames.size() << "assets.";
-
-    // store all valid hashes
-    copy_if(begin(assetNames), end(assetNames),
-            inserter(_assetsOnDisk, begin(_assetsOnDisk)), AssetUtils::isValidHash);
-
-    QDir backupsDir { _backupsDirectory };
-    auto files = backupsDir.entryList({ MAPPINGS_PREFIX + "*.json" }, QDir::Files);
-    qDebug() << "Loading" << files.size() << "backups.";
-
-    for (const auto& fileName : files) {
-        auto filePath = backupsDir.filePath(fileName);
-        auto success = loadBackup(filePath);
-        if (!success) {
-            qCritical() << "Failed to load backup file" << filePath;
-            _allBackupsLoadedSuccessfully = false;
-        }
-    }
-
-    vector<AssetUtils::AssetHash> missingAssets;
-    set_difference(begin(_assetsInBackups), end(_assetsInBackups),
-                   begin(_assetsOnDisk), end(_assetsOnDisk),
-                   back_inserter(missingAssets));
-    if (missingAssets.size() > 0) {
-        qWarning() << "Found" << missingAssets.size() << "assets missing.";
-    }
-
-    vector<AssetUtils::AssetHash> deprecatedAssets;
-    set_difference(begin(_assetsOnDisk), end(_assetsOnDisk),
-                   begin(_assetsInBackups), end(_assetsInBackups),
-                   back_inserter(deprecatedAssets));
-
-    if (deprecatedAssets.size() > 0) {
-        qDebug() << "Found" << deprecatedAssets.size() << "assets to delete.";
-        if (_allBackupsLoadedSuccessfully) {
-            for (const auto& hash : deprecatedAssets) {
-                QFile::remove(_assetsDirectory + hash);
-            }
-        } else {
-            qWarning() << "Some backups did not load properly, aborting deleting for safety.";
-        }
-    }
-}
-
-bool BackupSupervisor::loadBackup(const QString& backupFile) {
-    _backups.push_back({ backupFile.toStdString(), {}, false });
-    auto& backup = _backups.back();
-
-    QFile file { backupFile };
-    if (!file.open(QFile::ReadOnly)) {
-        qCritical() << "Could not open backup file:" << backupFile;
-        backup.corruptedBackup = true;
-        return false;
-    }
-    QJsonParseError error;
-    auto document = QJsonDocument::fromJson(file.readAll(), &error);
-    if (document.isNull() || !document.isObject()) {
-        qCritical() << "Could not parse backup file to JSON object:" << backupFile;
-        qCritical() << "    Error:" << error.errorString();
-        backup.corruptedBackup = true;
-        return false;
-    }
-
-    auto jsonObject = document.object();
-    for (auto it = begin(jsonObject); it != end(jsonObject); ++it) {
-        const auto& assetPath = it.key();
-        const auto& assetHash = it.value().toString();
-
-        if (!AssetUtils::isValidHash(assetHash)) {
-            qCritical() << "Corrupted mapping in backup file" << backupFile << ":" << it.key();
-            backup.corruptedBackup = true;
-            return false;
-        }
-
-        backup.mappings[assetPath] = assetHash;
-        _assetsInBackups.insert(assetHash);
-    }
-
-    _backups.push_back(backup);
-    return true;
-}
-
-void BackupSupervisor::backupAssetServer() {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is already a backup/restore in progress.";
-        return;
-    }
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createGetAllMappingsRequest();
-
-    connect(request, &GetAllMappingsRequest::finished, this, [this](GetAllMappingsRequest* request) {
-        qDebug() << "Got" << request->getMappings().size() << "mappings!";
-
-        if (request->getError() != MappingRequest::NoError) {
-            qCritical() << "Could not complete backup.";
-            qCritical() << "    Error:" << request->getErrorString();
-            finishBackup();
-            request->deleteLater();
-            return;
-        }
-
-        if (!writeBackupFile(request->getMappings())) {
-            finishBackup();
-            request->deleteLater();
-            return;
-        }
-
-        assert(!_backups.empty());
-        const auto& mappings = _backups.back().mappings;
-        backupMissingFiles(mappings);
-
-        request->deleteLater();
-    });
-
-    startBackup();
-    request->start();
-}
-
-void BackupSupervisor::backupMissingFiles(const AssetUtils::Mappings& mappings) {
-    _assetsLeftToRequest.reserve(mappings.size());
-    for (auto& mapping : mappings) {
-        const auto& hash = mapping.second;
-        if (_assetsOnDisk.find(hash) == end(_assetsOnDisk)) {
-            _assetsLeftToRequest.push_back(hash);
-        }
-    }
-
-    backupNextMissingFile();
-}
-
-void BackupSupervisor::backupNextMissingFile() {
-    if (_assetsLeftToRequest.empty()) {
-        finishBackup();
-        return;
-    }
-
-    auto hash = _assetsLeftToRequest.back();
-    _assetsLeftToRequest.pop_back();
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto assetRequest = assetClient->createRequest(hash);
-
-    connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) {
-        if (request->getError() == AssetRequest::NoError) {
-            qDebug() << "Got" << request->getHash();
-
-            bool success = writeAssetFile(request->getHash(), request->getData());
-            if (!success) {
-                qCritical() << "Failed to write asset file" << request->getHash();
-            }
-        } else {
-            qCritical() << "Failed to backup asset" << request->getHash();
-        }
-
-        backupNextMissingFile();
-
-        request->deleteLater();
-    });
-
-    assetRequest->start();
-}
-
-bool BackupSupervisor::writeBackupFile(const AssetUtils::AssetMappings& mappings) {
-    auto filename = MAPPINGS_PREFIX + QDateTime::currentDateTimeUtc().toString(Qt::ISODate) + ".json";
-    QFile file { PathUtils::getAppDataPath() + BACKUPS_DIR + filename };
-    if (!file.open(QFile::WriteOnly)) {
-        qCritical() << "Could not open backup file" << file.fileName();
-        return false;
-    }
-
-    AssetServerBackup backup;
-    QJsonObject jsonObject;
-    for (auto& mapping : mappings) {
-        backup.mappings[mapping.first] = mapping.second.hash;
-        _assetsInBackups.insert(mapping.second.hash);
-        jsonObject.insert(mapping.first, mapping.second.hash);
-    }
-
-    QJsonDocument document(jsonObject);
-    file.write(document.toJson());
-
-    backup.filePath = file.fileName().toStdString();
-    _backups.push_back(backup);
-
-    return true;
-}
-
-bool BackupSupervisor::writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data) {
-    QDir assetsDir { _assetsDirectory };
-    QFile file { assetsDir.filePath(hash) };
-    if (!file.open(QFile::WriteOnly)) {
-        qCritical() << "Could not open backup file" << file.fileName();
-        return false;
-    }
-
-    file.write(data);
-
-    _assetsOnDisk.insert(hash);
-
-    return true;
-}
-
-void BackupSupervisor::restoreAssetServer(int backupIndex) {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is already a backup/restore in progress.";
-        return;
-    }
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createGetAllMappingsRequest();
-
-    connect(request, &GetAllMappingsRequest::finished, this, [this, backupIndex](GetAllMappingsRequest* request) {
-        if (request->getError() == MappingRequest::NoError) {
-            const auto& newMappings = _backups.at(backupIndex).mappings;
-            computeServerStateDifference(request->getMappings(), newMappings);
-
-            restoreAllAssets();
-        } else {
-            finishRestore();
-        }
-
-        request->deleteLater();
-    });
-
-    startRestore();
-    request->start();
-}
-
-void BackupSupervisor::computeServerStateDifference(const AssetUtils::AssetMappings& currentMappings,
-                                                    const AssetUtils::Mappings& newMappings) {
-    _mappingsLeftToSet.reserve((int)newMappings.size());
-    _assetsLeftToUpload.reserve((int)newMappings.size());
-    _mappingsLeftToDelete.reserve((int)currentMappings.size());
-
-    set<AssetUtils::AssetHash> currentAssets;
-    for (const auto& currentMapping : currentMappings) {
-        const auto& currentPath = currentMapping.first;
-        const auto& currentHash = currentMapping.second.hash;
-
-        if (newMappings.find(currentPath) == end(newMappings)) {
-            _mappingsLeftToDelete.push_back(currentPath);
-        }
-        currentAssets.insert(currentHash);
-    }
-
-    for (const auto& newMapping : newMappings) {
-        const auto& newPath = newMapping.first;
-        const auto& newHash = newMapping.second;
-
-        auto it = currentMappings.find(newPath);
-        if (it == end(currentMappings) || it->second.hash != newHash) {
-            _mappingsLeftToSet.push_back({ newPath, newHash });
-        }
-        if (currentAssets.find(newHash) == end(currentAssets)) {
-            _assetsLeftToUpload.push_back(newHash);
-        }
-    }
-
-    qDebug() << "Mappings to set:" << _mappingsLeftToSet.size();
-    qDebug() << "Mappings to del:" << _mappingsLeftToDelete.size();
-    qDebug() << "Assets to upload:" << _assetsLeftToUpload.size();
-}
-
-void BackupSupervisor::restoreAllAssets() {
-    restoreNextAsset();
-}
-
-void BackupSupervisor::restoreNextAsset() {
-    if (_assetsLeftToUpload.empty()) {
-        updateMappings();
-        return;
-    }
-
-    auto hash = _assetsLeftToUpload.back();
-    _assetsLeftToUpload.pop_back();
-
-    auto assetFilename = _assetsDirectory + hash;
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createUpload(assetFilename);
-
-    connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) {
-        if (request->getError() != AssetUpload::NoError) {
-            qCritical() << "Failed to restore asset:" << request->getFilename();
-            qCritical() << "    Error:" << request->getErrorString();
-        }
-
-        restoreNextAsset();
-
-        request->deleteLater();
-    });
-
-    request->start();
-}
-
-void BackupSupervisor::updateMappings() {
-    auto assetClient = DependencyManager::get<AssetClient>();
-    for (const auto& mapping : _mappingsLeftToSet) {
-        auto request = assetClient->createSetMappingRequest(mapping.first, mapping.second);
-        connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) {
-            if (request->getError() != MappingRequest::NoError) {
-                qCritical() << "Failed to set mapping:" << request->getPath();
-                qCritical() << "    Error:" << request->getErrorString();
-            }
-
-            if (--_mappingRequestsInFlight == 0) {
-                finishRestore();
-            }
-
-            request->deleteLater();
-        });
-
-        request->start();
-        ++_mappingRequestsInFlight;
-    }
-    _mappingsLeftToSet.clear();
-
-    auto request = assetClient->createDeleteMappingsRequest(_mappingsLeftToDelete);
-    connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) {
-        if (request->getError() != MappingRequest::NoError) {
-            qCritical() << "Failed to delete mappings";
-            qCritical() << "    Error:" << request->getErrorString();
-        }
-
-        if (--_mappingRequestsInFlight == 0) {
-            finishRestore();
-        }
-
-        request->deleteLater();
-    });
-    _mappingsLeftToDelete.clear();
-
-    request->start();
-    ++_mappingRequestsInFlight;
-}
-bool BackupSupervisor::deleteBackup(int backupIndex) {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is a backup/restore in progress.";
-        return false;
-    }
-    const auto& filePath = _backups.at(backupIndex).filePath;
-    auto success = QFile::remove(filePath.c_str());
-
-    loadAllBackups();
-
-    return success;
-}

domain-server/src/BackupSupervisor.h

@@ -1,85 +0,0 @@
-//
-//  BackupSupervisor.h
-//  domain-server/src
-//
-//  Created by Clement Brisset on 1/12/18.
-//  Copyright 2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#ifndef hifi_BackupSupervisor_h
-#define hifi_BackupSupervisor_h
-
-#include <set>
-#include <map>
-
-#include <QObject>
-
-#include <AssetUtils.h>
-
-#include <ReceivedMessage.h>
-
-struct AssetServerBackup {
-    std::string filePath;
-    AssetUtils::Mappings mappings;
-    bool corruptedBackup;
-};
-
-class BackupSupervisor : public QObject {
-    Q_OBJECT
-
-public:
-    BackupSupervisor();
-
-    void backupAssetServer();
-    void restoreAssetServer(int backupIndex);
-    bool deleteBackup(int backupIndex);
-
-    const std::vector<AssetServerBackup>& getBackups() const { return _backups; };
-
-    bool backupInProgress() const { return _backupInProgress; }
-    bool restoreInProgress() const { return _restoreInProgress; }
-
-private:
-    void loadAllBackups();
-    bool loadBackup(const QString& backupFile);
-
-    void startBackup() { _backupInProgress = true; }
-    void finishBackup() { _backupInProgress = false; }
-    void backupMissingFiles(const AssetUtils::Mappings& mappings);
-    void backupNextMissingFile();
-    bool writeBackupFile(const AssetUtils::AssetMappings& mappings);
-    bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data);
-
-    void startRestore() { _restoreInProgress = true; }
-    void finishRestore() { _restoreInProgress = false; }
-    void computeServerStateDifference(const AssetUtils::AssetMappings& currentMappings,
-                                      const AssetUtils::Mappings& newMappings);
-    void restoreAllAssets();
-    void restoreNextAsset();
-    void updateMappings();
-
-    QString _backupsDirectory;
-    QString _assetsDirectory;
-
-    // Internal storage for backups on disk
-    bool _allBackupsLoadedSuccessfully { false };
-    std::vector<AssetServerBackup> _backups;
-    std::set<AssetUtils::AssetHash> _assetsInBackups;
-    std::set<AssetUtils::AssetHash> _assetsOnDisk;
-
-    // Internal storage for backup in progress
-    bool _backupInProgress { false };
-    std::vector<AssetUtils::AssetHash> _assetsLeftToRequest;
-
-    // Internal storage for restore in progress
-    bool _restoreInProgress { false };
-    std::vector<AssetUtils::AssetHash> _assetsLeftToUpload;
-    std::vector<std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash>> _mappingsLeftToSet;
-    AssetUtils::AssetPathList _mappingsLeftToDelete;
-    int _mappingRequestsInFlight { 0 };
-};
-
-#endif /* hifi_BackupSupervisor_h */

domain-server/src/ContentSettingsBackupHandler.cpp

@@ -0,0 +1,74 @@
+//
+//  ContentSettingsBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Stephen Birarda on 2/15/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "ContentSettingsBackupHandler.h"
+
+#include <quazip5/quazip.h>
+#include <quazip5/quazipfile.h>
+
+ContentSettingsBackupHandler::ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager) :
+    _settingsManager(domainServerSettingsManager)
+{
+}
+
+static const QString CONTENT_SETTINGS_BACKUP_FILENAME = "content-settings.json";
+
+void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+
+    // grab the content settings as JSON, excluding default values and values hidden from backup
+    QJsonObject contentSettingsJSON = _settingsManager.settingsResponseObjectForType(
+        "", // include all settings types
+        DomainServerSettingsManager::Authenticated, DomainServerSettingsManager::NoDomainSettings,
+        DomainServerSettingsManager::IncludeContentSettings, DomainServerSettingsManager::NoDefaultSettings,
+        DomainServerSettingsManager::ForBackup
+    );
+
+    // make a QJsonDocument using the object
+    QJsonDocument contentSettingsDocument { contentSettingsJSON };
+
+    QuaZipFile zipFile { &zip };
+
+    if (zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(CONTENT_SETTINGS_BACKUP_FILENAME))) {
+        if (zipFile.write(contentSettingsDocument.toJson()) == -1) {
+            qCritical().nospace() << "Failed to write to " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+
+        zipFile.close();
+
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCritical().nospace() << "Failed to zip " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+    } else {
+        qCritical().nospace() << "Failed to open " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+    }
+}
+
+void ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    if (!zip.setCurrentFile(CONTENT_SETTINGS_BACKUP_FILENAME)) {
+        qWarning() << "Failed to find" << CONTENT_SETTINGS_BACKUP_FILENAME << "while recovering backup";
+        return;
+    }
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::ReadOnly)) {
+        qCritical() << "Failed to open" << CONTENT_SETTINGS_BACKUP_FILENAME << "in backup";
+        return;
+    }
+
+    auto rawData = zipFile.readAll();
+    zipFile.close();
+
+    QJsonDocument jsonDocument = QJsonDocument::fromJson(rawData);
+
+    if (!_settingsManager.restoreSettingsFromObject(jsonDocument.object(), ContentSettings)) {
+        qCritical() << "Failed to restore settings from" << CONTENT_SETTINGS_BACKUP_FILENAME << "in content archive";
+    }
+}

domain-server/src/ContentSettingsBackupHandler.h

@@ -0,0 +1,43 @@
+//
+//  ContentSettingsBackupHandler.h
+//  domain-server/src
+//
+//  Created by Stephen Birarda on 2/15/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_ContentSettingsBackupHandler_h
+#define hifi_ContentSettingsBackupHandler_h
+
+#include "BackupHandler.h"
+#include "DomainServerSettingsManager.h"
+
+class ContentSettingsBackupHandler : public BackupHandlerInterface {
+public:
+    ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override { return { true, 1.0f }; }
+    std::pair<bool, float> getRecoveryStatus() override { return { false, 1.0f }; }
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override {}
+
+    void loadingComplete() override {}
+
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+
+    void deleteBackup(const QString& backupName) override {}
+
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override {}
+
+    bool isCorruptedBackup(const QString& backupName) override { return false; }
+
+private:
+    DomainServerSettingsManager& _settingsManager;
+};
+
+#endif // hifi_ContentSettingsBackupHandler_h

domain-server/src/DomainContentBackupManager.cpp

@@ -0,0 +1,599 @@
+//
+//  DomainContentBackupManager.cpp
+//  libraries/domain-server/src
+//
+//  Created by Ryan Huffman on 1/01/18.
+//  Adapted from OctreePersistThread
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "DomainContentBackupManager.h"
+
+#include <chrono>
+#include <thread>
+
+#include <cstdio>
+#include <fstream>
+#include <time.h>
+
+#include <QBuffer>
+#include <QDateTime>
+#include <QDebug>
+#include <QDir>
+#include <QDirIterator>
+#include <QFile>
+#include <QJsonArray>
+#include <QJsonObject>
+#include <QJsonDocument>
+
+#include <quazip5/quazip.h>
+
+#include <NumericalConstants.h>
+#include <PerfStat.h>
+#include <PathUtils.h>
+#include <shared/QtHelpers.h>
+
+#include "DomainServer.h"
+
+const std::chrono::seconds DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL { 30 };
+
+// Backup format looks like: daily_backup-TIMESTAMP.zip
+static const QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
+static const QString DATETIME_FORMAT_RE { "\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}" };
+static const QString AUTOMATIC_BACKUP_PREFIX { "autobackup-" };
+static const QString MANUAL_BACKUP_PREFIX { "backup-" };
+static const QString MANUAL_BACKUP_NAME_RE { "[a-zA-Z0-9\\-_ ]+" };
+
+void DomainContentBackupManager::addBackupHandler(BackupHandlerPointer handler) {
+    _backupHandlers.push_back(std::move(handler));
+}
+
+DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory,
+                                                       const QVariantList& backupRules,
+                                                       std::chrono::milliseconds persistInterval,
+                                                       bool debugTimestampNow) :
+    _backupDirectory(backupDirectory), _persistInterval(persistInterval), _lastCheck(p_high_resolution_clock::now())
+{
+
+    setObjectName("DomainContentBackupManager");
+
+    // Make sure the backup directory exists.
+    QDir(_backupDirectory).mkpath(".");
+
+    parseBackupRules(backupRules);
+}
+
+void DomainContentBackupManager::parseBackupRules(const QVariantList& backupRules) {
+    qCDebug(domain_server) << "BACKUP RULES:";
+
+    for (const QVariant& value : backupRules) {
+        QVariantMap map = value.toMap();
+
+        int interval = map["backupInterval"].toInt();
+        int count = map["maxBackupVersions"].toInt();
+        auto name = map["Name"].toString();
+        auto format = name.toLower();
+        QRegExp matchDisallowedCharacters { "[^a-zA-Z0-9\\-_]+" };
+        format.replace(matchDisallowedCharacters, "_");
+
+        qCDebug(domain_server) << "    Name:" << name;
+        qCDebug(domain_server) << "        format:" << format;
+        qCDebug(domain_server) << "        interval:" << interval;
+        qCDebug(domain_server) << "        count:" << count;
+
+        BackupRule newRule = { name, interval, format, count, 0 };
+
+        newRule.lastBackupSeconds = getMostRecentBackupTimeInSecs(format);
+
+        if (newRule.lastBackupSeconds > 0) {
+            auto now = QDateTime::currentSecsSinceEpoch();
+            auto sinceLastBackup = now - newRule.lastBackupSeconds;
+            qCDebug(domain_server).noquote() << "        lastBackup:" <<  formatSecTime(sinceLastBackup) << "ago";
+        } else {
+            qCDebug(domain_server) << "        lastBackup: NEVER";
+        }
+
+        _backupRules.push_back(newRule);
+    }
+}
+
+void DomainContentBackupManager::refreshBackupRules() {
+    for (auto& backup : _backupRules) {
+        backup.lastBackupSeconds = getMostRecentBackupTimeInSecs(backup.extensionFormat);
+    }
+}
+
+int64_t DomainContentBackupManager::getMostRecentBackupTimeInSecs(const QString& format) {
+    int64_t mostRecentBackupInSecs = 0;
+
+    QString mostRecentBackupFileName;
+    QDateTime mostRecentBackupTime;
+
+    bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
+
+    if (recentBackup) {
+        mostRecentBackupInSecs = mostRecentBackupTime.toSecsSinceEpoch();
+    }
+
+    return mostRecentBackupInSecs;
+}
+
+void DomainContentBackupManager::setup() {
+    auto backups = getAllBackups();
+    for (auto& backup : backups) {
+        QFile backupFile { backup.absolutePath };
+        if (!backupFile.open(QIODevice::ReadOnly)) {
+            qCritical() << "Could not open file:" << backup.absolutePath;
+            qCritical() << "    ERROR:" << backupFile.errorString();
+            continue;
+        }
+
+        QuaZip zip { &backupFile };
+        if (!zip.open(QuaZip::mdUnzip)) {
+            qCritical() << "Could not open backup archive:" << backup.absolutePath;
+            qCritical() << "    ERROR:" << zip.getZipError();
+            continue;
+        }
+
+        for (auto& handler : _backupHandlers) {
+            handler->loadBackup(backup.id, zip);
+        }
+
+        zip.close();
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->loadingComplete();
+    }
+}
+
+bool DomainContentBackupManager::process() {
+    if (isStillRunning()) {
+        constexpr int64_t MSECS_TO_USECS = 1000;
+        constexpr int64_t USECS_TO_SLEEP = 10 * MSECS_TO_USECS;  // every 10ms
+        std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
+
+        if (_isRecovering) {
+            bool isStillRecovering = any_of(begin(_backupHandlers), end(_backupHandlers), [](const BackupHandlerPointer& handler) {
+                return handler->getRecoveryStatus().first;
+            });
+
+            if (!isStillRecovering) {
+                _isRecovering = false;
+                _recoveryFilename = "";
+                emit recoveryCompleted();
+            }
+        }
+
+        auto now = p_high_resolution_clock::now();
+        auto sinceLastSave = now - _lastCheck;
+        if (sinceLastSave > _persistInterval) {
+            _lastCheck = now;
+
+            if (!_isRecovering) {
+                backup();
+            }
+        }
+    }
+
+    return isStillRunning();
+}
+
+void DomainContentBackupManager::shutdown() {
+    // Destroy handlers on the correct thread so that they can cleanup timers
+    _backupHandlers.clear();
+}
+
+void DomainContentBackupManager::aboutToFinish() {
+    _stopThread = true;
+}
+
+bool DomainContentBackupManager::getMostRecentBackup(const QString& format,
+                                                     QString& mostRecentBackupFileName,
+                                                     QDateTime& mostRecentBackupTime) {
+    QRegExp formatRE { AUTOMATIC_BACKUP_PREFIX + QRegExp::escape(format) + "\\-(" + DATETIME_FORMAT_RE + ")" + "\\.zip" };
+
+    QStringList filters;
+    filters << AUTOMATIC_BACKUP_PREFIX + format + "*.zip";
+
+    bool bestBackupFound = false;
+    QString bestBackupFile;
+    QDateTime bestBackupFileTime;
+
+    // Iterate over all of the backup files in the persist location
+    QDirIterator dirIterator(_backupDirectory, filters, QDir::Files | QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
+    while (dirIterator.hasNext()) {
+        dirIterator.next();
+        auto fileName = dirIterator.fileInfo().fileName();
+
+        if (formatRE.exactMatch(fileName)) {
+            auto datetime = formatRE.cap(1);
+            auto createdAt = QDateTime::fromString(datetime, DATETIME_FORMAT);
+
+            if (!createdAt.isValid()) {
+                qDebug() << "Skipping backup with invalid timestamp: " << datetime;
+                continue;
+            }
+
+            qDebug() << "Checking " << dirIterator.fileInfo().filePath();
+
+            // Based on last modified date, track the most recently modified file as the best backup
+            if (createdAt > bestBackupFileTime) {
+                bestBackupFound = true;
+                bestBackupFile = dirIterator.filePath();
+                bestBackupFileTime = createdAt;
+            }
+        } else {
+            qDebug() << "NO match: " << fileName << formatRE;
+        }
+    }
+
+    // If we found a backup then return the results
+    if (bestBackupFound) {
+        mostRecentBackupFileName = bestBackupFile;
+        mostRecentBackupTime = bestBackupFileTime;
+    }
+    return bestBackupFound;
+}
+
+void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, const QString& backupName) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "deleteBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, backupName));
+        return;
+    }
+
+    if (_isRecovering && backupName == _recoveryFilename) {
+        promise->resolve({
+            { "success", false }
+        });
+        return;
+    }
+
+    QDir backupDir { _backupDirectory };
+    QFile backupFile { backupDir.filePath(backupName) };
+    auto success = backupFile.remove();
+
+    refreshBackupRules();
+
+    for (auto& handler : _backupHandlers) {
+        handler->deleteBackup(backupName);
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip) {
+    if (!zip.open(QuaZip::Mode::mdUnzip)) {
+        qWarning() << "Failed to unzip file: " << backupName;
+        return false;
+    } else {
+        _isRecovering = true;
+        _recoveryFilename = backupName;
+
+        for (auto& handler : _backupHandlers) {
+            handler->recoverBackup(backupName, zip);
+        }
+
+        qDebug() << "Successfully started recovering from " << backupName;
+        return true;
+    }
+}
+
+void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, const QString& backupName) {
+    if (_isRecovering) {
+        promise->resolve({
+            { "success", false }
+        });
+        return;
+    };
+
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "recoverFromBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, backupName));
+        return;
+    }
+
+    qDebug() << "Recovering from" << backupName;
+
+    bool success { false };
+    QDir backupDir { _backupDirectory };
+    auto backupFilePath { backupDir.filePath(backupName) };
+    QFile backupFile { backupFilePath };
+    if (backupFile.open(QIODevice::ReadOnly)) {
+        QuaZip zip { &backupFile };
+
+        success = recoverFromBackupZip(backupName, zip);
+
+        backupFile.close();
+    } else {
+        success = false;
+        qWarning() << "Failed to open backup file for reading: " << backupFilePath;
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup) {
+
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "recoverFromUploadedBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(QByteArray, uploadedBackup));
+        return;
+    }
+
+    qDebug() << "Recovering from uploaded content archive";
+
+    // create a buffer and then a QuaZip from that buffer
+    QBuffer uploadedBackupBuffer { &uploadedBackup };
+    QuaZip uploadedZip { &uploadedBackupBuffer };
+
+    QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
+    bool success = recoverFromBackupZip(backupName, uploadedZip);
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
+
+    QDir backupDir { _backupDirectory };
+    auto matchingFiles =
+            backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + "*.zip", MANUAL_BACKUP_PREFIX + "*.zip" },
+                                    QDir::Files | QDir::NoSymLinks, QDir::Name);
+    QString prefixFormat = "(" + QRegExp::escape(AUTOMATIC_BACKUP_PREFIX) + "|" + QRegExp::escape(MANUAL_BACKUP_PREFIX) + ")";
+    QString nameFormat = "(.+)";
+    QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
+    QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
+
+    std::vector<BackupItemInfo> backups;
+
+    for (const auto& fileInfo : matchingFiles) {
+        auto fileName = fileInfo.fileName();
+        if (backupNameFormat.exactMatch(fileName)) {
+            auto type = backupNameFormat.cap(1);
+            auto name = backupNameFormat.cap(2);
+            auto dateTime = backupNameFormat.cap(3);
+            auto createdAt = QDateTime::fromString(dateTime, DATETIME_FORMAT);
+            if (!createdAt.isValid()) {
+                qDebug().nospace() << "Skipping backup (" << fileName << ") with invalid timestamp: " << dateTime;
+                continue;
+            }
+
+            backups.emplace_back(fileInfo.fileName(), name, fileInfo.absoluteFilePath(), createdAt,
+                                 type == MANUAL_BACKUP_PREFIX);
+        }
+    }
+
+    return backups;
+}
+
+void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise promise) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "getAllBackupsAndStatus", Q_ARG(MiniPromise::Promise, promise));
+        return;
+    }
+
+    auto backups = getAllBackups();
+
+    QVariantList variantBackups;
+
+    for (auto& backup : backups) {
+        bool isAvailable { true };
+        bool isCorrupted { false };
+        float availabilityProgress { 0.0f };
+        for (auto& handler : _backupHandlers) {
+            bool handlerIsAvailable { true };
+            float progress { 0.0f };
+            std::tie(handlerIsAvailable, progress) = handler->isAvailable(backup.id);
+            isAvailable &= handlerIsAvailable;
+            availabilityProgress += progress / _backupHandlers.size();
+
+            isCorrupted = isCorrupted || handler->isCorruptedBackup(backup.id);
+        }
+        variantBackups.push_back(QVariantMap({
+            { "id", backup.id },
+            { "name", backup.name },
+            { "createdAtMillis", backup.createdAt.toMSecsSinceEpoch() },
+            { "isAvailable", isAvailable },
+            { "availabilityProgress", availabilityProgress },
+            { "isManualBackup", backup.isManualBackup },
+            { "isCorrupted", isCorrupted }
+        }));
+    }
+
+    float recoveryProgress = 0.0f;
+    bool isRecovering = _isRecovering.load();
+    if (_isRecovering) {
+        for (auto& handler : _backupHandlers) {
+            float progress = handler->getRecoveryStatus().second;
+            recoveryProgress += progress / _backupHandlers.size();
+        }
+    }
+
+    QVariantMap status { 
+        { "isRecovering", isRecovering },
+        { "recoveringBackupId", _recoveryFilename },
+        { "recoveryProgress", recoveryProgress }
+    };
+
+    QVariantMap info {
+        { "backups", variantBackups },
+        { "status", status }
+    };
+
+    promise->resolve(info);
+}
+
+void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
+    QDir backupDir { _backupDirectory };
+    if (backupDir.exists() && rule.maxBackupVersions > 0) {
+        qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name;
+
+        auto matchingFiles =
+                backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
+
+        int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions;
+        if (backupsToDelete <= 0) {
+            qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, no backups need to be deleted";
+        } else {
+            qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, deleting " << backupsToDelete << "backup(s)";
+            for (int i = 0; i < backupsToDelete; ++i) {
+                auto fileInfo = matchingFiles[i].absoluteFilePath();
+                QFile backupFile(fileInfo);
+                if (backupFile.remove()) {
+                    qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName();
+                } else {
+                    qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName();
+                }
+            }
+            qCDebug(domain_server) << "Done removing old backup versions";
+        }
+    } else {
+        qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "."
+                                << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
+                                << " No need to roll backups";
+    }
+}
+
+void DomainContentBackupManager::backup() {
+    auto nowDateTime = QDateTime::currentDateTime();
+    auto nowSeconds = nowDateTime.toSecsSinceEpoch();
+
+    for (BackupRule& rule : _backupRules) {
+        auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds;
+
+        qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup
+                                << "] "
+                                << "compared to backup interval [" << rule.intervalSeconds << "]...";
+
+        if (secondsSinceLastBackup > rule.intervalSeconds) {
+            qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name
+                                    << "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now...";
+
+            bool success;
+            QString path;
+            std::tie(success, path) =  createBackup(AUTOMATIC_BACKUP_PREFIX, rule.extensionFormat);
+            if (!success) {
+                qCWarning(domain_server) << "Failed to create backup for" << rule.name << "at" << path;
+                continue;
+            }
+
+            qDebug() << "Created backup: " << path;
+
+            rule.lastBackupSeconds = nowSeconds;
+
+            removeOldBackupVersions(rule);
+        } else {
+            qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]...";
+        }
+    }
+}
+
+void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, QString fileName) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "consolidateBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(QString, fileName));
+        return;
+    }
+
+    QDir backupDir { _backupDirectory };
+    if (!backupDir.exists()) {
+        qCritical() << "Backup directory does not exist, bailing consolidation of backup";
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    auto filePath = backupDir.absoluteFilePath(fileName);
+
+    auto copyFilePath = QDir::tempPath() + "/" + fileName;
+
+    {
+        QFile copyFile(copyFilePath);
+        copyFile.remove();
+        copyFile.close();
+    }
+    auto copySuccess = QFile::copy(filePath, copyFilePath);
+    if (!copySuccess) {
+        qCritical() << "Failed to create copy of backup.";
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    QuaZip zip(copyFilePath);
+    if (!zip.open(QuaZip::mdAdd)) {
+        qCritical() << "Could not open backup archive:" << filePath;
+        qCritical() << "    ERROR:" << zip.getZipError();
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->consolidateBackup(fileName, zip);
+    }
+
+    zip.close();
+
+    if (zip.getZipError() != UNZ_OK) {
+        qCritical() << "Failed to consolidate backup: " << zip.getZipError();
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    promise->resolve({
+        { "success", true },
+        { "backupFilePath", copyFilePath }
+    });
+}
+
+void DomainContentBackupManager::createManualBackup(MiniPromise::Promise promise, const QString& name) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "createManualBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, name));
+        return;
+    }
+
+
+    QRegExp nameRE { MANUAL_BACKUP_NAME_RE };
+    bool success;
+
+    if (!nameRE.exactMatch(name)) {
+        qDebug() << "Cannot create manual backup with invalid name: " << name;
+        success = false;
+    } else {
+        QString path;
+        std::tie(success, path) = createBackup(MANUAL_BACKUP_PREFIX, name);
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+std::pair<bool, QString> DomainContentBackupManager::createBackup(const QString& prefix, const QString& name) {
+    auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT);
+    auto fileName = prefix + name + "-" + timestamp + ".zip";
+    auto path = _backupDirectory + "/" + fileName;
+    QuaZip zip(path);
+    if (!zip.open(QuaZip::mdAdd)) {
+        qCWarning(domain_server) << "Failed to open zip file at " << path;
+        qCWarning(domain_server) << "    ERROR:" << zip.getZipError();
+        return { false, path };
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->createBackup(fileName, zip);
+    }
+
+    zip.close();
+
+    return { true, path };
+}

domain-server/src/DomainContentBackupManager.h

@@ -0,0 +1,106 @@
+//
+//  DomainContentBackupManager.h
+//  libraries/domain-server/src
+//
+//  Created by Ryan Huffman on 1/01/18.
+//  Adapted from OctreePersistThread
+//  Copyright 2018 High Fidelity, Inc.
+//
+//
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_DomainContentBackupManager_h
+#define hifi_DomainContentBackupManager_h
+
+#include <QString>
+#include <QVector>
+#include <QDateTime>
+
+#include <GenericThread.h>
+
+#include "BackupHandler.h"
+
+#include <shared/MiniPromises.h>
+
+#include <PortableHighResolutionClock.h>
+
+struct BackupItemInfo {
+    BackupItemInfo(QString pId, QString pName, QString pAbsolutePath, QDateTime pCreatedAt, bool pIsManualBackup) :
+        id(pId), name(pName), absolutePath(pAbsolutePath), createdAt(pCreatedAt), isManualBackup(pIsManualBackup) { };
+
+    QString id;
+    QString name;
+    QString absolutePath;
+    QDateTime createdAt;
+    bool isManualBackup;
+};
+
+class DomainContentBackupManager : public GenericThread {
+    Q_OBJECT
+public:
+    class BackupRule {
+    public:
+        QString name;
+        int intervalSeconds;
+        QString extensionFormat;
+        int maxBackupVersions;
+        qint64 lastBackupSeconds;
+    };
+
+    static const std::chrono::seconds DEFAULT_PERSIST_INTERVAL;
+
+    DomainContentBackupManager(const QString& rootBackupDirectory,
+                               const QVariantList& settings,
+                               std::chrono::milliseconds persistInterval = DEFAULT_PERSIST_INTERVAL,
+                               bool debugTimestampNow = false);
+
+    std::vector<BackupItemInfo> getAllBackups();
+    void addBackupHandler(BackupHandlerPointer handler);
+    void aboutToFinish();  /// call this to inform the persist thread that the owner is about to finish to support final persist
+    void replaceData(QByteArray data);
+
+public slots:
+    void getAllBackupsAndStatus(MiniPromise::Promise promise);
+    void createManualBackup(MiniPromise::Promise promise, const QString& name);
+    void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
+    void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
+    void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
+    void consolidateBackup(MiniPromise::Promise promise, QString fileName);
+
+signals:
+    void loadCompleted();
+    void recoveryCompleted();
+
+protected:
+    /// Implements generic processing behavior for this thread.
+    virtual void setup() override;
+    virtual bool process() override;
+    virtual void shutdown() override;
+
+    void backup();
+    void removeOldBackupVersions(const BackupRule& rule);
+    void refreshBackupRules();
+    bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
+    int64_t getMostRecentBackupTimeInSecs(const QString& format);
+    void parseBackupRules(const QVariantList& backupRules);
+
+    std::pair<bool, QString> createBackup(const QString& prefix, const QString& name);
+
+    bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip);
+
+private:
+    const QString _backupDirectory;
+    std::vector<BackupHandlerPointer> _backupHandlers;
+    std::chrono::milliseconds _persistInterval { 0 };
+
+    std::atomic<bool> _isRecovering { false };
+    QString _recoveryFilename { };
+
+    p_high_resolution_clock::time_point _lastCheck;
+    std::vector<BackupRule> _backupRules;
+};
+
+#endif  // hifi_DomainContentBackupManager_h

domain-server/src/DomainGatekeeper.cpp

@@ -435,10 +435,11 @@ SharedNodePointer DomainGatekeeper::processAgentConnectRequest(const NodeConnect
     if (!userPerms.can(NodePermissions::Permission::canConnectPastMaxCapacity) && !isWithinMaxCapacity()) {
         // we can't allow this user to connect because we are at max capacity
         QString redirectOnMaxCapacity;
-        const QVariant* redirectOnMaxCapacityVariant =
-            valueForKeyPath(_server->_settingsManager.getSettingsMap(), MAXIMUM_USER_CAPACITY_REDIRECT_LOCATION);
-        if (redirectOnMaxCapacityVariant && redirectOnMaxCapacityVariant->canConvert<QString>()) {
-            redirectOnMaxCapacity = redirectOnMaxCapacityVariant->toString();
+
+        QVariant redirectOnMaxCapacityVariant =
+            _server->_settingsManager.valueForKeyPath(MAXIMUM_USER_CAPACITY_REDIRECT_LOCATION);
+        if (redirectOnMaxCapacityVariant.canConvert<QString>()) {
+            redirectOnMaxCapacity = redirectOnMaxCapacityVariant.toString();
             qDebug() << "Redirection domain:" << redirectOnMaxCapacity;
         }
 
@@ -610,9 +611,9 @@ bool DomainGatekeeper::verifyUserSignature(const QString& username,
 
 bool DomainGatekeeper::isWithinMaxCapacity() {
     // find out what our maximum capacity is
-    const QVariant* maximumUserCapacityVariant =
-        valueForKeyPath(_server->_settingsManager.getSettingsMap(), MAXIMUM_USER_CAPACITY);
-    unsigned int maximumUserCapacity = maximumUserCapacityVariant ? maximumUserCapacityVariant->toUInt() : 0;
+    QVariant maximumUserCapacityVariant =
+        _server->_settingsManager.valueForKeyPath(MAXIMUM_USER_CAPACITY);
+    unsigned int maximumUserCapacity = maximumUserCapacityVariant.isValid() ? maximumUserCapacityVariant.toUInt() : 0;
 
     if (maximumUserCapacity > 0) {
         unsigned int connectedUsers = _server->countConnectedUsers();

domain-server/src/DomainMetadata.cpp

@@ -84,21 +84,22 @@ void DomainMetadata::descriptorsChanged() {
     // get descriptors
     assert(_metadata[DESCRIPTORS].canConvert<QVariantMap>());
     auto& state = *static_cast<QVariantMap*>(_metadata[DESCRIPTORS].data());
-    auto& settings = static_cast<DomainServer*>(parent())->_settingsManager.getSettingsMap();
-    auto& descriptors = static_cast<DomainServer*>(parent())->_settingsManager.getDescriptorsMap();
+
+    static const QString DESCRIPTORS_GROUP_KEYPATH = "descriptors";
+    auto descriptorsMap = static_cast<DomainServer*>(parent())->_settingsManager.valueForKeyPath(DESCRIPTORS).toMap();
 
     // copy simple descriptors (description/maturity)
-    state[Descriptors::DESCRIPTION] = descriptors[Descriptors::DESCRIPTION]; 
-    state[Descriptors::MATURITY] = descriptors[Descriptors::MATURITY];
+    state[Descriptors::DESCRIPTION] = descriptorsMap[Descriptors::DESCRIPTION];
+    state[Descriptors::MATURITY] = descriptorsMap[Descriptors::MATURITY];
 
     // copy array descriptors (hosts/tags)
-    state[Descriptors::HOSTS] = descriptors[Descriptors::HOSTS].toList();
-    state[Descriptors::TAGS] = descriptors[Descriptors::TAGS].toList();
+    state[Descriptors::HOSTS] = descriptorsMap[Descriptors::HOSTS].toList();
+    state[Descriptors::TAGS] = descriptorsMap[Descriptors::TAGS].toList();
 
     // parse capacity
     static const QString CAPACITY = "security.maximum_user_capacity";
-    const QVariant* capacityVariant = valueForKeyPath(settings, CAPACITY);
-    unsigned int capacity = capacityVariant ? capacityVariant->toUInt() : 0;
+    QVariant capacityVariant = static_cast<DomainServer*>(parent())->_settingsManager.valueForKeyPath(CAPACITY);
+    unsigned int capacity = capacityVariant.isValid() ? capacityVariant.toUInt() : 0;
     state[Descriptors::CAPACITY] = capacity;
 
 #if DEV_BUILD || PR_BUILD

domain-server/src/DomainServer.cpp

@@ -24,6 +24,7 @@
 #include <QTimer>
 #include <QUrlQuery>
 #include <QCommandLineParser>
+#include <QUuid>
 
 #include <AccountManager.h>
 #include <AssetClient.h>
@@ -44,10 +45,20 @@
 #include <Trace.h>
 #include <StatTracker.h>
 
+#include "AssetsBackupHandler.h"
+#include "ContentSettingsBackupHandler.h"
 #include "DomainServerNodeData.h"
+#include "EntitiesBackupHandler.h"
 #include "NodeConnectionData.h"
 
+#include <Gzip.h>
+
+#include <OctreeDataUtils.h>
+
+Q_LOGGING_CATEGORY(domain_server, "hifi.domain_server")
+
 const QString ACCESS_TOKEN_KEY_PATH = "metaverse.access_token";
+const QString DomainServer::REPLACEMENT_FILE_EXTENSION = ".replace";
 
 int const DomainServer::EXIT_CODE_REBOOT = 234923;
 
@@ -64,8 +75,8 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection,
                                               std::initializer_list<QString> optionalData,
                                               bool requireAccessToken) {
 
-    auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-    if (accessTokenVariant == nullptr && requireAccessToken) {
+    auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+    if (!accessTokenVariant.isValid() && requireAccessToken) {
         connection->respond(HTTPConnection::StatusCode400, "User access token has not been set");
         return true;
     }
@@ -101,8 +112,8 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection,
     req.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
     req.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
 
-    if (accessTokenVariant != nullptr) {
-        auto accessTokenHeader = QString("Bearer ") + accessTokenVariant->toString();
+    if (accessTokenVariant.isValid()) {
+        auto accessTokenHeader = QString("Bearer ") + accessTokenVariant.toString();
         req.setRawHeader("Authorization", accessTokenHeader.toLatin1());
     }
 
@@ -280,6 +291,27 @@ DomainServer::DomainServer(int argc, char* argv[]) :
             qDebug() << "Ignoring subnet in whitelist, invalid ip portion: " << subnet;
         }
     }
+
+    if (QDir(getEntitiesDirPath()).mkpath(".")) {
+        qCDebug(domain_server) << "Created entities data directory";
+    }
+    maybeHandleReplacementEntityFile();
+
+
+    static const QString BACKUP_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
+    auto backupRulesVariant = _settingsManager.valueOrDefaultValueForKeyPath(BACKUP_RULES_KEYPATH);
+
+    _contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), backupRulesVariant.toList()));
+
+    connect(_contentManager.get(), &DomainContentBackupManager::started, _contentManager.get(), [this](){
+        _contentManager->addBackupHandler(BackupHandlerPointer(new EntitiesBackupHandler(getEntitiesFilePath(), getEntitiesReplacementFilePath())));
+        _contentManager->addBackupHandler(BackupHandlerPointer(new AssetsBackupHandler(getContentBackupDir())));
+        _contentManager->addBackupHandler(BackupHandlerPointer(new ContentSettingsBackupHandler(_settingsManager)));
+    });
+
+    _contentManager->initialize(true);
+
+    connect(_contentManager.get(), &DomainContentBackupManager::recoveryCompleted, this, &DomainServer::restart);
 }
 
 void DomainServer::parseCommandLine() {
@@ -345,6 +377,11 @@ void DomainServer::parseCommandLine() {
 DomainServer::~DomainServer() {
     qInfo() << "Domain Server is shutting down.";
 
+    if (_contentManager) {
+        _contentManager->aboutToFinish();
+        _contentManager->terminate();
+    }
+
     // cleanup the AssetClient thread
     DependencyManager::destroy<AssetClient>();
     _assetClientThread.quit();
@@ -377,8 +414,8 @@ bool DomainServer::optionallyReadX509KeyAndCertificate() {
     const QString X509_PRIVATE_KEY_OPTION = "key";
     const QString X509_KEY_PASSPHRASE_ENV = "DOMAIN_SERVER_KEY_PASSPHRASE";
 
-    QString certPath = _settingsManager.getSettingsMap().value(X509_CERTIFICATE_OPTION).toString();
-    QString keyPath = _settingsManager.getSettingsMap().value(X509_PRIVATE_KEY_OPTION).toString();
+    QString certPath = _settingsManager.valueForKeyPath(X509_CERTIFICATE_OPTION).toString();
+    QString keyPath = _settingsManager.valueForKeyPath(X509_PRIVATE_KEY_OPTION).toString();
 
     if (!certPath.isEmpty() && !keyPath.isEmpty()) {
         // the user wants to use the following cert and key for HTTPS
@@ -421,8 +458,7 @@ bool DomainServer::optionallySetupOAuth() {
     const QString OAUTH_CLIENT_SECRET_ENV = "DOMAIN_SERVER_CLIENT_SECRET";
     const QString REDIRECT_HOSTNAME_OPTION = "hostname";
 
-    const QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-    _oauthProviderURL = QUrl(settingsMap.value(OAUTH_PROVIDER_URL_OPTION).toString());
+    _oauthProviderURL = QUrl(_settingsManager.valueForKeyPath(OAUTH_PROVIDER_URL_OPTION).toString());
 
     // if we don't have an oauth provider URL then we default to the default node auth url
     if (_oauthProviderURL.isEmpty()) {
@@ -432,9 +468,9 @@ bool DomainServer::optionallySetupOAuth() {
     auto accountManager = DependencyManager::get<AccountManager>();
     accountManager->setAuthURL(_oauthProviderURL);
 
-    _oauthClientID = settingsMap.value(OAUTH_CLIENT_ID_OPTION).toString();
+    _oauthClientID = _settingsManager.valueForKeyPath(OAUTH_CLIENT_ID_OPTION).toString();
     _oauthClientSecret = QProcessEnvironment::systemEnvironment().value(OAUTH_CLIENT_SECRET_ENV);
-    _hostname = settingsMap.value(REDIRECT_HOSTNAME_OPTION).toString();
+    _hostname = _settingsManager.valueForKeyPath(REDIRECT_HOSTNAME_OPTION).toString();
 
     if (!_oauthClientID.isEmpty()) {
         if (_oauthProviderURL.isEmpty()
@@ -459,11 +495,11 @@ static const QString METAVERSE_DOMAIN_ID_KEY_PATH = "metaverse.id";
 
 void DomainServer::getTemporaryName(bool force) {
     // check if we already have a domain ID
-    const QVariant* idValueVariant = valueForKeyPath(_settingsManager.getSettingsMap(), METAVERSE_DOMAIN_ID_KEY_PATH);
+    QVariant idValueVariant = _settingsManager.valueForKeyPath(METAVERSE_DOMAIN_ID_KEY_PATH);
 
     qInfo() << "Requesting temporary domain name";
-    if (idValueVariant) {
-        qDebug() << "A domain ID is already present in domain-server settings:" << idValueVariant->toString();
+    if (idValueVariant.isValid()) {
+        qDebug() << "A domain ID is already present in domain-server settings:" << idValueVariant.toString();
         if (force) {
             qDebug() << "Requesting temporary domain name to replace current ID:" << getID();
         } else {
@@ -503,9 +539,6 @@ void DomainServer::handleTempDomainSuccess(QNetworkReply& requestReply) {
         auto settingsDocument = QJsonDocument::fromJson(newSettingsJSON.toUtf8());
         _settingsManager.recurseJSONObjectAndOverwriteSettings(settingsDocument.object(), DomainSettings);
 
-        // store the new ID and auto networking setting on disk
-        _settingsManager.persistToFile();
-
         // store the new token to the account info
         auto accountManager = DependencyManager::get<AccountManager>();
         accountManager->setTemporaryDomain(id, key);
@@ -607,8 +640,6 @@ void DomainServer::setupNodeListAndAssignments() {
     QVariant localPortValue = _settingsManager.valueOrDefaultValueForKeyPath(CUSTOM_LOCAL_PORT_OPTION);
     int domainServerPort = localPortValue.toInt();
 
-    QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-
     int domainServerDTLSPort = INVALID_PORT;
 
     if (_isUsingDTLS) {
@@ -616,8 +647,9 @@ void DomainServer::setupNodeListAndAssignments() {
 
         const QString CUSTOM_DTLS_PORT_OPTION = "dtls-port";
 
-        if (settingsMap.contains(CUSTOM_DTLS_PORT_OPTION)) {
-            domainServerDTLSPort = (unsigned short) settingsMap.value(CUSTOM_DTLS_PORT_OPTION).toUInt();
+        auto dtlsPortVariant = _settingsManager.valueForKeyPath(CUSTOM_DTLS_PORT_OPTION);
+        if (dtlsPortVariant.isValid()) {
+            domainServerDTLSPort = (unsigned short) dtlsPortVariant.toUInt();
         }
     }
 
@@ -647,9 +679,9 @@ void DomainServer::setupNodeListAndAssignments() {
         nodeList->setSessionUUID(_overridingDomainID);
         isMetaverseDomain = true; // assume metaverse domain
     } else {
-        const QVariant* idValueVariant = valueForKeyPath(settingsMap, METAVERSE_DOMAIN_ID_KEY_PATH);
-        if (idValueVariant) {
-            nodeList->setSessionUUID(idValueVariant->toString());
+        QVariant idValueVariant = _settingsManager.valueForKeyPath(METAVERSE_DOMAIN_ID_KEY_PATH);
+        if (idValueVariant.isValid()) {
+            nodeList->setSessionUUID(idValueVariant.toString());
             isMetaverseDomain = true; // if we have an ID, we'll assume we're a metaverse domain
         } else {
             nodeList->setSessionUUID(QUuid::createUuid()); // Use random UUID
@@ -691,13 +723,18 @@ void DomainServer::setupNodeListAndAssignments() {
     packetReceiver.registerListener(PacketType::ICEServerHeartbeatDenied, this, "processICEServerHeartbeatDenialPacket");
     packetReceiver.registerListener(PacketType::ICEServerHeartbeatACK, this, "processICEServerHeartbeatACK");
 
+    packetReceiver.registerListener(PacketType::OctreeDataFileRequest, this, "processOctreeDataRequestMessage");
+    packetReceiver.registerListener(PacketType::OctreeDataPersist, this, "processOctreeDataPersistMessage");
+
+    packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacementRequest");
+    packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURLRequest");
+
     // set a custom packetVersionMatch as the verify packet operator for the udt::Socket
     nodeList->setPacketFilterOperator(&DomainServer::isPacketVerified);
 
     _assetClientThread.setObjectName("AssetClient Thread");
     auto assetClient = DependencyManager::set<AssetClient>();
     assetClient->moveToThread(&_assetClientThread);
-    QObject::connect(&_assetClientThread, &QThread::started, assetClient.data(), &AssetClient::init);
     _assetClientThread.start();
 
     // add whatever static assignments that have been parsed to the queue
@@ -712,10 +749,10 @@ bool DomainServer::resetAccountManagerAccessToken() {
         QString accessToken = QProcessEnvironment::systemEnvironment().value(ENV_ACCESS_TOKEN_KEY);
 
         if (accessToken.isEmpty()) {
-            const QVariant* accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
+            QVariant accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
 
-            if (accessTokenVariant && accessTokenVariant->canConvert(QMetaType::QString)) {
-                accessToken = accessTokenVariant->toString();
+            if (accessTokenVariant.canConvert(QMetaType::QString)) {
+                accessToken = accessTokenVariant.toString();
             } else {
                 qWarning() << "No access token is present. Some operations that use the metaverse API will fail.";
                 qDebug() << "Set an access token via the web interface, in your user config"
@@ -846,31 +883,26 @@ void DomainServer::updateICEServerAddresses() {
 }
 
 void DomainServer::parseAssignmentConfigs(QSet<Assignment::Type>& excludedTypes) {
-    const QString ASSIGNMENT_CONFIG_REGEX_STRING = "config-([\\d]+)";
-    QRegExp assignmentConfigRegex(ASSIGNMENT_CONFIG_REGEX_STRING);
-
-    const QVariantMap& settingsMap = _settingsManager.getSettingsMap();
+    const QString ASSIGNMENT_CONFIG_PREFIX = "config-";
 
     // scan for assignment config keys
-    QStringList variantMapKeys = settingsMap.keys();
-    int configIndex = variantMapKeys.indexOf(assignmentConfigRegex);
+    for (int i = 0; i < Assignment::AllTypes; ++i) {
+        QVariant assignmentConfigVariant = _settingsManager.valueOrDefaultValueForKeyPath(ASSIGNMENT_CONFIG_PREFIX + QString::number(i));
 
-    while (configIndex != -1) {
-        // figure out which assignment type this matches
-        Assignment::Type assignmentType = (Assignment::Type) assignmentConfigRegex.cap(1).toInt();
+        if (assignmentConfigVariant.isValid()) {
+            // figure out which assignment type this matches
+            Assignment::Type assignmentType = static_cast<Assignment::Type>(i);
 
-        if (assignmentType < Assignment::AllTypes && !excludedTypes.contains(assignmentType)) {
-            QVariant mapValue = settingsMap[variantMapKeys[configIndex]];
-            QVariantList assignmentList = mapValue.toList();
+            if (!excludedTypes.contains(assignmentType)) {
+                QVariantList assignmentList = assignmentConfigVariant.toList();
 
-            if (assignmentType != Assignment::AgentType) {
-                createStaticAssignmentsForType(assignmentType, assignmentList);
+                if (assignmentType != Assignment::AgentType) {
+                    createStaticAssignmentsForType(assignmentType, assignmentList);
+                }
+
+                excludedTypes.insert(assignmentType);
             }
-
-            excludedTypes.insert(assignmentType);
         }
-
-        configIndex = variantMapKeys.indexOf(assignmentConfigRegex, configIndex + 1);
     }
 }
 
@@ -882,10 +914,10 @@ void DomainServer::addStaticAssignmentToAssignmentHash(Assignment* newAssignment
 
 void DomainServer::populateStaticScriptedAssignmentsFromSettings() {
     const QString PERSISTENT_SCRIPTS_KEY_PATH = "scripts.persistent_scripts";
-    const QVariant* persistentScriptsVariant = valueForKeyPath(_settingsManager.getSettingsMap(), PERSISTENT_SCRIPTS_KEY_PATH);
+    QVariant persistentScriptsVariant = _settingsManager.valueOrDefaultValueForKeyPath(PERSISTENT_SCRIPTS_KEY_PATH);
 
-    if (persistentScriptsVariant) {
-        QVariantList persistentScriptsList = persistentScriptsVariant->toList();
+    if (persistentScriptsVariant.isValid()) {
+        QVariantList persistentScriptsList = persistentScriptsVariant.toList();
         foreach(const QVariant& persistentScriptVariant, persistentScriptsList) {
             QVariantMap persistentScript = persistentScriptVariant.toMap();
 
@@ -1695,10 +1727,96 @@ void DomainServer::sendHeartbeatToIceServer() {
     } else {
         qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server.";
         qDebug() << "Waiting for" << _iceServerAddr << "host lookup response";
-
     }
 }
 
+void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message) {
+    qDebug() << "Received octree data persist message";
+    auto data = message->readAll();
+    auto filePath = getEntitiesFilePath();
+
+    QDir dir(getEntitiesDirPath());
+    if (!dir.exists()) {
+        qCDebug(domain_server) << "Creating entities content directory:" << dir.absolutePath();
+        dir.mkpath(".");
+    }
+
+    QFile f(filePath);
+    if (f.open(QIODevice::WriteOnly)) {
+        f.write(data);
+        OctreeUtils::RawEntityData entityData;
+        if (entityData.readOctreeDataInfoFromData(data)) {
+            qCDebug(domain_server) << "Wrote new entities file" << entityData.id << entityData.version;
+        } else {
+            qCDebug(domain_server) << "Failed to read new octree data info";
+        }
+    } else {
+        qCDebug(domain_server) << "Failed to write new entities file:" << filePath;
+    }
+}
+
+QString DomainServer::getContentBackupDir() {
+    return PathUtils::getAppDataFilePath("backups");
+}
+
+QString DomainServer::getEntitiesDirPath() {
+    return PathUtils::getAppDataFilePath("entities");
+}
+
+QString DomainServer::getEntitiesFilePath() {
+    return PathUtils::getAppDataFilePath("entities/models.json.gz");
+}
+
+QString DomainServer::getEntitiesReplacementFilePath() {
+    return getEntitiesFilePath().append(REPLACEMENT_FILE_EXTENSION);
+}
+
+void DomainServer::processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message) {
+    qDebug() << "Got request for octree data from " << message->getSenderSockAddr();
+
+    maybeHandleReplacementEntityFile();
+
+    bool remoteHasExistingData { false };
+    QUuid id;
+    int version;
+    message->readPrimitive(&remoteHasExistingData);
+    if (remoteHasExistingData) {
+        constexpr size_t UUID_SIZE_BYTES = 16;
+        auto idData = message->read(UUID_SIZE_BYTES);
+        id = QUuid::fromRfc4122(idData);
+        message->readPrimitive(&version);
+        qCDebug(domain_server) << "Entity server does have existing data: ID(" << id << ") DataVersion(" << version << ")";
+    } else {
+        qCDebug(domain_server) << "Entity server does not have existing data";
+    }
+    auto entityFilePath = getEntitiesFilePath();
+
+    auto reply = NLPacketList::create(PacketType::OctreeDataFileReply, QByteArray(), true, true);
+    OctreeUtils::RawEntityData data;
+    if (data.readOctreeDataInfoFromFile(entityFilePath)) {
+        if (data.id == id && data.version <= version) {
+            qCDebug(domain_server) << "ES has sufficient octree data, not sending data";
+            reply->writePrimitive(false);
+        } else {
+            qCDebug(domain_server) << "Sending newer octree data to ES: ID(" << data.id << ") DataVersion(" << data.version << ")";
+            QFile file(entityFilePath);
+            if (file.open(QIODevice::ReadOnly)) {
+                reply->writePrimitive(true);
+                reply->write(file.readAll());
+            } else {
+                qCDebug(domain_server) << "Unable to load entity file";
+                reply->writePrimitive(false);
+            }
+        }
+    } else {
+        qCDebug(domain_server) << "Domain server does not have valid octree data";
+        reply->writePrimitive(false);
+    }
+
+    auto nodeList = DependencyManager::get<LimitedNodeList>();
+    nodeList->sendPacketList(std::move(reply), message->getSenderSockAddr());
+}
+
 void DomainServer::processNodeJSONStatsPacket(QSharedPointer<ReceivedMessage> packetList, SharedNodePointer sendingNode) {
     auto nodeData = static_cast<DomainServerNodeData*>(sendingNode->getLinkedData());
     if (nodeData) {
@@ -1808,23 +1926,25 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
     const QString URI_ASSIGNMENT = "/assignment";
     const QString URI_NODES = "/nodes";
     const QString URI_SETTINGS = "/settings";
-    const QString URI_ENTITY_FILE_UPLOAD = "/content/upload";
+    const QString URI_CONTENT_UPLOAD = "/content/upload";
     const QString URI_RESTART = "/restart";
     const QString URI_API_PLACES = "/api/places";
     const QString URI_API_DOMAINS = "/api/domains";
     const QString URI_API_DOMAINS_ID = "/api/domains/";
+    const QString URI_API_BACKUPS = "/api/backups";
+    const QString URI_API_BACKUPS_ID = "/api/backups/";
+    const QString URI_API_BACKUPS_RECOVER = "/api/backups/recover/";
 
     const QString UUID_REGEX_STRING = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
 
     auto nodeList = DependencyManager::get<LimitedNodeList>();
 
-    auto getSetting = [this](QString keyPath, QVariant& value) -> bool {
-        QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-        QVariant* var = valueForKeyPath(settingsMap, keyPath);
-        if (var == nullptr) {
+    auto getSetting = [this](QString keyPath, QVariant value) -> bool {
+
+        value = _settingsManager.valueForKeyPath(keyPath);
+        if (!value.isValid()) {
             return false;
         }
-        value = *var;
         return true;
     };
 
@@ -1892,8 +2012,8 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
     if (connection->requestOperation() == QNetworkAccessManager::GetOperation) {
         const QString URI_WIZARD = "/wizard/";
         const QString WIZARD_COMPLETED_ONCE_KEY_PATH = "wizard.completed_once";
-        const QVariant* wizardCompletedOnce = valueForKeyPath(_settingsManager.getSettingsMap(), WIZARD_COMPLETED_ONCE_KEY_PATH);
-        const bool completedOnce = wizardCompletedOnce && wizardCompletedOnce->toBool();
+        QVariant wizardCompletedOnce = _settingsManager.valueForKeyPath(WIZARD_COMPLETED_ONCE_KEY_PATH);
+        const bool completedOnce = wizardCompletedOnce.isValid() && wizardCompletedOnce.toBool();
 
         if (url.path() != URI_WIZARD && url.path().endsWith('/') && !completedOnce) {
             // First visit, redirect to the wizard
@@ -1997,6 +2117,37 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             // send the response
             connection->respond(HTTPConnection::StatusCode200, nodesDocument.toJson(), qPrintable(JSON_MIME_TYPE));
 
+            return true;
+        } else if (url.path() == URI_API_BACKUPS) {
+            auto deferred = makePromise("getAllBackupsAndStatus");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
+
+                connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->getAllBackupsAndStatus(deferred);
+            return true;
+        } else if (url.path().startsWith(URI_API_BACKUPS_ID)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
+            auto deferred = makePromise("consolidateBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                if (success) {
+                    auto path = result["backupFilePath"].toString();
+                    auto file { std::unique_ptr<QFile>(new QFile(path)) };
+                    if (file->open(QIODevice::ReadOnly)) {
+                        connection->respond(HTTPConnection::StatusCode200, std::move(file));
+                    } else {
+                        qCritical(domain_server) << "Unable to load consolidated backup at:" << path << result;
+                        connection->respond(HTTPConnection::StatusCode500, "Error opening backup");
+                    }
+                } else {
+                    connection->respond(HTTPConnection::StatusCode400);
+                }
+            });
+            _contentManager->consolidateBackup(deferred, id);
+
             return true;
         } else if (url.path() == URI_RESTART) {
             connection->respond(HTTPConnection::StatusCode200);
@@ -2084,17 +2235,52 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             connection->respond(HTTPConnection::StatusCode200);
 
             return true;
-        } else if (url.path() == URI_ENTITY_FILE_UPLOAD) {
+        } else if (url.path() == URI_CONTENT_UPLOAD) {
             // this is an entity file upload, ask the HTTPConnection to parse the data
             QList<FormData> formData = connection->parseFormData();
 
             if (formData.size() > 0 && formData[0].second.size() > 0) {
-                // invoke our method to hand the new octree file off to the octree server
-                QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
-                                          Qt::QueuedConnection, Q_ARG(QByteArray, formData[0].second));
+                auto& firstFormData = formData[0];
+
+                // check the file extension to see what kind of file this is
+                // to make sure we handle this filetype for a content restore
+                auto dispositionValue = QString(firstFormData.first.value("Content-Disposition"));
+                auto formDataFilenameRegex = QRegExp("filename=\"(.+)\"");
+                auto matchIndex = formDataFilenameRegex.indexIn(dispositionValue);
+
+                QString uploadedFilename = "";
+                if (matchIndex != -1) {
+                    uploadedFilename = formDataFilenameRegex.cap(1);
+                }
+
+                if (uploadedFilename.endsWith(".json", Qt::CaseInsensitive)
+                    || uploadedFilename.endsWith(".json.gz", Qt::CaseInsensitive)) {
+                    // invoke our method to hand the new octree file off to the octree server
+                    QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
+                                              Qt::QueuedConnection, Q_ARG(QByteArray, firstFormData.second));
+
+                    // respond with a 200 for success
+                    connection->respond(HTTPConnection::StatusCode200);
+                } else if (uploadedFilename.endsWith(".zip", Qt::CaseInsensitive)) {
+                    auto deferred = makePromise("recoverFromUploadedBackup");
+
+                    deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                        QJsonObject rootJSON;
+                        auto success = result["success"].toBool();
+                        rootJSON["success"] = success;
+                        QJsonDocument docJSON(rootJSON);
+                        connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                            JSON_MIME_TYPE.toUtf8());
+                    });
+
+                    _contentManager->recoverFromUploadedBackup(deferred, firstFormData.second);
+
+                    return true;
+                } else {
+                    // we don't have handling for this filetype, send back a 400 for failure
+                    connection->respond(HTTPConnection::StatusCode400);
+                }
 
-                // respond with a 200 for success
-                connection->respond(HTTPConnection::StatusCode200);
             } else {
                 // respond with a 400 for failure
                 connection->respond(HTTPConnection::StatusCode400);
@@ -2102,16 +2288,50 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
             return true;
 
+        } else if (url.path() == URI_API_BACKUPS) {
+            auto params = connection->parseUrlEncodedForm();
+            auto it = params.find("name");
+            if (it == params.end()) {
+                connection->respond(HTTPConnection::StatusCode400, "Bad request, missing `name`");
+                return true;
+            }
+
+            auto deferred = makePromise("createManualBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->createManualBackup(deferred, it.value());
+
+            return true;
+
         } else if (url.path() == "/domain_settings") {
-            auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-            if (!accessTokenVariant) {
+            auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+            if (!accessTokenVariant.isValid()) {
                 connection->respond(HTTPConnection::StatusCode400);
                 return true;
             }
 
         } else if (url.path() == URI_API_DOMAINS) {
-
             return forwardMetaverseAPIRequest(connection, "/api/v1/domains", "domain", { "label" });
+
+        } else if (url.path().startsWith(URI_API_BACKUPS_RECOVER)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_RECOVER).length());
+            auto deferred = makePromise("recoverFromBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->recoverFromBackup(deferred, id);
+            return true;
         }
     } else if (connection->requestOperation() == QNetworkAccessManager::PutOperation) {
         if (url.path() == URI_API_DOMAINS) {
@@ -2124,8 +2344,8 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             return forwardMetaverseAPIRequest(connection, "/api/v1/domains/" + domainID, "domain",
                                               { }, { "network_address", "network_port", "label" });
         }  else if (url.path() == URI_API_PLACES) {
-            auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-            if (!accessTokenVariant->isValid()) {
+            auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+            if (!accessTokenVariant.isValid()) {
                 connection->respond(HTTPConnection::StatusCode400, "User access token has not been set");
                 return true;
             }
@@ -2173,7 +2393,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
             QUrl url { NetworkingConstants::METAVERSE_SERVER_URL().toString() + "/api/v1/places/" + place_id };
 
-            url.setQuery("access_token=" + accessTokenVariant->toString());
+            url.setQuery("access_token=" + accessTokenVariant.toString());
 
             QNetworkRequest req(url);
             req.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
@@ -2200,7 +2420,22 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
         QRegExp allNodesDeleteRegex(ALL_NODE_DELETE_REGEX_STRING);
         QRegExp nodeDeleteRegex(NODE_DELETE_REGEX_STRING);
 
-        if (nodeDeleteRegex.indexIn(url.path()) != -1) {
+        if (url.path().startsWith(URI_API_BACKUPS_ID)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
+            auto deferred = makePromise("deleteBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->deleteBackup(deferred, id);
+
+            return true;
+
+        } else if (nodeDeleteRegex.indexIn(url.path()) != -1) {
             // this is a request to DELETE one node by UUID
 
             // pull the captured string, if it exists
@@ -2353,10 +2588,11 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
 
     const QByteArray UNAUTHENTICATED_BODY = "You do not have permission to access this domain-server.";
 
-    QVariantMap& settingsMap = _settingsManager.getSettingsMap();
+    QVariant adminUsersVariant = _settingsManager.valueForKeyPath(ADMIN_USERS_CONFIG_KEY);
+    QVariant adminRolesVariant = _settingsManager.valueForKeyPath(ADMIN_ROLES_CONFIG_KEY);
 
     if (!_oauthProviderURL.isEmpty()
-        && (settingsMap.contains(ADMIN_USERS_CONFIG_KEY) || settingsMap.contains(ADMIN_ROLES_CONFIG_KEY))) {
+        && (adminUsersVariant.isValid() || adminRolesVariant.isValid())) {
         QString cookieString = connection->requestHeaders().value(HTTP_COOKIE_HEADER_KEY);
 
         const QString COOKIE_UUID_REGEX_STRING = HIFI_SESSION_COOKIE_KEY + "=([\\d\\w-]+)($|;)";
@@ -2367,7 +2603,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             cookieUUID = cookieUUIDRegex.cap(1);
         }
 
-        if (valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)) {
+        if (_settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).isValid()) {
             qDebug() << "Config file contains web admin settings for OAuth and basic HTTP authentication."
                 << "These cannot be combined - using OAuth for authentication.";
         }
@@ -2377,13 +2613,13 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             DomainServerWebSessionData sessionData = _cookieSessionHash.value(cookieUUID);
             QString profileUsername = sessionData.getUsername();
 
-            if (settingsMap.value(ADMIN_USERS_CONFIG_KEY).toStringList().contains(profileUsername)) {
+            if (_settingsManager.valueForKeyPath(ADMIN_USERS_CONFIG_KEY).toStringList().contains(profileUsername)) {
                 // this is an authenticated user
                 return true;
             }
 
             // loop the roles of this user and see if they are in the admin-roles array
-            QStringList adminRolesArray = settingsMap.value(ADMIN_ROLES_CONFIG_KEY).toStringList();
+            QStringList adminRolesArray = _settingsManager.valueForKeyPath(ADMIN_ROLES_CONFIG_KEY).toStringList();
 
             if (!adminRolesArray.isEmpty()) {
                 foreach(const QString& userRole, sessionData.getRoles()) {
@@ -2428,7 +2664,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             // we don't know about this user yet, so they are not yet authenticated
             return false;
         }
-    } else if (valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)) {
+    } else if (_settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).isValid()) {
         // config file contains username and password combinations for basic auth
         const QByteArray BASIC_AUTH_HEADER_KEY = "Authorization";
 
@@ -2447,10 +2683,10 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
                     QString headerPassword = credentialList[1];
 
                     // we've pulled a username and password - now check if there is a match in our basic auth hash
-                    QString settingsUsername = valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)->toString();
-                    const QVariant* settingsPasswordVariant = valueForKeyPath(settingsMap, BASIC_AUTH_PASSWORD_KEY_PATH);
+                    QString settingsUsername = _settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).toString();
+                    QVariant settingsPasswordVariant = _settingsManager.valueForKeyPath(BASIC_AUTH_PASSWORD_KEY_PATH);
 
-                    QString settingsPassword = settingsPasswordVariant ? settingsPasswordVariant->toString() : "";
+                    QString settingsPassword = settingsPasswordVariant.isValid() ? settingsPasswordVariant.toString() : "";
                     QString hexHeaderPassword = headerPassword.isEmpty() ?
                         "" : QCryptographicHash::hash(headerPassword.toUtf8(), QCryptographicHash::Sha256).toHex();
                         
@@ -2587,13 +2823,14 @@ ReplicationServerInfo serverInformationFromSettings(QVariantMap serverMap, Repli
 }
 
 void DomainServer::updateReplicationNodes(ReplicationServerDirection direction) {
-    auto settings = _settingsManager.getSettingsMap();
 
-    if (settings.contains(BROADCASTING_SETTINGS_KEY)) {
+    auto broadcastSettingsVariant = _settingsManager.valueForKeyPath(BROADCASTING_SETTINGS_KEY);
+
+    if (broadcastSettingsVariant.isValid()) {
         auto nodeList = DependencyManager::get<LimitedNodeList>();
         std::vector<HifiSockAddr> replicationNodesInSettings;
 
-        auto replicationSettings = settings.value(BROADCASTING_SETTINGS_KEY).toMap();
+        auto replicationSettings = broadcastSettingsVariant.toMap();
 
         QString serversKey = direction == Upstream ? "upstream_servers" : "downstream_servers";
         QString replicationDirection = direction == Upstream ? "upstream" : "downstream";
@@ -2669,13 +2906,12 @@ void DomainServer::updateUpstreamNodes() {
 
 void DomainServer::updateReplicatedNodes() {
     // Make sure we have downstream nodes in our list
-    auto settings = _settingsManager.getSettingsMap();
-
     static const QString REPLICATED_USERS_KEY = "users";
     _replicatedUsernames.clear();
-    
-    if (settings.contains(BROADCASTING_SETTINGS_KEY)) {
-        auto replicationSettings = settings.value(BROADCASTING_SETTINGS_KEY).toMap();
+
+    auto replicationVariant = _settingsManager.valueForKeyPath(BROADCASTING_SETTINGS_KEY);
+    if (replicationVariant.isValid()) {
+        auto replicationSettings = replicationVariant.toMap();
         if (replicationSettings.contains(REPLICATED_USERS_KEY)) {
             auto usersSettings = replicationSettings.value(REPLICATED_USERS_KEY).toList();
             for (auto& username : usersSettings) {
@@ -2863,17 +3099,17 @@ void DomainServer::processPathQueryPacket(QSharedPointer<ReceivedMessage> messag
 
         // check out paths in the _configMap to see if we have a match
         auto keypath = QString(PATHS_SETTINGS_KEYPATH_FORMAT).arg(SETTINGS_PATHS_KEY).arg(pathQuery);
-        const QVariant* pathMatch = valueForKeyPath(_settingsManager.getSettingsMap(), keypath);
+        QVariant pathMatch = _settingsManager.valueForKeyPath(keypath);
 
-        if (pathMatch || pathQuery == INDEX_PATH) {
+        if (pathMatch.isValid() || pathQuery == INDEX_PATH) {
             // we got a match, respond with the resulting viewpoint
             auto nodeList = DependencyManager::get<LimitedNodeList>();
 
             QString responseViewpoint;
 
             // if we didn't match the path BUT this is for the index path then send back our default
-            if (pathMatch) {
-                responseViewpoint = pathMatch->toMap()[PATH_VIEWPOINT_KEY].toString();
+            if (pathMatch.isValid()) {
+                responseViewpoint = pathMatch.toMap()[PATH_VIEWPOINT_KEY].toString();
             } else {
                 const QString DEFAULT_INDEX_PATH = "/0,0,0/0,0,0,1";
                 responseViewpoint = DEFAULT_INDEX_PATH;
@@ -3105,19 +3341,101 @@ void DomainServer::setupGroupCacheRefresh() {
     }
 }
 
-void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
-    // enumerate the nodes and find any octree type servers with active sockets
+void DomainServer::maybeHandleReplacementEntityFile() {
+    const auto replacementFilePath = getEntitiesReplacementFilePath();
+    OctreeUtils::RawEntityData data;
+    if (!data.readOctreeDataInfoFromFile(replacementFilePath)) {
+        qCWarning(domain_server) << "Replacement file could not be read, it either doesn't exist or is invalid.";
+    } else {
+        qCDebug(domain_server) << "Replacing existing entity date with replacement file";
 
-    auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
-    limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
-        return node->getType() == NodeType::EntityServer && node->getActiveSocket();
-    }, [&octreeFile, limitedNodeList](const SharedNodePointer& octreeNode) {
-        // setup a packet to send to this octree server with the new octree file data
-        auto octreeFilePacketList = NLPacketList::create(PacketType::OctreeFileReplacement, QByteArray(), true, true);
-        octreeFilePacketList->write(octreeFile);
+        QFile replacementFile(replacementFilePath);
+        if (!replacementFile.remove()) {
+            // If we can't remove the replacement file, we are at risk of getting into a state where
+            // we continually replace the primary entity file with the replacement entity file.
+            qCWarning(domain_server) << "Unable to remove replacement file, bailing";
+        } else {
+            data.resetIdAndVersion();
+            auto gzippedData = data.toGzippedByteArray();
 
-        qDebug() << "Sending an octree file replacement of" << octreeFile.size() << "bytes to" << octreeNode;
-
-        limitedNodeList->sendPacketList(std::move(octreeFilePacketList), *octreeNode);
-    });
+            QFile currentFile(getEntitiesFilePath());
+            if (!currentFile.open(QIODevice::WriteOnly)) {
+                qCWarning(domain_server)
+                    << "Failed to update entities data file with replacement file, unable to open entities file for writing";
+            } else {
+                currentFile.write(gzippedData);
+            }
+        }
+    }
+}
+
+void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
+    //Assume we have compressed data
+    auto compressedOctree = octreeFile;
+    QByteArray jsonOctree;
+
+    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
+    if (!wasCompressed) {
+        // the source was not compressed, assume we were sent regular JSON data
+        jsonOctree = compressedOctree;
+    }
+
+    OctreeUtils::RawEntityData data;
+    if (data.readOctreeDataInfoFromData(jsonOctree)) {
+        data.resetIdAndVersion();
+
+        gzip(data.toByteArray(), compressedOctree);
+
+        // write the compressed octree data to a special file
+        auto replacementFilePath = getEntitiesReplacementFilePath();
+        QFile replacementFile(replacementFilePath);
+        if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
+            // we've now written our replacement file, time to take the server down so it can
+            // process it when it comes back up
+            qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
+
+            QMetaObject::invokeMethod(this, "restart", Qt::QueuedConnection);
+        } else {
+            qWarning() << "Could not write replacement octree data to file - refusing to process";
+        }
+    } else {
+        qDebug() << "Received replacement octree file that is invalid - refusing to process";
+    }
+}
+
+void DomainServer::handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message) {
+    qInfo() << "Received request to replace content from a url";
+    auto node = DependencyManager::get<LimitedNodeList>()->findNodeWithAddr(message->getSenderSockAddr());
+    if (node) {
+        qDebug() << "Found node: " << node->getCanReplaceContent();
+    }
+    if (node->getCanReplaceContent()) {
+        // Convert message data into our URL
+        QString url(message->getMessage());
+        QUrl modelsURL = QUrl(url, QUrl::StrictMode);
+        QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+        QNetworkRequest request(modelsURL);
+        QNetworkReply* reply = networkAccessManager.get(request);
+
+        qDebug() << "Downloading JSON from: " << modelsURL;
+
+        connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
+            QNetworkReply::NetworkError networkError = reply->error();
+            if (networkError == QNetworkReply::NoError) {
+                handleOctreeFileReplacement(reply->readAll());
+            } else {
+                qDebug() << "Error downloading JSON from specified file: " << modelsURL;
+            }
+        });
+    }
+}
+
+
+
+
+void DomainServer::handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message) {
+    auto node = DependencyManager::get<NodeList>()->nodeWithUUID(message->getSourceID());
+    if (node->getCanReplaceContent()) {
+        handleOctreeFileReplacement(message->readAll());
+    }
 }

domain-server/src/DomainServer.h

@@ -26,15 +26,20 @@
 #include <HTTPSConnection.h>
 #include <LimitedNodeList.h>
 
-#include "BackupSupervisor.h"
+#include "AssetsBackupHandler.h"
 #include "DomainGatekeeper.h"
 #include "DomainMetadata.h"
 #include "DomainServerSettingsManager.h"
 #include "DomainServerWebSessionData.h"
 #include "WalletTransaction.h"
+#include "DomainContentBackupManager.h"
 
 #include "PendingAssignedNodeData.h"
 
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(domain_server)
+
 typedef QSharedPointer<Assignment> SharedAssignmentPointer;
 typedef QMultiHash<QUuid, WalletTransaction*> TransactionHash;
 
@@ -65,6 +70,8 @@ public:
     bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
     bool handleHTTPSRequest(HTTPSConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
 
+    static const QString REPLACEMENT_FILE_EXTENSION;
+
 public slots:
     /// Called by NodeList to inform us a node has been added
     void nodeAdded(SharedNodePointer node);
@@ -84,6 +91,13 @@ private slots:
     void processICEServerHeartbeatDenialPacket(QSharedPointer<ReceivedMessage> message);
     void processICEServerHeartbeatACK(QSharedPointer<ReceivedMessage> message);
 
+    void handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeFileReplacement(QByteArray octreeFile);
+
+    void processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message);
+    void processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message);
+
     void setupPendingAssignmentCredits();
     void sendPendingTransactionsToServer();
 
@@ -91,8 +105,7 @@ private slots:
     void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); }
     void sendHeartbeatToIceServer();
 
-    void handleConnectedNode(SharedNodePointer newNode);
-
+    void handleConnectedNode(SharedNodePointer newNode); 
     void handleTempDomainSuccess(QNetworkReply& requestReply);
     void handleTempDomainError(QNetworkReply& requestReply);
 
@@ -109,8 +122,6 @@ private slots:
     void handleSuccessfulICEServerAddressUpdate(QNetworkReply& requestReply);
     void handleFailedICEServerAddressUpdate(QNetworkReply& requestReply);
 
-    void handleOctreeFileReplacement(QByteArray octreeFile);
-
     void updateReplicatedNodes();
     void updateDownstreamNodes();
     void updateUpstreamNodes();
@@ -127,6 +138,13 @@ private:
     const QUuid& getID();
     void parseCommandLine();
 
+    QString getContentBackupDir();
+    QString getEntitiesDirPath();
+    QString getEntitiesFilePath();
+    QString getEntitiesReplacementFilePath();
+
+    void maybeHandleReplacementEntityFile();
+
     void setupNodeListAndAssignments();
     bool optionallySetupOAuth();
     bool optionallyReadX509KeyAndCertificate();
@@ -252,6 +270,8 @@ private:
     bool _sendICEServerAddressToMetaverseAPIInProgress { false };
     bool _sendICEServerAddressToMetaverseAPIRedo { false };
 
+    std::unique_ptr<DomainContentBackupManager> _contentManager { nullptr };
+
     QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
 
     QThread _assetClientThread;

domain-server/src/DomainServerSettingsManager.cpp

@@ -19,6 +19,7 @@
 #include <QtCore/QJsonArray>
 #include <QtCore/QJsonObject>
 #include <QtCore/QStandardPaths>
+#include <QtCore/QThread>
 #include <QtCore/QUrl>
 #include <QtCore/QUrlQuery>
 
@@ -32,9 +33,13 @@
 #include <SettingHelpers.h>
 #include <AvatarData.h> //for KillAvatarReason
 #include <FingerprintUtils.h>
+
 #include "DomainServerNodeData.h"
 
 const QString SETTINGS_DESCRIPTION_RELATIVE_PATH = "/resources/describe-settings.json";
+const QString SETTINGS_PATH = "/settings";
+const QString SETTINGS_PATH_JSON = SETTINGS_PATH + ".json";
+const QString CONTENT_SETTINGS_PATH_JSON = "/content-settings.json";
 
 const QString DESCRIPTION_SETTINGS_KEY = "settings";
 const QString SETTING_DEFAULT_KEY = "default";
@@ -187,6 +192,9 @@ void DomainServerSettingsManager::processSettingsRequestPacket(QSharedPointer<Re
 }
 
 void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList) {
+    // since we're called from the DomainServerSettingsManager constructor, we don't take a write lock here
+    // even though we change the underlying config map
+
     _argumentList = argumentList;
 
     _configMap.loadConfig(_argumentList);
@@ -390,6 +398,7 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
             _standardAgentPermissions[NodePermissions::standardNameLocalhost]->set(NodePermissions::Permission::canRezTemporaryCertifiedEntities);
             packPermissions();
         }
+
         if (oldVersion < 2.0) {
             const QString WIZARD_COMPLETED_ONCE = "wizard.completed_once";
 
@@ -397,6 +406,7 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
 
             *wizardCompletedOnce = QVariant(true);
         }
+
         if (oldVersion < 2.1) {
             // convert old avatar scale settings into avatar height.
 
@@ -418,6 +428,21 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
             }
         }
 
+        if (oldVersion < 2.2) {
+            // migrate entity server rolling backup intervals to new location for automatic content archive intervals
+
+            const QString ENTITY_SERVER_BACKUPS_KEYPATH = "entity_server_settings.backups";
+            const QString AUTO_CONTENT_ARCHIVES_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
+
+            QVariant* previousBackupsVariant = _configMap.valueForKeyPath(ENTITY_SERVER_BACKUPS_KEYPATH);
+
+            if (previousBackupsVariant) {
+                auto migratedBackupsVariant = _configMap.valueForKeyPath(AUTO_CONTENT_ARCHIVES_RULES_KEYPATH, true);
+                *migratedBackupsVariant = *previousBackupsVariant;
+            }
+        }
+
+
         // write the current description version to our settings
         *versionVariant = _descriptionVersion;
 
@@ -428,17 +453,6 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
     unpackPermissions();
 }
 
-QVariantMap& DomainServerSettingsManager::getDescriptorsMap() {
-    static const QString DESCRIPTORS{ "descriptors" };
-
-    auto& settingsMap = getSettingsMap();
-    if (!getSettingsMap().contains(DESCRIPTORS)) {
-        settingsMap.insert(DESCRIPTORS, QVariantMap());
-    }
-
-    return *static_cast<QVariantMap*>(getSettingsMap()[DESCRIPTORS].data());
-}
-
 void DomainServerSettingsManager::initializeGroupPermissions(NodePermissionsMap& permissionsRows,
                                                              QString groupName, NodePermissionsPointer perms) {
     // this is called when someone has used the domain-settings webpage to add a group.  They type the group's name
@@ -467,6 +481,9 @@ void DomainServerSettingsManager::initializeGroupPermissions(NodePermissionsMap&
 void DomainServerSettingsManager::packPermissionsForMap(QString mapName,
                                                         NodePermissionsMap& permissionsRows,
                                                         QString keyPath) {
+    // grab a write lock on the settings mutex since we're about to change the config map
+    QWriteLocker locker(&_settingsLock);
+
     // find (or create) the "security" section of the settings map
     QVariant* security = _configMap.valueForKeyPath("security", true);
     if (!security->canConvert(QMetaType::QVariantMap)) {
@@ -556,15 +573,20 @@ bool DomainServerSettingsManager::unpackPermissionsForKeypath(const QString& key
 
     mapPointer->clear();
 
-    QVariant* permissions = _configMap.valueForKeyPath(keyPath, true);
-    if (!permissions->canConvert(QMetaType::QVariantList)) {
+    QVariant permissions = valueOrDefaultValueForKeyPath(keyPath);
+
+    if (!permissions.isValid()) {
+        // we don't have a permissions object to unpack for this keypath, bail
+        return false;
+    }
+
+    if (!permissions.canConvert(QMetaType::QVariantList)) {
         qDebug() << "Failed to extract permissions for key path" << keyPath << "from settings.";
-        (*permissions) = QVariantList();
     }
 
     bool needPack = false;
 
-    QList<QVariant> permissionsList = permissions->toList();
+    QList<QVariant> permissionsList = permissions.toList();
     foreach (QVariant permsHash, permissionsList) {
         NodePermissionsPointer perms { new NodePermissions(permsHash.toMap()) };
         QString id = perms->getID();
@@ -591,6 +613,11 @@ bool DomainServerSettingsManager::unpackPermissionsForKeypath(const QString& key
 void DomainServerSettingsManager::unpackPermissions() {
     // transfer details from _configMap to _agentPermissions
 
+    // NOTE: Defaults for standard permissions (anonymous, friends, localhost, logged-in) used
+    // to be set here and then immediately persisted to the config JSON file.
+    // They have since been moved to describe-settings.json as the default value for AGENT_STANDARD_PERMISSIONS_KEYPATH.
+    // In order to change the default standard permissions you must change the default value in describe-settings.json.
+
     bool needPack = false;
 
     needPack |= unpackPermissionsForKeypath(AGENT_STANDARD_PERMISSIONS_KEYPATH, &_standardAgentPermissions);
@@ -650,57 +677,39 @@ void DomainServerSettingsManager::unpackPermissions() {
             }
     });
 
-    // if any of the standard names are missing, add them
-    foreach(const QString& standardName, NodePermissions::standardNames) {
-        NodePermissionsKey standardKey { standardName, 0 };
-        if (!_standardAgentPermissions.contains(standardKey)) {
-            // we don't have permissions for one of the standard groups, so we'll add them now
-            NodePermissionsPointer perms { new NodePermissions(standardKey) };
-
-            if (standardKey == NodePermissions::standardNameLocalhost) {
-                // the localhost user is granted all permissions by default
-                perms->setAll(true);
-            } else {
-                // anonymous, logged in, and friend users get connect permissions by default
-                perms->set(NodePermissions::Permission::canConnectToDomain);
-                perms->set(NodePermissions::Permission::canRezTemporaryCertifiedEntities);
-            }
-
-            // add the permissions to the standard map
-            _standardAgentPermissions[standardKey] = perms;
-
-            // this will require a packing of permissions
-            needPack = true;
-        }
-    }
-
     needPack |= ensurePermissionsForGroupRanks();
 
     if (needPack) {
         packPermissions();
     }
 
-    #ifdef WANT_DEBUG
+#ifdef WANT_DEBUG
     qDebug() << "--------------- permissions ---------------------";
-    QList<QHash<NodePermissionsKey, NodePermissionsPointer>> permissionsSets;
-    permissionsSets << _standardAgentPermissions.get() << _agentPermissions.get()
-                    << _groupPermissions.get() << _groupForbiddens.get()
-                    << _ipPermissions.get() << _macPermissions.get()
-                    << _machineFingerprintPermissions.get();
+    std::array<NodePermissionsMap*, 7> permissionsSets {{
+        &_standardAgentPermissions, &_agentPermissions,
+        &_groupPermissions, &_groupForbiddens,
+        &_ipPermissions, &_macPermissions,
+        &_machineFingerprintPermissions
+    }};
 
     foreach (auto permissionSet, permissionsSets) {
-        QHashIterator<NodePermissionsKey, NodePermissionsPointer> i(permissionSet);
-        while (i.hasNext()) {
-            i.next();
-            NodePermissionsPointer perms = i.value();
+        auto& permissionKeyMap = permissionSet->get();
+        auto it = permissionKeyMap.begin();
+
+        while (it != permissionKeyMap.end()) {
+
+            NodePermissionsPointer perms = it->second;
             if (perms->isGroup()) {
-                qDebug() << i.key() << perms->getGroupID() << perms;
+                qDebug() << it->first << perms->getGroupID() << perms;
             } else {
-                qDebug() << i.key() << perms;
+                qDebug() << it->first << perms;
             }
+
+            ++it;
         }
     }
-    #endif
+#endif
+
 }
 
 bool DomainServerSettingsManager::ensurePermissionsForGroupRanks() {
@@ -1048,12 +1057,22 @@ NodePermissions DomainServerSettingsManager::getForbiddensForGroup(const QUuid&
     return getForbiddensForGroup(groupKey.first, groupKey.second);
 }
 
+QVariant DomainServerSettingsManager::valueForKeyPath(const QString& keyPath) {
+    QReadLocker locker(&_settingsLock);
+    auto foundValue = _configMap.valueForKeyPath(keyPath);
+    return foundValue ? *foundValue : QVariant();
+}
+
 QVariant DomainServerSettingsManager::valueOrDefaultValueForKeyPath(const QString& keyPath) {
+    QReadLocker locker(&_settingsLock);
     const QVariant* foundValue = _configMap.valueForKeyPath(keyPath);
 
     if (foundValue) {
         return *foundValue;
     } else {
+        // we don't need the settings lock anymore since we're done reading from the config map
+        locker.unlock();
+
         int dotIndex = keyPath.indexOf('.');
 
         QString groupKey = keyPath.mid(0, dotIndex);
@@ -1092,9 +1111,6 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
             // we recurse one level deep below each group for the appropriate setting
             bool restartRequired = recurseJSONObjectAndOverwriteSettings(postedObject, endpointType);
 
-            // store whatever the current _settingsMap is to file
-            persistToFile();
-
             // return success to the caller
             QString jsonSuccess = "{\"status\": \"success\"}";
             connection->respond(HTTPConnection::StatusCode200, jsonSuccess.toUtf8(), "application/json");
@@ -1152,17 +1168,20 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
 
             QJsonObject rootObject;
 
-            bool forDomainSettings = (url.path() == SETTINGS_PATH_JSON);
-            bool forContentSettings = (url.path() == CONTENT_SETTINGS_PATH_JSON);;
+            DomainSettingsInclusion domainSettingsInclusion = (url.path() == SETTINGS_PATH_JSON)
+                ? IncludeDomainSettings : NoDomainSettings;
+            ContentSettingsInclusion contentSettingsInclusion = (url.path() == CONTENT_SETTINGS_PATH_JSON)
+                ? IncludeContentSettings : NoContentSettings;
 
-            rootObject[SETTINGS_RESPONSE_DESCRIPTION_KEY] = forDomainSettings
+            rootObject[SETTINGS_RESPONSE_DESCRIPTION_KEY] = (url.path() == SETTINGS_PATH_JSON)
                 ? _domainSettingsDescription : _contentSettingsDescription;
 
             // grab a domain settings object for all types, filtered for the right class of settings
             // and exclude default values
-            rootObject[SETTINGS_RESPONSE_VALUE_KEY] = settingsResponseObjectForType("", true,
-                                                                                    forDomainSettings, forContentSettings,
-                                                                                    true);
+            rootObject[SETTINGS_RESPONSE_VALUE_KEY] = settingsResponseObjectForType("", Authenticated,
+                                                                                    domainSettingsInclusion,
+                                                                                    contentSettingsInclusion,
+                                                                                    IncludeDefaultSettings);
 
             connection->respond(HTTPConnection::StatusCode200, QJsonDocument(rootObject).toJson(), "application/json");
 
@@ -1174,7 +1193,8 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
         } else if (url.path() == SETTINGS_BACKUP_PATH) {
             // grab the settings backup as an authenticated user
             // for the domain settings type only, excluding hidden and default values
-            auto currentDomainSettingsJSON = settingsResponseObjectForType("", true, true, false, false, true);
+            auto currentDomainSettingsJSON = settingsResponseObjectForType("", Authenticated, IncludeDomainSettings,
+                                                                           NoContentSettings, NoDefaultSettings, ForBackup);
 
             // setup headers that tell the client to download the file wth a special name
             Headers downloadHeaders;
@@ -1196,6 +1216,10 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
 }
 
 bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType) {
+
+    // grab a write lock since we're about to change the settings map
+    QWriteLocker locker(&_settingsLock);
+
     QJsonArray* filteredDescriptionArray = settingsType == DomainSettings
         ? &_domainSettingsDescription : &_contentSettingsDescription;
 
@@ -1277,6 +1301,9 @@ bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settings
                     }
                 } else {
                     // we have a value to restore, use update setting to set it
+                    // but clear the existing value first so that no merging between the restored settings
+                    // and existing settings occurs
+                    variantValue->clear();
 
                     // we might need to re-grab config group map in case it didn't exist when we looked for it before
                     // but was created by the call to valueForKeyPath before
@@ -1310,18 +1337,24 @@ bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settings
     } else {
         // restore completed, persist the new settings
         qDebug() << "Restore completed, persisting restored settings to file";
+
+        // let go of the write lock since we're done making changes to the config map
+        locker.unlock();
+
         persistToFile();
         return true;
     }
 }
 
-QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QString& typeValue, bool isAuthenticated,
-                                                                       bool includeDomainSettings,
-                                                                       bool includeContentSettings,
-                                                                       bool includeDefaults, bool isForBackup) {
+QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QString& typeValue,
+                                                                       SettingsRequestAuthentication authentication,
+                                                                       DomainSettingsInclusion domainSettingsInclusion,
+                                                                       ContentSettingsInclusion contentSettingsInclusion,
+                                                                       DefaultSettingsInclusion defaultSettingsInclusion,
+                                                                       SettingsBackupFlag settingsBackupFlag) {
     QJsonObject responseObject;
 
-    if (!typeValue.isEmpty() || isAuthenticated) {
+    if (!typeValue.isEmpty() || authentication == Authenticated) {
         // convert the string type value to a QJsonValue
         QJsonValue queryType = typeValue.isEmpty() ? QJsonValue() : QJsonValue(typeValue.toInt());
 
@@ -1329,9 +1362,10 @@ QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QSt
 
         // only enumerate the requested settings type (domain setting or content setting)
         QJsonArray* filteredDescriptionArray = &_descriptionArray;
-        if (includeDomainSettings && !includeContentSettings) {
+
+        if (domainSettingsInclusion == IncludeDomainSettings && contentSettingsInclusion != IncludeContentSettings) {
             filteredDescriptionArray = &_domainSettingsDescription;
-        } else if (includeContentSettings && !includeDomainSettings) {
+        } else if (contentSettingsInclusion == IncludeContentSettings && domainSettingsInclusion != IncludeDomainSettings) {
             filteredDescriptionArray = &_contentSettingsDescription;
         }
 
@@ -1354,35 +1388,35 @@ QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QSt
                 bool includedInBackups = !settingObject.contains(DESCRIPTION_BACKUP_FLAG_KEY)
                     || settingObject[DESCRIPTION_BACKUP_FLAG_KEY].toBool();
 
-                if (!settingObject[VALUE_HIDDEN_FLAG_KEY].toBool() && (!isForBackup || includedInBackups)) {
+                if (!settingObject[VALUE_HIDDEN_FLAG_KEY].toBool() && (settingsBackupFlag != ForBackup || includedInBackups)) {
                     QJsonArray affectedTypesArray = settingObject[AFFECTED_TYPES_JSON_KEY].toArray();
                     if (affectedTypesArray.isEmpty()) {
                         affectedTypesArray = groupObject[AFFECTED_TYPES_JSON_KEY].toArray();
                     }
 
                     if (affectedTypesArray.contains(queryType) ||
-                        (queryType.isNull() && isAuthenticated)) {
+                        (queryType.isNull() && authentication == Authenticated)) {
                         QString settingName = settingObject[DESCRIPTION_NAME_KEY].toString();
 
                         // we need to check if the settings map has a value for this setting
                         QVariant variantValue;
 
                         if (!groupKey.isEmpty()) {
-                             QVariant settingsMapGroupValue = _configMap.value(groupKey);
+                             QVariant settingsMapGroupValue = valueForKeyPath(groupKey);
 
                             if (!settingsMapGroupValue.isNull()) {
                                 variantValue = settingsMapGroupValue.toMap().value(settingName);
                             }
                         } else {
-                            variantValue = _configMap.value(settingName);
+                            variantValue = valueForKeyPath(settingName);
                         }
 
                         // final check for inclusion
                         // either we include default values or we don't but this isn't a default value
-                        if (includeDefaults || !variantValue.isNull()) {
+                        if ((defaultSettingsInclusion == IncludeDefaultSettings) || variantValue.isValid()) {
                             QJsonValue result;
 
-                            if (variantValue.isNull()) {
+                            if (!variantValue.isValid()) {
                                 // no value for this setting, pass the default
                                 if (settingObject.contains(SETTING_DEFAULT_KEY)) {
                                     result = settingObject[SETTING_DEFAULT_KEY];
@@ -1521,6 +1555,10 @@ QJsonObject DomainServerSettingsManager::settingDescriptionFromGroup(const QJson
 
 bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject,
                                                                         SettingsType settingsType) {
+
+    // take a write lock since we're about to overwrite settings in the config map
+    QWriteLocker locker(&_settingsLock);
+
     static const QString SECURITY_ROOT_KEY = "security";
     static const QString AC_SUBNET_WHITELIST_KEY = "ac_subnet_whitelist";
     static const QString BROADCASTING_KEY = "broadcasting";
@@ -1618,6 +1656,12 @@ bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJ
         }
     }
 
+    // we're done making changes to the config map, let go of our read lock
+    locker.unlock();
+
+    // store whatever the current config map is to file
+    persistToFile();
+
     return needRestart;
 }
 
@@ -1644,6 +1688,9 @@ bool permissionVariantLessThan(const QVariant &v1, const QVariant &v2) {
 }
 
 void DomainServerSettingsManager::sortPermissions() {
+    // take a write lock since we're about to change the config map data
+    QWriteLocker locker(&_settingsLock);
+
     // sort the permission-names
     QVariant* standardPermissions = _configMap.valueForKeyPath(AGENT_STANDARD_PERMISSIONS_KEYPATH);
     if (standardPermissions && standardPermissions->canConvert(QMetaType::QVariantList)) {
@@ -1680,11 +1727,15 @@ void DomainServerSettingsManager::persistToFile() {
     QFile settingsFile(_configMap.getUserConfigFilename());
 
     if (settingsFile.open(QIODevice::WriteOnly)) {
+        // take a read lock so we can grab the config and write it to file
+        QReadLocker locker(&_settingsLock);
         settingsFile.write(QJsonDocument::fromVariant(_configMap.getConfig()).toJson());
     } else {
         qCritical("Could not write to JSON settings file. Unable to persist settings.");
 
         // failed to write, reload whatever the current config state is
+        // with a write lock since we're about to overwrite the config map
+        QWriteLocker locker(&_settingsLock);
         _configMap.loadConfig(_argumentList);
     }
 }

domain-server/src/DomainServerSettingsManager.h

@@ -27,9 +27,6 @@
 
 const QString SETTINGS_PATHS_KEY = "paths";
 
-const QString SETTINGS_PATH = "/settings";
-const QString SETTINGS_PATH_JSON = SETTINGS_PATH + ".json";
-const QString CONTENT_SETTINGS_PATH_JSON = "/content-settings.json";
 const QString AGENT_STANDARD_PERMISSIONS_KEYPATH = "security.standard_permissions";
 const QString AGENT_PERMISSIONS_KEYPATH = "security.permissions";
 const QString IP_PERMISSIONS_KEYPATH = "security.ip_permissions";
@@ -37,6 +34,7 @@ const QString MAC_PERMISSIONS_KEYPATH = "security.mac_permissions";
 const QString MACHINE_FINGERPRINT_PERMISSIONS_KEYPATH = "security.machine_fingerprint_permissions";
 const QString GROUP_PERMISSIONS_KEYPATH = "security.group_permissions";
 const QString GROUP_FORBIDDENS_KEYPATH = "security.group_forbiddens";
+const QString AUTOMATIC_CONTENT_ARCHIVES_GROUP = "automatic_content_archives";
 
 using GroupByUUIDKey = QPair<QUuid, QUuid>; // groupID, rankID
 
@@ -52,11 +50,12 @@ public:
     bool handleAuthenticatedHTTPRequest(HTTPConnection* connection, const QUrl& url);
 
     void setupConfigMap(const QStringList& argumentList);
+
+    // each of the three methods in this group takes a read lock of _settingsLock
+    // and cannot be called when the a write lock is held by the same thread
     QVariant valueOrDefaultValueForKeyPath(const QString& keyPath);
-
-    QVariantMap& getSettingsMap() { return _configMap.getConfig(); }
-
-    QVariantMap& getDescriptorsMap();
+    QVariant valueForKeyPath(const QString& keyPath);
+    bool containsKeyPath(const QString& keyPath) { return valueForKeyPath(keyPath).isValid(); }
 
     // these give access to anonymous/localhost/logged-in settings from the domain-server settings page
     bool haveStandardPermissionsForName(const QString& name) const { return _standardAgentPermissions.contains(name, 0); }
@@ -111,6 +110,24 @@ public:
 
     void debugDumpGroupsState();
 
+    enum SettingsRequestAuthentication { NotAuthenticated, Authenticated };
+    enum DomainSettingsInclusion { NoDomainSettings, IncludeDomainSettings };
+    enum ContentSettingsInclusion { NoContentSettings, IncludeContentSettings };
+    enum DefaultSettingsInclusion { NoDefaultSettings, IncludeDefaultSettings };
+    enum SettingsBackupFlag { NotForBackup, ForBackup };
+
+    /// thread safe method to retrieve a JSON representation of settings
+    QJsonObject settingsResponseObjectForType(const QString& typeValue,
+                                              SettingsRequestAuthentication authentication = NotAuthenticated,
+                                              DomainSettingsInclusion domainSettingsInclusion = IncludeDomainSettings,
+                                              ContentSettingsInclusion contentSettingsInclusion = IncludeContentSettings,
+                                              DefaultSettingsInclusion defaultSettingsInclusion = IncludeDefaultSettings,
+                                              SettingsBackupFlag settingsBackupFlag = NotForBackup);
+    /// thread safe method to restore settings from a JSON object
+    Q_INVOKABLE bool restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType);
+
+    bool recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject, SettingsType settingsType);
+
 signals:
     void updateNodePermissions();
     void settingsUpdated();
@@ -130,21 +147,17 @@ private:
     QStringList _argumentList;
 
     QJsonArray filteredDescriptionArray(bool isContentSettings);
-    QJsonObject settingsResponseObjectForType(const QString& typeValue, bool isAuthenticated = false,
-                                              bool includeDomainSettings = true, bool includeContentSettings = true,
-                                              bool includeDefaults = true, bool isForBackup = false);
-    bool recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject, SettingsType settingsType);
-
     void updateSetting(const QString& key, const QJsonValue& newValue, QVariantMap& settingMap,
                        const QJsonObject& settingDescription);
     QJsonObject settingDescriptionFromGroup(const QJsonObject& groupObject, const QString& settingName);
     void sortPermissions();
+
+    // you cannot be holding the _settingsLock when persisting to file from the same thread
+    // since it may take either a read lock or write lock and recursive locking doesn't allow a change in type
     void persistToFile();
 
     void splitSettingsDescription();
 
-    bool restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType);
-
     double _descriptionVersion;
 
     QJsonArray _descriptionArray;
@@ -152,10 +165,10 @@ private:
     QJsonArray _contentSettingsDescription;
     QJsonObject _settingsMenuGroups;
 
+    // any method that calls valueForKeyPath on this _configMap must get a write lock it keeps until it
+    // is done with the returned QVariant*
     HifiConfigVariantMap _configMap;
 
-    friend class DomainServer;
-
     // these cause calls to metaverse's group api
     void apiGetGroupID(const QString& groupName);
     void apiGetGroupRanks(const QUuid& groupID);
@@ -189,6 +202,9 @@ private:
 
     // keep track of answers to api queries about which users are in which groups
     QHash<QString, QHash<QUuid, QUuid>> _groupMembership; // QHash<user-name, QHash<group-id, rank-id>>
+
+    /// guard read/write access from multiple threads to settings 
+    QReadWriteLock _settingsLock { QReadWriteLock::Recursive };
 };
 
 #endif // hifi_DomainServerSettingsManager_h

domain-server/src/EntitiesBackupHandler.cpp

@@ -0,0 +1,83 @@
+//
+//  EntitiesBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/14/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntitiesBackupHandler.h"
+
+#include <QDebug>
+
+#include <quazip5/quazip.h>
+#include <quazip5/quazipfile.h>
+
+#include <OctreeDataUtils.h>
+
+EntitiesBackupHandler::EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath) :
+    _entitiesFilePath(entitiesFilePath),
+    _entitiesReplacementFilePath(entitiesReplacementFilePath)
+{
+}
+
+static const QString ENTITIES_BACKUP_FILENAME = "models.json.gz";
+
+void EntitiesBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+    QFile entitiesFile { _entitiesFilePath };
+
+    if (entitiesFile.open(QIODevice::ReadOnly)) {
+        QuaZipFile zipFile { &zip };
+        if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(ENTITIES_BACKUP_FILENAME, _entitiesFilePath))) {
+            qCritical().nospace() << "Failed to open " << ENTITIES_BACKUP_FILENAME << " for writing in zip";
+            return;
+        }
+        auto entityData = entitiesFile.readAll();
+        if (zipFile.write(entityData) != entityData.size()) {
+            qCritical() << "Failed to write entities file to backup";
+            zipFile.close();
+            return;
+        }
+        zipFile.close();
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCritical().nospace() << "Failed to zip " << ENTITIES_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+    }
+}
+
+void EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    if (!zip.setCurrentFile(ENTITIES_BACKUP_FILENAME)) {
+        qWarning() << "Failed to find" << ENTITIES_BACKUP_FILENAME << "while recovering backup";
+        return;
+    }
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::ReadOnly)) {
+        qCritical() << "Failed to open" << ENTITIES_BACKUP_FILENAME << "in backup";
+        return;
+    }
+    auto rawData = zipFile.readAll();
+
+    zipFile.close();
+
+    OctreeUtils::RawEntityData data;
+    if (!data.readOctreeDataInfoFromData(rawData)) {
+        qCritical() << "Unable to parse octree data during backup recovery";
+        return;
+    }
+
+    data.resetIdAndVersion();
+
+    if (zipFile.getZipError() != UNZ_OK) {
+        qCritical().nospace() << "Failed to unzip " << ENTITIES_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        return;
+    }
+
+    QFile entitiesFile { _entitiesReplacementFilePath };
+
+    if (entitiesFile.open(QIODevice::WriteOnly)) {
+        entitiesFile.write(data.toGzippedByteArray());
+    }
+}

domain-server/src/EntitiesBackupHandler.h

@@ -0,0 +1,47 @@
+//
+//  EntitiesBackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/14/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_EntitiesBackupHandler_h
+#define hifi_EntitiesBackupHandler_h
+
+#include "BackupHandler.h"
+
+class EntitiesBackupHandler : public BackupHandlerInterface {
+public:
+    EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override { return { true, 1.0f }; }
+    std::pair<bool, float> getRecoveryStatus() override { return { false, 1.0f }; }
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override {}
+
+    void loadingComplete() override {}
+
+    // Create a skeleton backup
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+
+    // Recover from a full backup
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+
+    // Delete a skeleton backup
+    void deleteBackup(const QString& backupName) override {}
+
+    // Create a full backup
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override {}
+
+    bool isCorruptedBackup(const QString& backupName) override { return false; }
+
+private:
+    QString _entitiesFilePath;
+    QString _entitiesReplacementFilePath;
+};
+
+#endif /* hifi_EntitiesBackupHandler_h */

interface/src/Application.cpp

@@ -6358,13 +6358,14 @@ bool Application::askToWearAvatarAttachmentUrl(const QString& url) {
 void Application::replaceDomainContent(const QString& url) {
     qCDebug(interfaceapp) << "Attempting to replace domain content: " << url;
     QByteArray urlData(url.toUtf8());
-    auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
+    auto limitedNodeList = DependencyManager::get<NodeList>();
+    const auto& domainHandler = limitedNodeList->getDomainHandler();
     limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
         return node->getType() == NodeType::EntityServer && node->getActiveSocket();
-    }, [&urlData, limitedNodeList](const SharedNodePointer& octreeNode) {
+    }, [&urlData, limitedNodeList, &domainHandler](const SharedNodePointer& octreeNode) {
         auto octreeFilePacket = NLPacket::create(PacketType::OctreeFileReplacementFromUrl, urlData.size(), true);
         octreeFilePacket->write(urlData);
-        limitedNodeList->sendPacket(std::move(octreeFilePacket), *octreeNode);
+        limitedNodeList->sendPacket(std::move(octreeFilePacket), domainHandler.getSockAddr());
     });
     auto addressManager = DependencyManager::get<AddressManager>();
     addressManager->handleLookupString(DOMAIN_SPAWNING_POINT);

interface/src/ui/DomainConnectionModel.cpp

@@ -98,4 +98,4 @@ void DomainConnectionModel::refresh() {
     //inform view that we want refresh data
     beginResetModel();
     endResetModel();
-}
+}

libraries/embedded-webserver/src/HTTPConnection.cpp

@@ -133,12 +133,56 @@ QList<FormData> HTTPConnection::parseFormData() const {
 }
 
 void HTTPConnection::respond(const char* code, const QByteArray& content, const char* contentType, const Headers& headers) {
+    respondWithStatusAndHeaders(code, contentType, headers, content.size());
+
+    _socket->write(content);
+
+    _socket->disconnectFromHost();
+
+    // make sure we receive no further read notifications
+    disconnect(_socket, &QTcpSocket::readyRead, this, nullptr);
+}
+
+void HTTPConnection::respond(const char* code, std::unique_ptr<QIODevice> device, const char* contentType, const Headers& headers) {
+    _responseDevice = std::move(device);
+
+    if (_responseDevice->isSequential()) {
+        qWarning() << "Error responding to HTTPConnection: sequential IO devices not supported";
+        respondWithStatusAndHeaders(StatusCode500, contentType, headers, 0);
+        _socket->disconnect(SIGNAL(readyRead()), this);
+        _socket->disconnectFromHost();
+        return;
+    }
+
+    int totalToBeWritten = _responseDevice->size();
+    respondWithStatusAndHeaders(code, contentType, headers, totalToBeWritten);
+
+    if (_responseDevice->atEnd()) {
+        _socket->disconnectFromHost();
+    } else {
+        connect(_socket, &QTcpSocket::bytesWritten, this, [this, totalToBeWritten](size_t bytes) mutable {
+            constexpr size_t HTTP_RESPONSE_CHUNK_SIZE = 1024 * 10;
+            if (!_responseDevice->atEnd()) {
+                totalToBeWritten -= _socket->write(_responseDevice->read(HTTP_RESPONSE_CHUNK_SIZE));
+                if (_responseDevice->atEnd()) {
+                    _socket->disconnectFromHost();
+                    disconnect(_socket, &QTcpSocket::bytesWritten, this, nullptr);
+                }
+            }
+        });
+
+    }
+
+    // make sure we receive no further read notifications
+    disconnect(_socket, &QTcpSocket::readyRead, this, nullptr);
+}
+
+void HTTPConnection::respondWithStatusAndHeaders(const char* code, const char* contentType, const Headers& headers, qint64 contentLength) {
     _socket->write("HTTP/1.1 ");
+
     _socket->write(code);
     _socket->write("\r\n");
 
-    int csize = content.size();
-
     for (Headers::const_iterator it = headers.constBegin(), end = headers.constEnd();
             it != end; it++) {
         _socket->write(it.key());
@@ -146,9 +190,10 @@ void HTTPConnection::respond(const char* code, const QByteArray& content, const
         _socket->write(it.value());
         _socket->write("\r\n");
     }
-    if (csize > 0) {
+
+    if (contentLength > 0) {
         _socket->write("Content-Length: ");
-        _socket->write(QByteArray::number(csize));
+        _socket->write(QByteArray::number(contentLength));
         _socket->write("\r\n");
 
         _socket->write("Content-Type: ");
@@ -156,21 +201,16 @@ void HTTPConnection::respond(const char* code, const QByteArray& content, const
         _socket->write("\r\n");
     }
     _socket->write("Connection: close\r\n\r\n");
-
-    if (csize > 0) {
-        _socket->write(content);
-    }
-
-    // make sure we receive no further read notifications
-    _socket->disconnect(SIGNAL(readyRead()), this);
-
-    _socket->disconnectFromHost();
 }
 
 void HTTPConnection::readRequest() {
     if (!_socket->canReadLine()) {
         return;
     }
+    if (!_requestUrl.isEmpty()) {
+        qDebug() << "Request URL was already set";
+        return;
+    }
     // parse out the method and resource
     QByteArray line = _socket->readLine().trimmed();
     if (line.startsWith("HEAD")) {

libraries/embedded-webserver/src/HTTPConnection.h

@@ -26,6 +26,8 @@
 #include <QPair>
 #include <QUrl>
 
+#include <memory>
+
 class QTcpSocket;
 class HTTPManager;
 class MaskFilter;
@@ -87,6 +89,9 @@ public:
     void respond (const char* code, const QByteArray& content = QByteArray(),
         const char* contentType = DefaultContentType,
         const Headers& headers = Headers());
+    void respond (const char* code, std::unique_ptr<QIODevice> device,
+        const char* contentType = DefaultContentType,
+        const Headers& headers = Headers());
 
 protected slots:
 
@@ -100,6 +105,7 @@ protected slots:
     void readContent ();
 
 protected:
+    void respondWithStatusAndHeaders(const char* code, const char* contentType, const Headers& headers, qint64 size);
 
     /// The parent HTTP manager
     HTTPManager* _parentManager;
@@ -127,6 +133,9 @@ protected:
 
     /// The content of the request.
     QByteArray _requestContent;
+
+    /// Response content
+    std::unique_ptr<QIODevice> _responseDevice;
 };
 
 #endif // hifi_HTTPConnection_h

libraries/embedded-webserver/src/HTTPManager.cpp

@@ -98,13 +98,14 @@ bool HTTPManager::handleHTTPRequest(HTTPConnection* connection, const QUrl& url,
             // file exists, serve it
             static QMimeDatabase mimeDatabase;
             
-            QFile localFile(filePath);
-            localFile.open(QIODevice::ReadOnly);
-            QByteArray localFileData = localFile.readAll();
+            auto localFile = std::unique_ptr<QFile>(new QFile(filePath));
+            localFile->open(QIODevice::ReadOnly);
+            QByteArray localFileData;
             
             QFileInfo localFileInfo(filePath);
             
             if (localFileInfo.completeSuffix() == "shtml") {
+                localFileData = localFile->readAll();
                 // this is a file that may have some SSI statements
                 // the only thing we support is the include directive, but check the contents for that
                 
@@ -153,8 +154,12 @@ bool HTTPManager::handleHTTPRequest(HTTPConnection* connection, const QUrl& url,
                 ? QString { "text/html" }
                 : mimeDatabase.mimeTypeForFile(filePath).name();
 
-            connection->respond(HTTPConnection::StatusCode200, localFileData, qPrintable(mimeType));
-            
+            if (localFileData.isNull()) {
+                connection->respond(HTTPConnection::StatusCode200, std::move(localFile), qPrintable(mimeType));
+            } else {
+                connection->respond(HTTPConnection::StatusCode200, localFileData, qPrintable(mimeType));
+            }
+
             return true;
         }
     }

libraries/entities/src/EntityTree.cpp

@@ -2272,6 +2272,8 @@ bool EntityTree::writeToMap(QVariantMap& entityDescription, OctreeElementPointer
     if (! entityDescription.contains("Entities")) {
         entityDescription["Entities"] = QVariantList();
     }
+    entityDescription["DataVersion"] = _persistDataVersion;
+    entityDescription["Id"] = _persistID;
     QScriptEngine scriptEngine;
     RecurseOctreeToMapOperator theOperator(entityDescription, element, &scriptEngine, skipDefaultValues,
                                             skipThoseWithBadParents, _myAvatar);
@@ -2284,6 +2286,14 @@ bool EntityTree::readFromMap(QVariantMap& map) {
     int contentVersion = map["Version"].toInt();
     bool needsConversion = (contentVersion < (int)EntityVersion::ZoneLightInheritModes);
 
+    if (map.contains("Id")) {
+        _persistID = map["Id"].toUuid();
+    }
+
+    if (map.contains("DataVersion")) {
+        _persistDataVersion = map["DataVersion"].toInt();
+    }
+
     // map will have a top-level list keyed as "Entities".  This will be extracted
     // and iterated over.  Each member of this list is converted to a QVariantMap, then
     // to a QScriptValue, and then to EntityItemProperties.  These properties are used

libraries/image/CMakeLists.txt

@@ -5,7 +5,8 @@ link_hifi_libraries(shared gpu)
 if (NOT ANDROID)
     add_dependency_external_projects(nvtt)
     find_package(NVTT REQUIRED)
+
     target_include_directories(${TARGET_NAME} PRIVATE ${NVTT_INCLUDE_DIRS})
     target_link_libraries(${TARGET_NAME} ${NVTT_LIBRARIES})
     add_paths_to_fixup_libs(${NVTT_DLL_PATH})
-endif()
+endif()

libraries/networking/src/AssetClient.cpp

@@ -53,7 +53,7 @@ AssetClient::AssetClient() {
             this, &AssetClient::handleNodeClientConnectionReset);
 }
 
-void AssetClient::init() {
+void AssetClient::initCaching() {
     Q_ASSERT(QThread::currentThread() == thread());
 
     // Setup disk cache if not already

libraries/networking/src/AssetClient.h

@@ -64,7 +64,7 @@ public:
     Q_INVOKABLE AssetUpload* createUpload(const QByteArray& data);
 
 public slots:
-    void init();
+    void initCaching();
 
     void cacheInfoRequest(QObject* reciever, QString slot);
     MiniPromise::Promise cacheInfoRequestAsync(MiniPromise::Promise deferred = nullptr);

libraries/networking/src/AssetUtils.cpp

@@ -72,9 +72,8 @@ QByteArray loadFromCache(const QUrl& url) {
             qCDebug(asset_client) << url.toDisplayString() << "not in disk cache";
         }
 
-    } else {
-        qCWarning(asset_client) << "No disk cache to load assets from.";
     }
+
     return QByteArray();
 }
 
@@ -96,9 +95,8 @@ bool saveToCache(const QUrl& url, const QByteArray& file) {
             }
             qCWarning(asset_client) << "Could not save" << url.toDisplayString() << "to disk cache.";
         }
-    } else {
-        qCWarning(asset_client) << "No disk cache to save assets to.";
     }
+    
     return false;
 }
 

libraries/networking/src/BaseAssetScriptingInterface.cpp

@@ -47,7 +47,7 @@ bool BaseAssetScriptingInterface::initializeCache() {
     }
 
     // attempt to initialize the cache
-    QMetaObject::invokeMethod(assetClient().data(), "init");
+    QMetaObject::invokeMethod(assetClient().data(), "initCaching");
 
     Promise deferred = makePromise("BaseAssetScriptingInterface--queryCacheStatus");
     deferred->then([this](QVariantMap result) {

libraries/networking/src/LimitedNodeList.cpp

@@ -315,8 +315,10 @@ bool LimitedNodeList::packetSourceAndHashMatchAndTrackBandwidth(const udt::Packe
         }
 
         if (sourceNode) {
-            if (!PacketTypeEnum::getNonVerifiedPackets().contains(headerType) &&
-                !isDomainServer()) {
+            bool verifiedPacket = !PacketTypeEnum::getNonVerifiedPackets().contains(headerType);
+            bool ignoreVerification = isDomainServer() && PacketTypeEnum::getDomainIgnoredVerificationPackets().contains(headerType);
+
+            if (verifiedPacket && !ignoreVerification) {
 
                 QByteArray packetHeaderHash = NLPacket::verificationHashInHeader(packet);
                 QByteArray expectedHash = NLPacket::hashForPacketAndSecret(packet, sourceNode->getConnectionSecret());
@@ -326,6 +328,7 @@ bool LimitedNodeList::packetSourceAndHashMatchAndTrackBandwidth(const udt::Packe
                     static QMultiMap<QUuid, PacketType> hashDebugSuppressMap;
 
                     if (!hashDebugSuppressMap.contains(sourceID, headerType)) {
+                        qCDebug(networking) << packetHeaderHash << expectedHash;
                         qCDebug(networking) << "Packet hash mismatch on" << headerType << "- Sender" << sourceID;
 
                         hashDebugSuppressMap.insert(sourceID, headerType);

libraries/networking/src/ResourceManager.cpp

@@ -31,7 +31,7 @@ ResourceManager::ResourceManager() {
 
     auto assetClient = DependencyManager::set<AssetClient>();
     assetClient->moveToThread(&_thread);
-    QObject::connect(&_thread, &QThread::started, assetClient.data(), &AssetClient::init);
+    QObject::connect(&_thread, &QThread::started, assetClient.data(), &AssetClient::initCaching);
 
     _thread.start();
 }

libraries/networking/src/ThreadedAssignment.h

@@ -18,8 +18,6 @@
 
 #include "Assignment.h"
 
-using DownstreamNodeFoundCallback = std::function<void(Node& downstreamNode)>;
-
 class ThreadedAssignment : public Assignment {
     Q_OBJECT
 public:
@@ -47,10 +45,10 @@ protected:
     QTimer _domainServerTimer;
     QTimer _statsTimer;
     int _numQueuedCheckIns { 0 };
-    
+
 protected slots:
     void domainSettingsRequestFailed();
-    
+
 private slots:
     void checkInWithDomainServerOrExit();
 };

libraries/networking/src/udt/PacketHeaders.h

@@ -126,6 +126,11 @@ public:
         EntityScriptCallMethod,
         ChallengeOwnershipRequest,
         ChallengeOwnershipReply,
+
+        OctreeDataFileRequest,
+        OctreeDataFileReply,
+        OctreeDataPersist,
+
         NUM_PACKET_TYPE
     };
 
@@ -165,6 +170,8 @@ public:
             << PacketTypeEnum::Value::DomainConnectionDenied << PacketTypeEnum::Value::DomainServerPathQuery
             << PacketTypeEnum::Value::DomainServerPathResponse << PacketTypeEnum::Value::DomainServerAddedNode
             << PacketTypeEnum::Value::DomainServerConnectionToken << PacketTypeEnum::Value::DomainSettingsRequest
+            << PacketTypeEnum::Value::OctreeDataFileRequest << PacketTypeEnum::Value::OctreeDataFileReply
+            << PacketTypeEnum::Value::OctreeDataPersist << PacketTypeEnum::Value::OctreeFileReplacementFromUrl
             << PacketTypeEnum::Value::DomainSettings << PacketTypeEnum::Value::ICEServerPeerInformation
             << PacketTypeEnum::Value::ICEServerQuery << PacketTypeEnum::Value::ICEServerHeartbeat
             << PacketTypeEnum::Value::ICEServerHeartbeatACK << PacketTypeEnum::Value::ICEPing
@@ -180,14 +187,19 @@ public:
 
     const static QSet<PacketTypeEnum::Value> getDomainSourcedPackets() {
         const static QSet<PacketTypeEnum::Value> DOMAIN_SOURCED_PACKETS = QSet<PacketTypeEnum::Value>()
-        << PacketTypeEnum::Value::AssetMappingOperation
-        << PacketTypeEnum::Value::AssetMappingOperationReply
-        << PacketTypeEnum::Value::AssetGet
-        << PacketTypeEnum::Value::AssetGetReply
-        << PacketTypeEnum::Value::AssetUpload
-        << PacketTypeEnum::Value::AssetUploadReply;
+            << PacketTypeEnum::Value::AssetMappingOperation
+            << PacketTypeEnum::Value::AssetGet
+            << PacketTypeEnum::Value::AssetUpload;
         return DOMAIN_SOURCED_PACKETS;
     }
+
+    const static QSet<PacketTypeEnum::Value> getDomainIgnoredVerificationPackets() {
+        const static QSet<PacketTypeEnum::Value> DOMAIN_IGNORED_VERIFICATION_PACKETS = QSet<PacketTypeEnum::Value>()
+            << PacketTypeEnum::Value::AssetMappingOperationReply
+            << PacketTypeEnum::Value::AssetGetReply
+            << PacketTypeEnum::Value::AssetUploadReply;
+        return DOMAIN_IGNORED_VERIFICATION_PACKETS;
+    }
 };
 
 using PacketType = PacketTypeEnum::Value;

libraries/networking/src/udt/Socket.cpp

@@ -517,7 +517,7 @@ void Socket::handleSocketError(QAbstractSocket::SocketError socketError) {
     static QString repeatedMessage
         = LogHandler::getInstance().addRepeatedMessageRegex(SOCKET_REGEX);
 
-    qCDebug(networking) << "udt::Socket error - " << socketError;
+    qCDebug(networking) << "udt::Socket error - " << socketError << _udpSocket.errorString();
 }
 
 void Socket::handleStateChanged(QAbstractSocket::SocketState socketState) {

libraries/octree/src/Octree.cpp

@@ -1778,11 +1778,9 @@ bool Octree::writeToFile(const char* fileName, const OctreeElementPointer& eleme
     return success;
 }
 
-bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& element, bool doGzip) {
+bool Octree::toJSON(QJsonDocument* doc, const OctreeElementPointer& element) {
     QVariantMap entityDescription;
 
-    qCDebug(octree, "Saving JSON SVO to file %s...", fileName);
-
     OctreeElementPointer top;
     if (element) {
         top = element;
@@ -1802,17 +1800,33 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
         return false;
     }
 
-    // convert the QVariantMap to JSON
-    QByteArray jsonData = QJsonDocument::fromVariant(entityDescription).toJson();
-    QByteArray jsonDataForFile;
+    *doc = QJsonDocument::fromVariant(entityDescription);
+    return true;
+}
 
-    if (doGzip) {
-        if (!gzip(jsonData, jsonDataForFile, -1)) {
-            qCritical("unable to gzip data while saving to json.");
-            return false;
-        }
-    } else {
-        jsonDataForFile = jsonData;
+bool Octree::toGzippedJSON(QByteArray* data, const OctreeElementPointer& element) {
+    QJsonDocument doc;
+    if (!toJSON(&doc, element)) {
+        qCritical("Failed to convert Entities to QVariantMap while converting to json.");
+        return false;
+    }
+
+    QByteArray jsonData = doc.toJson();
+
+    if (!gzip(jsonData, *data, -1)) {
+        qCritical("Unable to gzip data while saving to json.");
+        return false;
+    }
+
+    return true;
+}
+
+bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& element, bool doGzip) {
+    qCDebug(octree, "Saving JSON SVO to file %s...", fileName);
+
+    QByteArray jsonDataForFile;
+    if (!toGzippedJSON(&jsonDataForFile)) {
+        return false;
     }
 
     QFile persistFile(fileName);
@@ -1823,6 +1837,7 @@ bool Octree::writeToJSONFile(const char* fileName, const OctreeElementPointer& e
         qCritical("Could not write to JSON description of entities.");
     }
 
+
     return success;
 }
 

libraries/octree/src/Octree.h

@@ -28,6 +28,7 @@
 #include "OctreeElementBag.h"
 #include "OctreePacketData.h"
 #include "OctreeSceneStats.h"
+#include "OctreeUtils.h"
 
 class ReadBitstreamToTreeParams;
 class Octree;
@@ -283,8 +284,10 @@ public:
     void loadOctreeFile(const char* fileName);
 
     // Octree exporters
-    bool writeToFile(const char* filename, const OctreeElementPointer& element = NULL, QString persistAsFileType = "json.gz");
-    bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = NULL, bool doGzip = false);
+    bool toJSON(QJsonDocument* doc, const OctreeElementPointer& element = nullptr);
+    bool toGzippedJSON(QByteArray* data, const OctreeElementPointer& element = nullptr);
+    bool writeToFile(const char* filename, const OctreeElementPointer& element = nullptr, QString persistAsFileType = "json.gz");
+    bool writeToJSONFile(const char* filename, const OctreeElementPointer& element = nullptr, bool doGzip = false);
     virtual bool writeToMap(QVariantMap& entityDescription, OctreeElementPointer element, bool skipDefaultValues,
                             bool skipThoseWithBadParents) = 0;
 
@@ -326,6 +329,11 @@ public:
     virtual void dumpTree() { }
     virtual void pruneTree() { }
 
+    void setOctreeVersionInfo(QUuid id, int64_t dataVersion) {
+        _persistID = id;
+        _persistDataVersion = dataVersion;
+    }
+
     virtual void resetEditStats() { }
     virtual quint64 getAverageDecodeTime() const { return 0; }
     virtual quint64 getAverageLookupTime() const { return 0;  }
@@ -334,6 +342,8 @@ public:
     virtual quint64 getAverageLoggingTime() const { return 0;  }
     virtual quint64 getAverageFilterTime() const { return 0; }
 
+    void incrementPersistDataVersion() { _persistDataVersion++; }
+
 signals:
     void importSize(float x, float y, float z);
     void importProgress(int progress);
@@ -359,6 +369,9 @@ protected:
 
     OctreeElementPointer _rootElement = nullptr;
 
+    QUuid _persistID { QUuid::createUuid() };
+    int _persistDataVersion { 0 };
+
     bool _isDirty;
     bool _shouldReaverage;
     bool _stopImport;

libraries/octree/src/OctreeDataUtils.cpp

@@ -0,0 +1,128 @@
+//
+//  OctreeDataUtils.cpp
+//  libraries/octree/src
+//
+//  Created by Ryan Huffman 2018-02-26
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+
+#include "OctreeDataUtils.h"
+
+#include <Gzip.h>
+#include <udt/PacketHeaders.h>
+
+#include <QDebug>
+#include <QJsonObject>
+#include <QJsonDocument>
+#include <QFile>
+
+// Reads octree file and parses it into a QJsonDocument. Handles both gzipped and non-gzipped files.
+// Returns true if the file was successfully opened and parsed, otherwise false.
+// Example failures: file does not exist, gzipped file cannot be unzipped, invalid JSON.
+bool readOctreeFile(QString path, QJsonDocument* doc) {
+    QFile file(path);
+    if (!file.open(QIODevice::ReadOnly)) {
+        qCritical() << "Cannot open json file for reading: " << path;
+        return false;
+    }
+
+    QByteArray data = file.readAll();
+    QByteArray jsonData;
+
+    if (!gunzip(data, jsonData)) {
+        jsonData = data;
+    }
+
+    *doc = QJsonDocument::fromJson(jsonData);
+    return !doc->isNull();
+}
+
+bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromJSON(QJsonObject root) {
+    if (root.contains("Id") && root.contains("DataVersion")) {
+        id = root["Id"].toVariant().toUuid();
+        version = root["DataVersion"].toInt();
+    }
+    readSubclassData(root);
+    return true;
+}
+
+bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromData(QByteArray data) {
+    QByteArray jsonData;
+    if (gunzip(data, jsonData)) {
+        data = jsonData;
+    }
+
+    auto doc = QJsonDocument::fromJson(data);
+    if (doc.isNull()) {
+        return false;
+    }
+
+    auto root = doc.object();
+    return readOctreeDataInfoFromJSON(root);
+}
+
+// Reads octree file and parses it into a RawOctreeData object.
+// Returns false if readOctreeFile fails.
+bool OctreeUtils::RawOctreeData::readOctreeDataInfoFromFile(QString path) {
+    QJsonDocument doc;
+    if (!readOctreeFile(path, &doc)) {
+        return false;
+    }
+
+    auto root = doc.object();
+    return readOctreeDataInfoFromJSON(root);
+}
+
+QByteArray OctreeUtils::RawOctreeData::toByteArray() {
+    const auto protocolVersion = (int)versionForPacketType((PacketTypeEnum::Value)dataPacketType());
+    QJsonObject obj {
+        { "DataVersion", QJsonValue((qint64)version) },
+        { "Id", QJsonValue(id.toString()) },
+        { "Version", protocolVersion },
+    };
+
+    writeSubclassData(obj);
+
+    QJsonDocument doc;
+    doc.setObject(obj);
+
+    return doc.toJson();
+}
+
+QByteArray OctreeUtils::RawOctreeData::toGzippedByteArray() {
+    auto data = toByteArray();
+    QByteArray gzData;
+
+    if (!gzip(data, gzData, -1)) {
+        qCritical("Unable to gzip data while converting json.");
+        return QByteArray();
+    }
+
+    return gzData;
+}
+
+PacketType OctreeUtils::RawOctreeData::dataPacketType() const {
+    Q_ASSERT(false);
+    qCritical() << "Attemping to read packet type for incomplete base type 'RawOctreeData'";
+    return (PacketType)0;
+}
+
+void OctreeUtils::RawOctreeData::resetIdAndVersion() {
+    id = QUuid::createUuid();
+    version = OctreeUtils::INITIAL_VERSION;
+    qDebug() << "Reset octree data to: " << id << version;
+}
+
+void OctreeUtils::RawEntityData::readSubclassData(const QJsonObject& root) {
+    if (root.contains("Entities")) {
+        entityData = root["Entities"].toArray();
+    }
+}
+
+void OctreeUtils::RawEntityData::writeSubclassData(QJsonObject& root) const {
+    root["Entities"] = entityData;
+}
+
+PacketType OctreeUtils::RawEntityData::dataPacketType() const { return PacketType::EntityData; }

libraries/octree/src/OctreeDataUtils.h

@@ -0,0 +1,57 @@
+//
+//  OctreeDataUtils.h
+//  libraries/octree/src
+//
+//  Created by Ryan Huffman 2018-02-26
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+
+#ifndef hifi_OctreeDataUtils_h
+#define hifi_OctreeDataUtils_h
+
+#include <udt/PacketHeaders.h>
+
+#include <QJsonObject>
+#include <QUuid>
+#include <QJsonArray>
+
+namespace OctreeUtils {
+
+using Version = int64_t;
+constexpr Version INITIAL_VERSION = 0;
+
+//using PacketType = uint8_t;
+
+// RawOctreeData is an intermediate format between JSON and a fully deserialized Octree.
+class RawOctreeData {
+public:
+    QUuid id { QUuid() };
+    Version version { -1 };
+
+    virtual PacketType dataPacketType() const;
+
+    virtual void readSubclassData(const QJsonObject& root) { }
+    virtual void writeSubclassData(QJsonObject& root) const { }
+
+    void resetIdAndVersion();
+    QByteArray toByteArray();
+    QByteArray toGzippedByteArray();
+
+    bool readOctreeDataInfoFromData(QByteArray data);
+    bool readOctreeDataInfoFromFile(QString path);
+    bool readOctreeDataInfoFromJSON(QJsonObject root);
+};
+
+class RawEntityData : public RawOctreeData {
+    PacketType dataPacketType() const override;
+    void readSubclassData(const QJsonObject& root) override;
+    void writeSubclassData(QJsonObject& root) const override;
+
+    QJsonArray entityData;
+};
+
+}
+
+#endif // hifi_OctreeDataUtils_h

libraries/octree/src/OctreePersistThread.cpp

@@ -31,18 +31,20 @@
 
 #include "OctreeLogging.h"
 #include "OctreePersistThread.h"
+#include "OctreeUtils.h"
+#include "OctreeDataUtils.h"
 
 const int OctreePersistThread::DEFAULT_PERSIST_INTERVAL = 1000 * 30; // every 30 seconds
-const QString OctreePersistThread::REPLACEMENT_FILE_EXTENSION = ".replace";
 
 OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory, int persistInterval,
                                          bool wantBackup, const QJsonObject& settings, bool debugTimestampNow,
-                                         QString persistAsFileType) :
+                                         QString persistAsFileType, const QByteArray& replacementData) :
     _tree(tree),
     _filename(filename),
     _backupDirectory(backupDirectory),
     _persistInterval(persistInterval),
     _initialLoadComplete(false),
+    _replacementData(replacementData),
     _loadTimeUSecs(0),
     _lastCheck(0),
     _wantBackup(wantBackup),
@@ -52,6 +54,7 @@ OctreePersistThread::OctreePersistThread(OctreePointer tree, const QString& file
 {
     parseSettings(settings);
 
+
     // in case the persist filename has an extension that doesn't match the file type
     QString sansExt = fileNameWithoutExtension(_filename, PERSIST_EXTENSIONS);
     _filename = sansExt + "." + _persistAsFileType;
@@ -132,51 +135,54 @@ quint64 OctreePersistThread::getMostRecentBackupTimeInUsecs(const QString& forma
     return mostRecentBackupInUsecs;
 }
 
-void OctreePersistThread::possiblyReplaceContent() {
-    // before we load the normal file, check if there's a pending replacement file
-    auto replacementFileName = _filename + REPLACEMENT_FILE_EXTENSION;
+void OctreePersistThread::replaceData(QByteArray data) {
+    backupCurrentFile();
 
-    QFile replacementFile { replacementFileName };
-    if (replacementFile.exists()) {
-        // we have a replacement file to process
-        qDebug() << "Replacing models file with" << replacementFileName;
-
-        // first take the current models file and move it to a different filename, appended with the timestamp
-        QFile currentFile { _filename };
-        if (currentFile.exists()) {
-            static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss";
-            auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT);
-
-            if (currentFile.rename(backupFileName)) {
-                qDebug() << "Moved previous models file to" << backupFileName;
-            } else {
-                qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file";
-
-                if (!replacementFile.remove()) {
-                    qWarning() << "Could not remove replacement models file from" << replacementFileName
-                        << "- replacement will be re-attempted on next server restart";
-                    return;
-                }
-            }
-        }
-
-        // rename the replacement file to match what the persist thread is just about to read
-        if (!replacementFile.rename(_filename)) {
-            qWarning() << "Could not replace models file with" << replacementFileName << "- starting with empty models file";
-        }
+    QFile currentFile { _filename };
+    if (currentFile.open(QIODevice::WriteOnly)) {
+        currentFile.write(data);
+        qDebug() << "Wrote replacement data";
+    } else {
+        qWarning() << "Failed to write replacement data";
     }
 }
 
+// Return true if current file is backed up successfully or doesn't exist.
+bool OctreePersistThread::backupCurrentFile() {
+    // first take the current models file and move it to a different filename, appended with the timestamp
+    QFile currentFile { _filename };
+    if (currentFile.exists()) {
+        static const QString FILENAME_TIMESTAMP_FORMAT = "yyyyMMdd-hhmmss";
+        auto backupFileName = _filename + ".backup." + QDateTime::currentDateTime().toString(FILENAME_TIMESTAMP_FORMAT);
+
+        if (currentFile.rename(backupFileName)) {
+            qDebug() << "Moved previous models file to" << backupFileName;
+            return true;
+        } else {
+            qWarning() << "Could not backup previous models file to" << backupFileName << "- removing replacement models file";
+            return false;
+        }
+    }
+    return true;
+}
 
 bool OctreePersistThread::process() {
 
     if (!_initialLoadComplete) {
-        possiblyReplaceContent();
-
         quint64 loadStarted = usecTimestampNow();
         qCDebug(octree) << "loading Octrees from file: " << _filename << "...";
 
-        bool persistantFileRead;
+        if (!_replacementData.isNull()) {
+            replaceData(_replacementData);
+        }
+
+        OctreeUtils::RawOctreeData data;
+        if (data.readOctreeDataInfoFromFile(_filename)) {
+            qDebug() << "Setting entity version info to: " << data.id << data.version;
+            _tree->setOctreeVersionInfo(data.id, data.version);
+        }
+
+        bool persistentFileRead;
 
         _tree->withWriteLock([&] {
             PerformanceWarning warn(true, "Loading Octree File", true);
@@ -199,7 +205,7 @@ bool OctreePersistThread::process() {
                 qCDebug(octree) << "Loading Octree... lock file removed:" << lockFileName;
             }
 
-            persistantFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
+            persistentFileRead = _tree->readFromFile(qPrintable(_filename.toLocal8Bit()));
             _tree->pruneTree();
         });
 
@@ -207,7 +213,7 @@ bool OctreePersistThread::process() {
         _loadTimeUSecs = loadDone - loadStarted;
 
         _tree->clearDirtyBit(); // the tree is clean since we just loaded it
-        qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistantFileRead));
+        qCDebug(octree, "DONE loading Octrees from file... fileRead=%s", debug::valueOf(persistentFileRead));
 
         unsigned long nodeCount = OctreeElement::getNodeCount();
         unsigned long internalNodeCount = OctreeElement::getInternalNodeCount();
@@ -237,6 +243,11 @@ bool OctreePersistThread::process() {
         // want an uninitialized value for this, so we set it to the current time (startup of the server)
         time(&_lastPersistTime);
 
+        if (_replacementData.isNull()) {
+            sendLatestEntityDataToDS();
+        }
+        _replacementData.clear();
+
         emit loadCompleted();
     }
 
@@ -272,7 +283,6 @@ bool OctreePersistThread::process() {
     return isStillRunning();  // keep running till they terminate us
 }
 
-
 void OctreePersistThread::aboutToFinish() {
     qCDebug(octree) << "Persist thread about to finish...";
     persist();
@@ -302,6 +312,7 @@ void OctreePersistThread::persist() {
         backup(); // handle backup if requested        
         qCDebug(octree) << "persist operation DONE with backup...";
 
+        _tree->incrementPersistDataVersion();
 
         // create our "lock" file to indicate we're saving.
         QString lockFileName = _filename + ".lock";
@@ -319,6 +330,23 @@ void OctreePersistThread::persist() {
             remove(qPrintable(lockFileName));
             qCDebug(octree) << "saving Octree lock file removed:" << lockFileName;
         }
+
+        sendLatestEntityDataToDS();
+    }
+}
+
+void OctreePersistThread::sendLatestEntityDataToDS() {
+    qDebug() << "Sending latest entity data to DS";
+    auto nodeList = DependencyManager::get<NodeList>();
+    const DomainHandler& domainHandler = nodeList->getDomainHandler();
+
+    QByteArray data;
+    if (_tree->toGzippedJSON(&data)) {
+        auto message = NLPacketList::create(PacketType::OctreeDataPersist, QByteArray(), true, true);
+        message->write(data);
+        nodeList->sendPacketList(std::move(message), domainHandler.getSockAddr());
+    } else {
+        qCWarning(octree) << "Failed to persist octree to DS";
     }
 }
 
@@ -453,7 +481,6 @@ void OctreePersistThread::rollOldBackupVersions(const BackupRule& rule) {
     }
 }
 
-
 void OctreePersistThread::backup() {
     qCDebug(octree) << "backup operation wantBackup:" << _wantBackup;
     if (_wantBackup) {

libraries/octree/src/OctreePersistThread.h

@@ -18,7 +18,6 @@
 #include <GenericThread.h>
 #include "Octree.h"
 
-/// Generalized threaded processor for handling received inbound packets.
 class OctreePersistThread : public GenericThread {
     Q_OBJECT
 public:
@@ -32,11 +31,11 @@ public:
     };
 
     static const int DEFAULT_PERSIST_INTERVAL;
-    static const QString REPLACEMENT_FILE_EXTENSION;
 
     OctreePersistThread(OctreePointer tree, const QString& filename, const QString& backupDirectory,
                         int persistInterval = DEFAULT_PERSIST_INTERVAL, bool wantBackup = false,
-                        const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false, QString persistAsFileType="json.gz");
+                        const QJsonObject& settings = QJsonObject(), bool debugTimestampNow = false,
+                        QString persistAsFileType = "json.gz", const QByteArray& replacementData = QByteArray());
 
     bool isInitialLoadComplete() const { return _initialLoadComplete; }
     quint64 getLoadElapsedTime() const { return _loadTimeUSecs; }
@@ -61,7 +60,10 @@ protected:
     bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
     quint64 getMostRecentBackupTimeInUsecs(const QString& format);
     void parseSettings(const QJsonObject& settings);
-    void possiblyReplaceContent();
+    bool backupCurrentFile();
+
+    void replaceData(QByteArray data);
+    void sendLatestEntityDataToDS();
 
 private:
     OctreePointer _tree;
@@ -69,6 +71,7 @@ private:
     QString _backupDirectory;
     int _persistInterval;
     bool _initialLoadComplete;
+    QByteArray _replacementData;
 
     quint64 _loadTimeUSecs;
 

libraries/octree/src/OctreeUtils.cpp

@@ -17,7 +17,6 @@
 
 #include <AABox.h>
 
-
 float calculateRenderAccuracy(const glm::vec3& position,
         const AABox& bounds,
         float octreeSizeScale,
@@ -74,4 +73,4 @@ float getOrthographicAccuracySize(float octreeSizeScale, int boundaryLevelAdjust
     // Smallest visible element is 1cm
     const float smallestSize = 0.01f;
     return (smallestSize * MAX_VISIBILITY_DISTANCE_FOR_UNIT_ELEMENT) / boundaryDistanceForRenderLevel(boundaryLevelAdjust, octreeSizeScale);
-}
+}

libraries/octree/src/OctreeUtils.h

@@ -15,6 +15,7 @@
 #include "OctreeConstants.h"
 
 class AABox;
+class QJsonDocument;
 
 /// renderAccuracy represents a floating point "visibility" of an object based on it's view from the camera. At a simple
 /// level it returns 0.0f for things that are so small for the current settings that they could not be visible.

libraries/shared/src/GenericThread.cpp

@@ -37,7 +37,7 @@ void GenericThread::initialize(bool isThreaded, QThread::Priority priority) {
         // match the thread name to our object name
         _thread->setObjectName(objectName());
 
-        // when the worker thread is started, call our engine's run..
+        connect(_thread, &QThread::started, this, &GenericThread::started);
         connect(_thread, &QThread::started, this, &GenericThread::threadRoutine);
         connect(_thread, &QThread::finished, this, &GenericThread::finished);
 

libraries/shared/src/GenericThread.h

@@ -47,6 +47,7 @@ public slots:
     void threadRoutine();
 
 signals:
+    void started();
     void finished();
 
 protected:

libraries/shared/src/SharedUtil.cpp

@@ -127,7 +127,7 @@ void usecTimestampNowForceClockSkew(qint64 clockSkew) {
     ::usecTimestampNowAdjust = clockSkew;
 }
 
-static qint64 TIME_REFERENCE = 0; // in usec
+static std::atomic<qint64> TIME_REFERENCE { 0 }; // in usec
 static std::once_flag usecTimestampNowIsInitialized;
 static QElapsedTimer timestampTimer;
 
@@ -793,6 +793,10 @@ QString formatUsecTime(double usecs) {
     return formatUsecTime<double>(usecs);
 }
 
+QString formatSecTime(qint64 secs) {
+    return formatUsecTime(secs * 1000000);
+}
+
 
 QString formatSecondsElapsed(float seconds) {
     QString result;

libraries/shared/src/SharedUtil.h

@@ -189,6 +189,7 @@ QString formatUsecTime(float usecs);
 QString formatUsecTime(double usecs);
 QString formatUsecTime(quint64 usecs);
 QString formatUsecTime(qint64 usecs);
+QString formatSecTime(qint64 secs);
 
 QString formatSecondsElapsed(float seconds);
 bool similarStrings(const QString& stringA, const QString& stringB);

tools/atp-client/src/ATPClientApp.cpp

@@ -197,7 +197,7 @@ ATPClientApp::ATPClientApp(int argc, char* argv[]) :
     }
 
     auto assetClient = DependencyManager::set<AssetClient>();
-    assetClient->init();
+    assetClient->initCaching();
 
     if (_verbose) {
         qDebug() << "domain-server address is" << _domainServerAddress;