Merge remote-tracking branch 'upstream/master' into android_dev

.clang-format

@@ -1,12 +1,12 @@
 Language: Cpp
 Standard: Cpp11
-BasedOnStyle: "Chromium" 
+BasedOnStyle: "Chromium"
 ColumnLimit: 128
 IndentWidth: 4
 UseTab: Never
 
 BreakBeforeBraces: Custom
-BraceWrapping:   
+BraceWrapping:
   AfterEnum: true
   AfterClass: false
   AfterControlStatement: false
@@ -21,11 +21,11 @@ BraceWrapping:
 
 
 AccessModifierOffset: -4
-AllowShortFunctionsOnASingleLine: InlineOnly  
-BreakConstructorInitializers: BeforeColon 
-BreakConstructorInitializersBeforeComma: true
+AllowShortFunctionsOnASingleLine: InlineOnly
+BreakConstructorInitializers: AfterColon
+BreakConstructorInitializersBeforeComma: false
 IndentCaseLabels: true
-ReflowComments: false 
+ReflowComments: false
 Cpp11BracedListStyle: false
 ContinuationIndentWidth: 4
 ConstructorInitializerAllOnOneLineOrOnePerLine: false

assignment-client/src/Agent.cpp

@@ -340,7 +340,6 @@ void Agent::scriptRequestFinished() {
     request->deleteLater();
 }
 
-
 void Agent::executeScript() {
     _scriptEngine = scriptEngineFactory(ScriptEngine::AGENT_SCRIPT, _scriptContents, _payload);
 

assignment-client/src/assets/AssetServer.cpp

@@ -257,12 +257,10 @@ AssetServer::AssetServer(ReceivedMessage& message) :
     _transferTaskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT);
     _bakingTaskPool.setMaxThreadCount(1);
 
+    // Queue all requests until the Asset Server is fully setup
     auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
-    packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo");
-    packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload");
-    packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOperation");
-    
+    packetReceiver.registerListenerForTypes({ PacketType::AssetGet, PacketType::AssetGetInfo, PacketType::AssetUpload, PacketType::AssetMappingOperation }, this, "queueRequests");
+
 #ifdef Q_OS_WIN
     updateConsumedCores();
     QTimer* timer = new QTimer(this);
@@ -291,6 +289,7 @@ void AssetServer::aboutToFinish() {
         if (pendingRunnable) {
             it = _pendingBakes.erase(it);
         } else {
+            qDebug() << "Aborting bake for" << it.key();
             it.value()->abort();
             ++it;
         }
@@ -396,6 +395,7 @@ void AssetServer::completeSetup() {
 
         if (_fileMappings.size() > 0) {
             cleanupUnmappedFiles();
+            cleanupBakedFilesForDeletedAssets();
         }
 
         nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
@@ -417,10 +417,65 @@ void AssetServer::completeSetup() {
 
     PathUtils::removeTemporaryApplicationDirs();
     PathUtils::removeTemporaryApplicationDirs("Oven");
+
+    qCDebug(asset_server) << "Overriding temporary queuing packet handler.";
+    // We're fully setup, override the request queueing handler and replay all requests
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
+    packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo");
+    packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload");
+    packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOperation");
+
+    replayRequests();
+}
+
+void AssetServer::queueRequests(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode) {
+    qCDebug(asset_server) << "Queuing requests until fully setup";
+
+    QMutexLocker lock { &_queuedRequestsMutex };
+    _queuedRequests.push_back({ packet, senderNode });
+
+    // If we've stopped queueing but the callback was already in flight,
+    // then replay it immediately.
+    if (!_isQueueingRequests) {
+        lock.unlock();
+        replayRequests();
+    }
+}
+
+void AssetServer::replayRequests() {
+    RequestQueue queue;
+    {
+        QMutexLocker lock { &_queuedRequestsMutex };
+        std::swap(queue, _queuedRequests);
+        _isQueueingRequests = false;
+    }
+
+    qCDebug(asset_server) << "Replaying" << queue.size() << "requests.";
+
+    for (const auto& request : queue) {
+        switch (request.first->getType()) {
+            case PacketType::AssetGet:
+                handleAssetGet(request.first, request.second);
+                break;
+            case PacketType::AssetGetInfo:
+                handleAssetGetInfo(request.first, request.second);
+                break;
+            case PacketType::AssetUpload:
+                handleAssetUpload(request.first, request.second);
+                break;
+            case PacketType::AssetMappingOperation:
+                handleAssetMappingOperation(request.first, request.second);
+                break;
+            default:
+                qCWarning(asset_server) << "Unknown queued request type:" << request.first->getType();
+                break;
+        }
+    }
 }
 
 void AssetServer::cleanupUnmappedFiles() {
-    QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(AssetUtils::SHA256_HASH_HEX_LENGTH) + "}" };
+    QRegExp hashFileRegex { AssetUtils::ASSET_HASH_REGEX_STRING };
 
     auto files = _filesDirectory.entryInfoList(QDir::Files);
 
@@ -452,6 +507,38 @@ void AssetServer::cleanupUnmappedFiles() {
     }
 }
 
+void AssetServer::cleanupBakedFilesForDeletedAssets() {
+    qCInfo(asset_server) << "Performing baked asset cleanup for deleted assets";
+
+    std::set<AssetUtils::AssetHash> bakedHashes;
+
+    for (const auto& it : _fileMappings) {
+        // check if this is a mapping to baked content
+        if (it.first.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
+            // extract the hash from the baked mapping
+            AssetUtils::AssetHash hash = it.first.mid(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER.length(),
+                                                      AssetUtils::SHA256_HASH_HEX_LENGTH);
+
+            // add the hash to our set of hashes for which we have baked content
+            bakedHashes.insert(hash);
+        }
+    }
+
+    // enumerate the hashes for which we have baked content
+    for (const auto& hash : bakedHashes) {
+        // check if we have a mapping that points to this hash
+        auto matchingMapping = std::find_if(std::begin(_fileMappings), std::end(_fileMappings),
+                                            [&hash](const std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash> mappingPair) {
+            return mappingPair.second == hash;
+        });
+
+        if (matchingMapping == std::end(_fileMappings)) {
+            // we didn't find a mapping for this hash, remove any baked content we still have for it
+            removeBakedPathsForDeletedAsset(hash);
+        }
+    }
+}
+
 void AssetServer::handleAssetMappingOperation(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
     using AssetMappingOperationType = AssetUtils::AssetMappingOperationType;
 
@@ -1301,6 +1388,8 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina
 }
 
 void AssetServer::handleAbortedBake(QString originalAssetHash, QString assetPath) {
+    qDebug() << "Aborted bake:" << originalAssetHash;
+
     // for an aborted bake we don't do anything but remove the BakeAssetTask from our pending bakes
     _pendingBakes.remove(originalAssetHash);
 }

assignment-client/src/assets/AssetServer.h

@@ -44,6 +44,7 @@ public slots:
 private slots:
     void completeSetup();
 
+    void queueRequests(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetGetInfo(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetGet(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetUpload(QSharedPointer<ReceivedMessage> packetList, SharedNodePointer senderNode);
@@ -52,6 +53,8 @@ private slots:
     void sendStatsPacket() override;
 
 private:
+    void replayRequests();
+
     void handleGetMappingOperation(ReceivedMessage& message, NLPacketList& replyPacket);
     void handleGetAllMappingOperation(NLPacketList& replyPacket);
     void handleSetMappingOperation(ReceivedMessage& message, bool hasWriteAccess, NLPacketList& replyPacket);
@@ -80,6 +83,9 @@ private:
     /// Delete any unmapped files from the local asset directory
     void cleanupUnmappedFiles();
 
+    /// Delete any baked files for assets removed from the local asset directory
+    void cleanupBakedFilesForDeletedAssets();
+
     QString getPathToAssetHash(const AssetUtils::AssetHash& assetHash);
 
     std::pair<AssetUtils::BakingStatus, QString> getAssetStatus(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash);
@@ -115,6 +121,11 @@ private:
     QHash<AssetUtils::AssetHash, std::shared_ptr<BakeAssetTask>> _pendingBakes;
     QThreadPool _bakingTaskPool;
 
+    QMutex _queuedRequestsMutex;
+    bool _isQueueingRequests { true };
+    using RequestQueue = QVector<QPair<QSharedPointer<ReceivedMessage>, SharedNodePointer>>;
+    RequestQueue _queuedRequests;
+
     bool _wasColorTextureCompressionEnabled { false };
     bool _wasGrayscaleTextureCompressionEnabled { false  };
     bool _wasNormalTextureCompressionEnabled { false };

assignment-client/src/assets/BakeAssetTask.cpp

@@ -69,8 +69,10 @@ void BakeAssetTask::run() {
 
     _ovenProcess.reset(new QProcess());
 
+    QEventLoop loop;
+
     connect(_ovenProcess.get(), static_cast<void(QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished),
-            this, [this, tempOutputDir](int exitCode, QProcess::ExitStatus exitStatus) {
+            this, [&loop, this, tempOutputDir](int exitCode, QProcess::ExitStatus exitStatus) {
         qDebug() << "Baking process finished: " << exitCode << exitStatus;
 
         if (exitStatus == QProcess::CrashExit) {
@@ -108,6 +110,7 @@ void BakeAssetTask::run() {
             emit bakeFailed(_assetHash, _assetPath, errors);
         }
 
+        loop.quit();
     });
 
     qDebug() << "Starting oven for " << _assetPath;
@@ -117,11 +120,21 @@ void BakeAssetTask::run() {
         emit bakeFailed(_assetHash, _assetPath, errors);
         return;
     }
-    _ovenProcess->waitForFinished();
+
+    _isBaking = true;
+
+    loop.exec();
 }
 
 void BakeAssetTask::abort() {
-    if (!_wasAborted.exchange(true)) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "abort");
+        return;
+    }
+    qDebug() << "Aborting BakeAssetTask for" << _assetHash;
+    if (_ovenProcess->state() != QProcess::NotRunning) {
+        qDebug() << "Teminating oven process for" << _assetHash;
+        _wasAborted = true;
         _ovenProcess->terminate();
     }
 }

assignment-client/src/assets/BakeAssetTask.h

@@ -27,12 +27,14 @@ class BakeAssetTask : public QObject, public QRunnable {
 public:
     BakeAssetTask(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath);
 
+    // Thread-safe inspection methods
     bool isBaking() { return _isBaking.load(); }
+    bool wasAborted() const { return _wasAborted.load(); }
 
     void run() override;
 
+public slots:
     void abort();
-    bool wasAborted() const { return _wasAborted.load(); }
 
 signals:
     void bakeComplete(QString assetHash, QString assetPath, QString tempOutputDir, QVector<QString> outputFiles);

assignment-client/src/entities/EntityServer.cpp

@@ -116,7 +116,6 @@ void EntityServer::beforeRun() {
 void EntityServer::entityCreated(const EntityItem& newEntity, const SharedNodePointer& senderNode) {
 }
 
-
 // EntityServer will use the "special packets" to send list of recently deleted entities
 bool EntityServer::hasSpecialPacketsToSend(const SharedNodePointer& node) {
     bool shouldSendDeletedEntities = false;
@@ -277,7 +276,6 @@ int EntityServer::sendSpecialPackets(const SharedNodePointer& node, OctreeQueryN
     return totalBytes;
 }
 
-
 void EntityServer::pruneDeletedEntities() {
     EntityTreePointer tree = std::static_pointer_cast<EntityTree>(_tree);
     if (tree->hasAnyDeletedEntities()) {

assignment-client/src/entities/EntityServer.h

@@ -30,7 +30,6 @@ struct ViewerSendingStats {
 class SimpleEntitySimulation;
 using SimpleEntitySimulationPointer = std::shared_ptr<SimpleEntitySimulation>;
 
-
 class EntityServer : public OctreeServer, public NewlyCreatedEntityHook {
     Q_OBJECT
 public:
@@ -38,7 +37,7 @@ public:
     ~EntityServer();
 
     // Subclasses must implement these methods
-    virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override ;
+    virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override;
     virtual char getMyNodeType() const override { return NodeType::EntityServer; }
     virtual PacketType getMyQueryMessageType() const override { return PacketType::EntityQuery; }
     virtual const char* getMyServerName() const override { return MODEL_SERVER_NAME; }
@@ -82,12 +81,12 @@ private:
     QReadWriteLock _viewerSendingStatsLock;
     QMap<QUuid, QMap<QUuid, ViewerSendingStats>> _viewerSendingStats;
 
-    static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000; // 45m
-    static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000; // 1h
-    int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 45m
-    int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 1h
+    static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000;                    // 45m
+    static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000;                    // 1h
+    int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS;  // 45m
+    int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS;  // 1h
     QTimer _dynamicDomainVerificationTimer;
     void startDynamicDomainVerification();
 };
 
-#endif // hifi_EntityServer_h
+#endif  // hifi_EntityServer_h

assignment-client/src/main.cpp

@@ -9,22 +9,13 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
-#include <LogHandler.h>
+#include <BuildInfo.h>
 #include <SharedUtil.h>
 
 #include "AssignmentClientApp.h"
-#include <BuildInfo.h>
 
 int main(int argc, char* argv[]) {
-    disableQtBearerPoll(); // Fixes wifi ping spikes
-
-    QCoreApplication::setApplicationName(BuildInfo::ASSIGNMENT_CLIENT_NAME);
-    QCoreApplication::setOrganizationName(BuildInfo::MODIFIED_ORGANIZATION);
-    QCoreApplication::setOrganizationDomain(BuildInfo::ORGANIZATION_DOMAIN);
-    QCoreApplication::setApplicationVersion(BuildInfo::VERSION);
-
-    qInstallMessageHandler(LogHandler::verboseMessageHandler);
-    qInfo() << "Starting.";
+    setupHifiApplication(BuildInfo::ASSIGNMENT_CLIENT_NAME);
 
     AssignmentClientApp app(argc, argv);
     

assignment-client/src/octree/OctreeServer.cpp

@@ -33,6 +33,10 @@
 #include <PathUtils.h>
 #include <QtCore/QDir>
 
+#include <OctreeDataUtils.h>
+
+Q_LOGGING_CATEGORY(octree_server, "hifi.octree-server")
+
 int OctreeServer::_clientCount = 0;
 const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000;
 
@@ -84,6 +88,8 @@ int OctreeServer::_longProcessWait = 0;
 int OctreeServer::_shortProcessWait = 0;
 int OctreeServer::_noProcessWait = 0;
 
+static const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
+
 
 void OctreeServer::resetSendingStats() {
     _averageLoopTime.reset();
@@ -202,7 +208,6 @@ void OctreeServer::trackPacketSendingTime(float time) {
     }
 }
 
-
 void OctreeServer::trackProcessWaitTime(float time) {
     const float MAX_SHORT_TIME = 10.0f;
     const float MAX_LONG_TIME = 100.0f;
@@ -283,8 +288,6 @@ void OctreeServer::initHTTPManager(int port) {
     _httpManager = new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this, this);
 }
 
-const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
-
 bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) {
 
 #ifdef FORCE_CRASH
@@ -922,87 +925,6 @@ void OctreeServer::handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> me
     }
 }
 
-void OctreeServer::handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message) {
-    if (!_isFinished && !_isShuttingDown) {
-        // these messages are only allowed to come from the domain server, so make sure that is the case
-        auto nodeList = DependencyManager::get<NodeList>();
-        if (message->getSenderSockAddr() == nodeList->getDomainHandler().getSockAddr()) {
-            // it's far cleaner to load up the new content upon server startup
-            // so here we just store a special file at our persist path
-            // and then force a stop of the server so that it can pick it up when it relaunches
-            if (!_persistAbsoluteFilePath.isEmpty()) {
-                replaceContentFromMessageData(message->getMessage());
-            } else {
-                qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
-            }
-        } else {
-            qDebug() << "Received an octree file replacement that was not from our domain server - refusing to process";
-        }
-    }
-}
-
-// Message->getMessage() contains a QByteArray representation of the URL to download from
-void OctreeServer::handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message) {
-    qInfo() << "Received request to replace content from a url";
-    if (!_isFinished && !_isShuttingDown) {
-        // This call comes from Interface, so we skip our domain server check
-        // but confirm that we have permissions to replace content sets
-        if (DependencyManager::get<NodeList>()->getThisNodeCanReplaceContent()) {
-            if (!_persistAbsoluteFilePath.isEmpty()) {
-                // Convert message data into our URL
-                QString url(message->getMessage());
-                QUrl modelsURL = QUrl(url, QUrl::StrictMode);
-                QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
-                QNetworkRequest request(modelsURL);
-                QNetworkReply* reply = networkAccessManager.get(request);
-                connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
-                    QNetworkReply::NetworkError networkError = reply->error();
-                    if (networkError == QNetworkReply::NoError) {
-                        QByteArray contents = reply->readAll();
-                        replaceContentFromMessageData(contents);
-                    } else {
-                        qDebug() << "Error downloading JSON from specified file";
-                    }
-                });
-            } else {
-                qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
-            }
-        }
-    }
-}
-
-void OctreeServer::replaceContentFromMessageData(QByteArray content) {
-    //Assume we have compressed data
-    auto compressedOctree = content;
-    QByteArray jsonOctree;
-
-    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
-    if (!wasCompressed) {
-        // the source was not compressed, assume we were sent regular JSON data
-        jsonOctree = compressedOctree;
-    }
-    // check the JSON data to verify it is an object
-    if (QJsonDocument::fromJson(jsonOctree).isObject()) {
-        if (!wasCompressed) {
-            // source was not compressed, we compress it before we write it locally
-            gzip(jsonOctree, compressedOctree);
-        }
-        // write the compressed octree data to a special file
-        auto replacementFilePath = _persistAbsoluteFilePath.append(OctreePersistThread::REPLACEMENT_FILE_EXTENSION);
-        QFile replacementFile(replacementFilePath);
-        if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
-            // we've now written our replacement file, time to take the server down so it can
-            // process it when it comes back up
-            qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
-            setFinished(true);
-        } else {
-            qWarning() << "Could not write replacement octree data to file - refusing to process";
-        }
-    } else {
-        qDebug() << "Received replacement octree file that is invalid - refusing to process";
-    }
-}
-
 bool OctreeServer::readOptionBool(const QString& optionName, const QJsonObject& settingsSectionObject, bool& result) {
     result = false; // assume it doesn't exist
     bool optionAvailable = false;
@@ -1119,7 +1041,18 @@ void OctreeServer::readConfiguration() {
             _persistFilePath = getMyDefaultPersistFilename();
         }
 
+        QDir persistPath { _persistFilePath };
+
+        if (persistPath.isRelative()) {
+            // if the domain settings passed us a relative path, make an absolute path that is relative to the
+            // default data directory
+            _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
+        } else {
+            _persistAbsoluteFilePath = persistPath.absolutePath();
+        }
+
         qDebug() << "persistFilePath=" << _persistFilePath;
+        qDebug() << "persisAbsoluteFilePath=" << _persistAbsoluteFilePath;
 
         _persistAsFileType = "json.gz";
 
@@ -1200,20 +1133,94 @@ void OctreeServer::run() {
 }
 
 void OctreeServer::domainSettingsRequestComplete() {
+    if (_state != OctreeServerState::WaitingForDomainSettings) {
+        qCWarning(octree_server) << "Received domain settings after they have already been received";
+        return;
+    }
+
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
+    packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
+
+    packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
+
+    qDebug(octree_server) << "Received domain settings";
+
+    readConfiguration();
+
+    _state = OctreeServerState::WaitingForOctreeDataNegotation;
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    const DomainHandler& domainHandler = nodeList->getDomainHandler();
+
+    auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
+
+    OctreeUtils::RawOctreeData data;
+    qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
+    if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
+        qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
+        packet->writePrimitive(true);
+        auto id = data.id.toRfc4122();
+        packet->write(id);
+        packet->writePrimitive(data.version);
+    } else {
+        qCWarning(octree_server) << "No octree data found";
+        packet->writePrimitive(false);
+    }
+
+    qCDebug(octree_server) << "Sending request for octree data to DS";
+    nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
+} 
+
+void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
+    if (_state != OctreeServerState::WaitingForOctreeDataNegotation) {
+        qCWarning(octree_server) << "Server received ocree data file reply but is not currently negotiating.";
+        return;
+    }
+
+    bool includesNewData;
+    message->readPrimitive(&includesNewData);
+    QByteArray replaceData;
+    if (includesNewData) {
+        replaceData = message->readAll();
+        qDebug() << "Got reply to octree data file request, new data sent";
+    } else {
+        qDebug() << "Got reply to octree data file request, current entity data is sufficient";
+        
+        OctreeUtils::RawEntityData data;
+        qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
+        if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
+            if (data.id.isNull()) {
+                qCDebug(octree_server) << "Current octree data has a null id, updating";
+                data.resetIdAndVersion();
+
+                QFile file(_persistAbsoluteFilePath);
+                if (file.open(QIODevice::WriteOnly)) {
+                    auto entityData = data.toGzippedByteArray();
+                    file.write(entityData);
+                    file.close();
+                } else {
+                    qCDebug(octree_server) << "Failed to update octree data";
+                }
+            }
+        }
+    }
+
+    _state = OctreeServerState::Running;
+    beginRunning(replaceData);
+}
+
+void OctreeServer::beginRunning(QByteArray replaceData) {
+    if (_state != OctreeServerState::Running) {
+        qCWarning(octree_server) << "Server is not running";
+        return;
+    }
 
     auto nodeList = DependencyManager::get<NodeList>();
 
     // we need to ask the DS about agents so we can ping/reply with them
     nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
 
-    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
-    packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
-    packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacement");
-    packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURL");
-
-    readConfiguration();
-
     beforeRun(); // after payload has been processed
 
     connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
@@ -1233,17 +1240,6 @@ void OctreeServer::domainSettingsRequestComplete() {
 
     // if we want Persistence, set up the local file and persist thread
     if (_wantPersist) {
-        // If persist filename does not exist, let's see if there is one beside the application binary
-        // If there is, let's copy it over to our target persist directory
-        QDir persistPath { _persistFilePath };
-        _persistAbsoluteFilePath = persistPath.absolutePath();
-
-        if (persistPath.isRelative()) {
-            // if the domain settings passed us a relative path, make an absolute path that is relative to the
-            // default data directory
-            _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
-        }
-
         static const QString ENTITY_PERSIST_EXTENSION = ".json.gz";
 
         // force the persist file to end with .json.gz
@@ -1328,7 +1324,7 @@ void OctreeServer::domainSettingsRequestComplete() {
 
         // now set up PersistThread
         _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval,
-                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType);
+                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData);
         _persistThread->initialize(true);
     }
 

assignment-client/src/octree/OctreeServer.h

@@ -27,8 +27,18 @@
 #include "OctreeServerConsts.h"
 #include "OctreeInboundPacketProcessor.h"
 
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(octree_server)
+
 const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total
 
+enum class OctreeServerState {
+    WaitingForDomainSettings,
+    WaitingForOctreeDataNegotation,
+    Running
+};
+
 /// Handles assignments of type OctreeServer - sending octrees to various clients.
 class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler {
     Q_OBJECT
@@ -36,6 +46,8 @@ public:
     OctreeServer(ReceivedMessage& message);
     ~OctreeServer();
 
+    OctreeServerState _state { OctreeServerState::WaitingForDomainSettings };
+
     /// allows setting of run arguments
     void setArguments(int argc, char** argv);
 
@@ -137,8 +149,7 @@ private slots:
     void domainSettingsRequestComplete();
     void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message);
-    void handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
     void removeSendThread();
 
 protected:
@@ -159,12 +170,12 @@ protected:
     QString getFileLoadTime();
     QString getConfiguration();
     QString getStatusLink();
+
+    void beginRunning(QByteArray replaceData);
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
     virtual UniqueSendThread newSendThread(const SharedNodePointer& node);
 
-    void replaceContentFromMessageData(QByteArray content);
-
     int _argc;
     const char** _argv;
     char** _parsedArgV;

assignment-client/src/scripts/EntityScriptServer.cpp

@@ -178,7 +178,7 @@ void EntityScriptServer::updateEntityPPS() {
     int numRunningScripts = _entitiesScriptEngine->getNumRunningEntityScripts();
     int pps;
     if (std::numeric_limits<int>::max() / _entityPPSPerScript < numRunningScripts) {
-        qWarning() << QString("Integer multiplaction would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
+        qWarning() << QString("Integer multiplication would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
         pps = std::numeric_limits<int>::max();
         pps = std::min(_maxEntityPPS, pps);
     } else {

domain-server/CMakeLists.txt

@@ -24,7 +24,19 @@ symlink_or_copy_directory_beside_target(${_SHOULD_SYMLINK_RESOURCES} "${CMAKE_CU
 # link the shared hifi libraries
 include_hifi_library_headers(gpu)
 include_hifi_library_headers(graphics)
-link_hifi_libraries(embedded-webserver networking shared avatars)
+link_hifi_libraries(embedded-webserver networking shared avatars octree)
+
+target_zlib()
+
+add_dependency_external_projects(quazip)
+
+find_package(QuaZip REQUIRED)
+target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${QUAZIP_INCLUDE_DIRS})
+target_link_libraries(${TARGET_NAME} ${QUAZIP_LIBRARIES})
+
+if (WIN32)
+  add_paths_to_fixup_libs(${QUAZIP_DLL_PATH})
+endif ()
 
 # find OpenSSL
 find_package(OpenSSL REQUIRED)

domain-server/resources/describe-settings.json

@@ -1,5 +1,5 @@
 {
-  "version": 2.1,
+  "version": 2.2,
   "settings": [
     {
       "name": "metaverse",
@@ -306,7 +306,37 @@
             }
           ],
           "non-deletable-row-key": "permissions_id",
-          "non-deletable-row-values": [ "localhost", "anonymous", "logged-in" ]
+          "non-deletable-row-values": [ "localhost", "anonymous", "logged-in" ],
+          "default": [
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "anonymous"
+            },
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "friends"
+            },
+            {
+                "id_can_adjust_locks": true,
+                "id_can_connect": true,
+                "id_can_connect_past_max_capacity": true,
+                "id_can_kick": true,
+                "id_can_replace_content": true,
+                "id_can_rez": true,
+                "id_can_rez_certified": true,
+                "id_can_rez_tmp": true,
+                "id_can_rez_tmp_certified": true,
+                "id_can_write_to_asset_server": true,
+                "permissions_id": "localhost"
+            },
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "logged-in"
+            }
+          ]
         },
         {
           "name": "group_permissions",
@@ -1321,73 +1351,6 @@
           "default": "30000",
           "advanced": true
         },
-        {
-          "name": "backups",
-          "type": "table",
-          "label": "Backup Rules",
-          "help": "In this table you can define a set of rules for how frequently to backup copies of your entites content file.",
-          "numbered": false,
-          "can_add_new_rows": true,
-          "default": [
-            {
-              "Name": "Half Hourly Rolling",
-              "backupInterval": 1800,
-              "format": ".backup.halfhourly.%N",
-              "maxBackupVersions": 5
-            },
-            {
-              "Name": "Daily Rolling",
-              "backupInterval": 86400,
-              "format": ".backup.daily.%N",
-              "maxBackupVersions": 7
-            },
-            {
-              "Name": "Weekly Rolling",
-              "backupInterval": 604800,
-              "format": ".backup.weekly.%N",
-              "maxBackupVersions": 4
-            },
-            {
-              "Name": "Thirty Day Rolling",
-              "backupInterval": 2592000,
-              "format": ".backup.thirtyday.%N",
-              "maxBackupVersions": 12
-            }
-          ],
-          "columns": [
-            {
-              "name": "Name",
-              "label": "Name",
-              "can_set": true,
-              "placeholder": "Example",
-              "default": "Example"
-            },
-            {
-              "name": "format",
-              "label": "Rule Format",
-              "can_set": true,
-              "help": "Format used to create the extension for the backup of your persisted entities. Use a format with %N to get rolling. Or use date formatting like %Y-%m-%d.%H:%M:%S.%z",
-              "placeholder": ".backup.example.%N",
-              "default": ".backup.example.%N"
-            },
-            {
-              "name": "backupInterval",
-              "label": "Backup Interval in Seconds",
-              "help": "Interval between backup checks in seconds.",
-              "placeholder": 1800,
-              "default": 1800,
-              "can_set": true
-            },
-            {
-              "name": "maxBackupVersions",
-              "label": "Max Rolled Backup Versions",
-              "help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?",
-              "placeholder": 5,
-              "default": 5,
-              "can_set": true
-            }
-          ]
-        },
         {
           "name": "NoPersist",
           "type": "checkbox",
@@ -1649,6 +1612,67 @@
         }
       ]
     },
+    {
+      "name": "automatic_content_archives",
+      "label": "Automatic Content Archives",
+      "settings": [
+        {
+          "name": "backup_rules",
+          "type": "table",
+          "label": "Rolling Backup Rules",
+          "help": "Define how frequently to create automatic content archives",
+          "numbered": false,
+          "can_add_new_rows": true,
+          "default": [
+            {
+              "Name": "Half Hourly Rolling",
+              "backupInterval": 1800,
+              "maxBackupVersions": 5
+            },
+            {
+              "Name": "Daily Rolling",
+              "backupInterval": 86400,
+              "maxBackupVersions": 7
+            },
+            {
+              "Name": "Weekly Rolling",
+              "backupInterval": 604800,
+              "maxBackupVersions": 4
+            },
+            {
+              "Name": "Thirty Day Rolling",
+              "backupInterval": 2592000,
+              "maxBackupVersions": 12
+            }
+          ],
+          "columns": [
+            {
+              "name": "Name",
+              "label": "Name",
+              "can_set": true,
+              "placeholder": "Example",
+              "default": "Example"
+            },
+            {
+              "name": "backupInterval",
+              "label": "Backup Interval in Seconds",
+              "help": "Interval between backup checks in seconds.",
+              "placeholder": 1800,
+              "default": 1800,
+              "can_set": true
+            },
+            {
+              "name": "maxBackupVersions",
+              "label": "Max Rolled Backup Versions",
+              "help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?",
+              "placeholder": 5,
+              "default": 5,
+              "can_set": true
+            }
+          ]
+        }
+      ]
+    },
     {
       "name": "wizard",
       "label": "Setup Wizard",

domain-server/resources/web/base-settings-scripts.html

@@ -3,5 +3,4 @@
 <script src='/js/bootbox.min.js'></script>
 <script src='/js/form2js.min.js'></script>
 <script src='/js/bootstrap-switch.min.js'></script>
-<script src='/js/shared.js'></script>
 <script src='/js/base-settings.js'></script>

domain-server/resources/web/content/index.shtml

@@ -14,6 +14,8 @@
 
 <!--#include virtual="base-settings-scripts.html"-->
 
+<script src="js/moment-locale.min.js"></script>
+<script src="js/bootstrap-sortable.min.js"></script>
 <script src="js/content.js"></script>
 
 <!--#include virtual="page-end.html"-->

domain-server/resources/web/content/js/content.js

@@ -1,37 +1,437 @@
 $(document).ready(function(){
 
-  Settings.afterReloadActions = function() {};
+  var RESTORE_SETTINGS_UPLOAD_ID = 'restore-settings-button';
+  var RESTORE_SETTINGS_FILE_ID = 'restore-settings-file';
+  var UPLOAD_CONTENT_ALLOWED_DIV_ID = 'upload-content-allowed';
+  var UPLOAD_CONTENT_RECOVERING_DIV_ID = 'upload-content-recovering';
 
-  var frm = $('#upload-form');
-  frm.submit(function (ev) {
-    $.ajax({
-      type: frm.attr('method'),
-      url: frm.attr('action'),
-      data: new FormData($(this)[0]),
-      cache: false,
-      contentType: false,
-      processData: false,
-      success: function (data) {
-        swal({
-          title: 'Uploaded',
-          type: 'success',
-          text: 'Your Entity Server is restarting to replace its local content with the uploaded file.',
-          confirmButtonText: 'OK'
-        })
-      },
-      error: function (data) {
-        swal({
-          title: '',
-          type: 'error',
-          text: 'Your entities file could not be transferred to the Entity Server.</br>Verify that the file is a <i>.json</i> or <i>.json.gz</i> entities file and try again.',
-          html: true,
-          confirmButtonText: 'OK',
+  var isRestoring = false;
+
+  function progressBarHTML(extraClass, label) {
+    var html = "<div class='progress'>";
+    html += "<div class='" + extraClass + " progress-bar progress-bar-success progress-bar-striped active' role='progressbar' aria-valuemin='0' aria-valuemax='100'>";
+    html += label + "<span class='sr-only'></span></div></div>";
+    return html;
+  }
+
+  function setupBackupUpload() {
+    // construct the HTML needed for the settings backup panel
+    var html = "<div class='form-group'><div id='" + UPLOAD_CONTENT_ALLOWED_DIV_ID + "'>";
+
+    html += "<span class='help-block'>Upload a content archive (.zip) or entity file (.json, .json.gz) to replace the content of this domain.";
+    html += "<br/>Note: Your domain content will be replaced by the content you upload, but the existing backup files of your domain's content will not immediately be changed.</span>";
+
+    html += "<input id='restore-settings-file' name='restore-settings' type='file'>";
+    html += "<button type='button' id='" + RESTORE_SETTINGS_UPLOAD_ID + "' disabled='true' class='btn btn-primary'>Upload Content</button>";
+
+    html += "</div><div id='" + UPLOAD_CONTENT_RECOVERING_DIV_ID + "'>";
+    html += "<span class='help-block'>Restore in progress</span>";
+    html += progressBarHTML('recovery', 'Restoring');
+    html += "</div></div>";
+
+    $('#' + Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID + ' .panel-body').html(html);
+  }
+
+  // handle content archive or entity file upload
+
+  // when the selected file is changed, enable the button if there's a selected file
+  $('body').on('change', '#' + RESTORE_SETTINGS_FILE_ID, function() {
+    $('#' + RESTORE_SETTINGS_UPLOAD_ID).attr('disabled', $(this).val().length == 0);
+  });
+
+  // when the upload button is clicked, send the file to the DS
+  // and reload the page if restore was successful or
+  // show an error if not
+  $('body').on('click', '#' + RESTORE_SETTINGS_UPLOAD_ID, function(e){
+    e.preventDefault();
+
+    swalAreYouSure(
+      "Your domain content will be replaced by the uploaded content archive or entity file",
+      "Restore content",
+      function() {
+        var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
+
+        var fileFormData = new FormData();
+        fileFormData.append('restore-file', files[0]);
+
+        showSpinnerAlert("Uploading content to restore");
+
+        $.ajax({
+          url: '/content/upload',
+          type: 'POST',
+          cache: false,
+          processData: false,
+          contentType: false,
+          data: fileFormData
+        }).done(function(data, textStatus, jqXHR) {
+          isRestoring = true;
+
+          // immediately reload backup information since one should be restoring now
+          reloadBackupInformation();
+
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain content.\n"
+            + "Please ensure that the content archive or entity file is valid and try again."
+          );
         });
       }
-    });
-
-    ev.preventDefault();
-
-    showSpinnerAlert("Uploading Entities File");
+    );
   });
+
+  var GENERATE_ARCHIVE_BUTTON_ID = 'generate-archive-button';
+  var CONTENT_ARCHIVES_NORMAL_ID = 'content-archives-success';
+  var CONTENT_ARCHIVES_ERROR_ID = 'content-archives-error';
+  var AUTOMATIC_ARCHIVES_TABLE_ID = 'automatic-archives-table';
+  var AUTOMATIC_ARCHIVES_TBODY_ID = 'automatic-archives-tbody';
+  var MANUAL_ARCHIVES_TABLE_ID = 'manual-archives-table';
+  var MANUAL_ARCHIVES_TBODY_ID = 'manual-archives-tbody';
+  var AUTO_ARCHIVES_SETTINGS_LINK_ID = 'auto-archives-settings-link';
+  var ACTION_MENU_CLASS = 'action-menu';
+
+  var automaticBackups = [];
+  var manualBackups = [];
+
+  function setupContentArchives() {
+    // construct the HTML needed for the content archives panel
+    var html = "<div id='" + CONTENT_ARCHIVES_NORMAL_ID + "'><div class='form-group'>";
+    html += "<label class='control-label'>Automatic Content Archives</label>";
+    html += "<span class='help-block'>Your domain server makes regular archives of the content in your domain. In the list below, you can see and download all of your backups of domain content and content settings."
+    html += "<a href='/settings/#automatic_content_archives' id='" + AUTO_ARCHIVES_SETTINGS_LINK_ID + "'>Click here to manage automatic content archive intervals.</a></span>";
+    html += "</div>";
+    html += "<table class='table sortable' id='" + AUTOMATIC_ARCHIVES_TABLE_ID + "'>";
+
+    var backups_table_head = "<thead><tr class='gray-tr'><th>Archive Name</th><th data-defaultsort='desc'>Archive Date</th>"
+      + "<th data-defaultsort='disabled'></th><th class='" + ACTION_MENU_CLASS + "' data-defaultsort='disabled'>Actions</th>"
+      + "</tr></thead>";
+
+    html += backups_table_head;
+    html += "<tbody id='" + AUTOMATIC_ARCHIVES_TBODY_ID + "'></tbody></table>";
+    html += "<div class='form-group'>";
+    html += "<label class='control-label'>Manual Content Archives</label>";
+    html += "<span class='help-block'>You can generate and download an archive of your domain content right now. You can also download, delete and restore any archive listed.</span>";
+    html += "<button type='button' id='" + GENERATE_ARCHIVE_BUTTON_ID + "' class='btn btn-primary'>Generate New Archive</button>";
+    html += "</div>";
+    html += "<table class='table sortable' id='" + MANUAL_ARCHIVES_TABLE_ID + "'>";
+    html += backups_table_head;
+    html += "<tbody id='" + MANUAL_ARCHIVES_TBODY_ID + "'></tbody></table></div>";
+
+    html += "<div class='form-group' id='" + CONTENT_ARCHIVES_ERROR_ID + "' style='display:none;'>"
+      + "<span class='help-block'>There was a problem loading your list of automatic and manual content archives. "
+      + "Please reload the page to try again.</span></div>";
+
+    // put the base HTML in the content archives panel
+    $('#' + Settings.CONTENT_ARCHIVES_PANEL_ID + ' .panel-body').html(html);
+  }
+
+  var BACKUP_RESTORE_LINK_CLASS = 'restore-backup';
+  var BACKUP_DOWNLOAD_LINK_CLASS = 'download-backup';
+  var BACKUP_DELETE_LINK_CLASS = 'delete-backup';
+  var ACTIVE_BACKUP_ROW_CLASS = 'active-backup';
+  var CORRUPTED_ROW_CLASS = 'danger';
+
+  function reloadBackupInformation() {
+    // make a GET request to get backup information to populate the table
+    $.ajax({
+      url: '/api/backups',
+      cache: false
+    }).done(function(data) {
+
+      // split the returned data into manual and automatic manual backups
+      var splitBackups = _.partition(data.backups, function(value, index) {
+        return value.isManualBackup;
+      });
+
+      if (isRestoring && !data.status.isRecovering) {
+        // we were recovering and we finished - the DS is going to restart so show the restart modal
+        showRestartModal();
+        return;
+      }
+
+      isRestoring = data.status.isRecovering;
+
+      manualBackups = splitBackups[0];
+      automaticBackups = splitBackups[1];
+
+      // populate the backups tables with the backups
+      function createBackupTableRow(backup) {
+        return "<tr data-backup-id='" + backup.id + "' data-backup-name='" + backup.name + "'>"
+          + "<td data-value='" + backup.name.toLowerCase() + "'>" + backup.name + "</td><td data-value='" + backup.createdAtMillis + "'>"
+          + moment(backup.createdAtMillis).format('lll')
+          + "</td><td class='backup-status'></td><td class='" + ACTION_MENU_CLASS + "'>"
+          + "<div class='dropdown'><div class='dropdown-toggle' data-toggle='dropdown' aria-expanded='false'><span class='glyphicon glyphicon-option-vertical'></span></div>"
+          + "<ul class='dropdown-menu dropdown-menu-right'>"
+          + "<li><a class='" + BACKUP_RESTORE_LINK_CLASS + "' href='#'>Restore from here</a></li><li class='divider'></li>"
+          + "<li><a class='" + BACKUP_DOWNLOAD_LINK_CLASS + "' href='/api/backups/" + backup.id + "'>Download</a></li><li class='divider'></li>"
+          + "<li><a class='" + BACKUP_DELETE_LINK_CLASS + "' href='#' target='_blank'>Delete</a></li></ul></div></td>";
+      }
+
+      function updateProgressBars($progressBar, value) {
+        $progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%');
+        $progressBar.find('.sr-only').html(value + "% Complete");
+      }
+
+      // before we add any new rows and update existing ones
+      // remove our flag for active rows
+      $('.' + ACTIVE_BACKUP_ROW_CLASS).removeClass(ACTIVE_BACKUP_ROW_CLASS);
+
+      function updateOrAddTableRow(backup, tableBodyID) {
+        // check for a backup with this ID
+        var $backupRow = $("tr[data-backup-id='" + backup.id + "']");
+
+        if ($backupRow.length == 0) {
+          // create a new row and then add it to the table
+          $backupRow = $(createBackupTableRow(backup));
+          $('#' + tableBodyID).append($backupRow);
+        }
+
+        // update the row status column depending on if it is available or recovering
+        if (!backup.isAvailable) {
+          // add a progress bar to the status row for availability
+          $backupRow.find('td.backup-status').html(progressBarHTML('availability', 'Archiving'));
+
+          // set the value of the progress bar based on availability progress
+          updateProgressBars($backupRow.find('.progress-bar'), backup.availabilityProgress * 100);
+        } else if (backup.id == data.status.recoveringBackupId) {
+          // add a progress bar to the status row for recovery
+          $backupRow.find('td.backup-status').html(progressBarHTML('recovery', 'Restoring'));
+        } else if (backup.isCorrupted) {
+          // add text for corrupted status to row
+          $backupRow.find('td.backup-status').html('<span>Corrupted</span>');
+        } else {
+          // no special status for this row, use an empty status column
+          $backupRow.find('td.backup-status').html('');
+        }
+
+        // color the row red if it is corrupted
+        $backupRow.toggleClass(CORRUPTED_ROW_CLASS, backup.isCorrupted);
+
+        // disable restore if the backup is corrupted
+        $backupRow.find('a.' + BACKUP_RESTORE_LINK_CLASS).parent().toggleClass('disabled', backup.isCorrupted);
+
+        // toggle the dropdown menu depending on if the row is available
+        $backupRow.find('td.' + ACTION_MENU_CLASS + ' .dropdown').toggle(backup.isAvailable);
+
+        $backupRow.addClass(ACTIVE_BACKUP_ROW_CLASS);
+      }
+
+      if (automaticBackups.length > 0) {
+        for (var backupIndex in automaticBackups) {
+          updateOrAddTableRow(automaticBackups[backupIndex], AUTOMATIC_ARCHIVES_TBODY_ID);
+        }
+      }
+
+      if (manualBackups.length > 0) {
+        for (var backupIndex in manualBackups) {
+          updateOrAddTableRow(manualBackups[backupIndex], MANUAL_ARCHIVES_TBODY_ID);
+        }
+      }
+
+      // at this point, any rows that no longer have the ACTIVE_BACKUP_ROW_CLASS
+      // are deleted backups, so we remove them from the table
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID + ' tbody tr:not(.' + ACTIVE_BACKUP_ROW_CLASS + ')').remove();
+
+      // check if the restore action on all rows should be enabled or disabled
+      $('tr:not(.' + CORRUPTED_ROW_CLASS + ') .' + BACKUP_RESTORE_LINK_CLASS).parent().toggleClass('disabled', data.status.isRecovering);
+
+      // hide or show the manual content upload file and button depending on our recovering status
+      $('#' + UPLOAD_CONTENT_ALLOWED_DIV_ID).toggle(!data.status.isRecovering);
+      $('#' + UPLOAD_CONTENT_RECOVERING_DIV_ID).toggle(data.status.isRecovering);
+
+      // update the progress bars for current restore status
+      if (data.status.isRecovering) {
+        updateProgressBars($('.recovery.progress-bar'), data.status.recoveryProgress * 100);
+      }
+
+      // tell bootstrap sortable to update for the new rows
+      $.bootstrapSortable({ applyLast: true });
+
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID).toggle(true);
+      $('#' + CONTENT_ARCHIVES_ERROR_ID).toggle(false);
+
+    }).fail(function(){
+      // we've hit the very rare case where we couldn't load the list of backups from the domain server
+
+      // set our backups to empty
+      automaticBackups = [];
+      manualBackups = [];
+
+      // replace the content archives panel with a simple error message
+      // stating that the user should reload the page
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID).toggle(false);
+      $('#' + CONTENT_ARCHIVES_ERROR_ID).toggle(true);
+
+    }).always(function(){
+      // toggle showing or hiding the tables depending on if they have entries
+      $('#' + AUTOMATIC_ARCHIVES_TABLE_ID).toggle(automaticBackups.length > 0);
+      $('#' + MANUAL_ARCHIVES_TABLE_ID).toggle(manualBackups.length > 0);
+    });
+  }
+
+  // handle click in table to restore a given content backup
+  $('body').on('click', '.' + BACKUP_RESTORE_LINK_CLASS, function(e) {
+    // stop the default behaviour
+    e.preventDefault();
+
+    // if this is a disabled link, don't proceed with the restore
+    if ($(this).parent().hasClass('disabled')) {
+      return false;
+    }
+
+    // grab the name of this backup so we can show it in alerts
+    var backupName = $(this).closest('tr').attr('data-backup-name');
+
+    // grab the ID of this backup in case we need to send a POST
+    var backupID = $(this).closest('tr').attr('data-backup-id');
+
+    // make sure the user knows what is about to happen
+    swalAreYouSure(
+      "Your domain content will be replaced by the content archive " + backupName,
+      "Restore content",
+      function() {
+        // show a spinner while we send off our request
+        showSpinnerAlert("Starting restore of " +  backupName);
+
+        // setup an AJAX POST to request content restore
+        $.post('/api/backups/recover/' + backupID).done(function(data, textStatus, jqXHR) {
+          isRestoring = true;
+
+          // immediately reload our backup information since one should be restoring now
+          reloadBackupInformation();
+
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain content.\n"
+            + "If the problem persists, the content archive may be corrupted."
+          );
+        });
+      }
+    )
+  });
+
+  // handle click in table to delete a given content backup
+  $('body').on('click', '.' + BACKUP_DELETE_LINK_CLASS, function(e){
+    // stop the default behaviour
+    e.preventDefault();
+
+    // grab the name of this backup so we can show it in alerts
+    var backupName = $(this).closest('tr').attr('data-backup-name');
+
+    // grab the ID of this backup in case we need to send the DELETE request
+    var backupID = $(this).closest('tr').attr('data-backup-id');
+
+    // make sure the user knows what is about to happen
+    swalAreYouSure(
+      "The content archive " + backupName + " will be deleted and will no longer be available for restore or download from this page.",
+      "Delete content archive",
+      function() {
+        // show a spinner while we send off our request
+        showSpinnerAlert("Deleting content archive " +  backupName);
+
+        // setup an AJAX DELETE to request content archive delete
+        $.ajax({
+          url: '/api/backups/' + backupID,
+          type: 'DELETE'
+        }).done(function(data, textStatus, jqXHR) {
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was an unexpected error deleting the content archive"
+          );
+        }).always(function(){
+          // reload the list of content archives in case we deleted a backup
+          // or it's no longer an available backup for some other reason
+          reloadBackupInformation();
+        });
+      }
+    )
+  });
+
+  // handle click on automatic content archive settings link
+  $('body').on('click', '#' + AUTO_ARCHIVES_SETTINGS_LINK_ID, function(e) {
+    if (Settings.pendingChanges > 0) {
+      // don't follow the link right away, make sure the user knows they are about to leave
+      // the page and lose changes
+      e.preventDefault();
+
+      var settingsLink = $(this).attr('href');
+
+      swalAreYouSure(
+        "You have pending changes to content settings that have not been saved. They will be lost if you leave the page to manage automatic content archive intervals.",
+        "Proceed without Saving",
+        function() {
+          // user wants to drop their changes, switch pages
+          window.location =  settingsLink;
+        }
+      );
+    }
+  });
+
+  // handle click on manual archive creation button
+  $('body').on('click', '#' + GENERATE_ARCHIVE_BUTTON_ID, function(e) {
+    e.preventDefault();
+
+    // show a sweet alert to ask the user to provide a name for their content archive
+    swal({
+      title: "Generate a content archive",
+      type: "input",
+      text: "This will capture the state of all the content in your domain right now, which you can save as a backup and restore from later.",
+      confirmButtonText: "Generate Archive",
+      showCancelButton: true,
+      closeOnConfirm: false,
+      inputPlaceholder: 'Archive Name'
+    }, function(inputValue){
+      if (inputValue === false) {
+        return false;
+      }
+
+      if (inputValue === "") {
+        swal.showInputError("Please give the content archive a name.")
+        return false;
+      }
+
+      var MANUAL_ARCHIVE_NAME_REGEX = /^[a-zA-Z0-9\-_ ]+$/;
+      if (!MANUAL_ARCHIVE_NAME_REGEX.test(inputValue)) {
+        swal.showInputError("Valid characters include A-z, 0-9, ' ', '_', and '-'.");
+        return false;
+      }
+
+      // post the provided archive name to ask the server to kick off a manual backup
+      $.ajax({
+        type: 'POST',
+        url: '/api/backups',
+        data: {
+          'name': inputValue
+        }
+      }).done(function(data) {
+        // since we successfully setup a new content archive, reload the table of archives
+        // which should show that this archive is pending creation
+        swal.close();
+        reloadBackupInformation();
+      }).fail(function(jqXHR, textStatus, errorThrown) {
+        showErrorMessage(
+          "Error",
+          "There was an unexpected error creating the manual content archive"
+        )
+      });
+    });
+  });
+
+  Settings.extraGroupsAtIndex = Settings.extraContentGroupsAtIndex;
+
+  Settings.afterReloadActions = function() {
+    setupBackupUpload();
+    setupContentArchives();
+
+    // load the latest backups immediately
+    reloadBackupInformation();
+
+    // setup a timer to reload them every 5 seconds
+    setInterval(reloadBackupInformation, 5000);
+  };
 });

domain-server/resources/web/css/bootstrap-sortable.css

@@ -0,0 +1,110 @@
+/**
+ * adding sorting ability to HTML tables with Bootstrap styling
+ * @summary HTML tables sorting ability
+ * @version 2.0.0
+ * @requires tinysort, moment.js, jQuery
+ * @license MIT
+ * @author Matus Brlit (drvic10k)
+ * @copyright Matus Brlit (drvic10k), bootstrap-sortable contributors
+ */
+
+table.sortable span.sign {
+    display: block;
+    position: absolute;
+    top: 50%;
+    right: 5px;
+    font-size: 12px;
+    margin-top: -10px;
+    color: #bfbfc1;
+}
+
+table.sortable th:after {
+    display: block;
+    position: absolute;
+    top: 50%;
+    right: 5px;
+    font-size: 12px;
+    margin-top: -10px;
+    color: #bfbfc1;
+}
+
+table.sortable th.arrow:after {
+    content: '';
+}
+
+table.sortable span.arrow, span.reversed, th.arrow.down:after, th.reversedarrow.down:after, th.arrow.up:after, th.reversedarrow.up:after {
+    border-style: solid;
+    border-width: 5px;
+    font-size: 0;
+    border-color: #ccc transparent transparent transparent;
+    line-height: 0;
+    height: 0;
+    width: 0;
+    margin-top: -2px;
+}
+
+    table.sortable span.arrow.up, th.arrow.up:after {
+        border-color: transparent transparent #ccc transparent;
+        margin-top: -7px;
+    }
+
+table.sortable span.reversed, th.reversedarrow.down:after {
+    border-color: transparent transparent #ccc transparent;
+    margin-top: -7px;
+}
+
+    table.sortable span.reversed.up, th.reversedarrow.up:after {
+        border-color: #ccc transparent transparent transparent;
+        margin-top: -2px;
+    }
+
+table.sortable span.az:before, th.az.down:after {
+    content: "a .. z";
+}
+
+table.sortable span.az.up:before, th.az.up:after {
+    content: "z .. a";
+}
+
+table.sortable th.az.nosort:after, th.AZ.nosort:after, th._19.nosort:after, th.month.nosort:after {
+    content: "..";
+}
+
+table.sortable span.AZ:before, th.AZ.down:after {
+    content: "A .. Z";
+}
+
+table.sortable span.AZ.up:before, th.AZ.up:after {
+    content: "Z .. A";
+}
+
+table.sortable span._19:before, th._19.down:after {
+    content: "1 .. 9";
+}
+
+table.sortable span._19.up:before, th._19.up:after {
+    content: "9 .. 1";
+}
+
+table.sortable span.month:before, th.month.down:after {
+    content: "jan .. dec";
+}
+
+table.sortable span.month.up:before, th.month.up:after {
+    content: "dec .. jan";
+}
+
+table.sortable>thead th:not([data-defaultsort=disabled]) {
+    cursor: pointer;
+    position: relative;
+    top: 0;
+    left: 0;
+}
+
+table.sortable>thead th:hover:not([data-defaultsort=disabled]) {
+    background: #efefef;
+}
+
+table.sortable>thead th div.mozilla {
+    position: relative;
+}

domain-server/resources/web/css/style.css

@@ -355,21 +355,31 @@ table .headers + .headers td {
   }
 }
 
-ul.nav li.dropdown ul.dropdown-menu {
+ul.dropdown-menu {
   padding: 0px 0px;
 }
 
-ul.nav li.dropdown li a {
+ul.dropdown-menu li a {
   padding-top: 7px;
   padding-bottom: 7px;
 }
 
-ul.nav li.dropdown li a:hover {
+ul.dropdown-menu li a:hover {
   color: white;
   background-color: #337ab7;
 }
 
-ul.nav li.dropdown ul.dropdown-menu .divider {
+table ul.dropdown-menu li:first-child a:hover {
+  border-top-left-radius: 4px;
+  border-top-right-radius: 4px;
+}
+
+ul.dropdown-menu li:last-child a:hover {
+  border-bottom-left-radius: 4px;
+  border-bottom-right-radius: 4px;
+}
+
+ul.dropdown-menu .divider {
   margin: 0px 0;
 }
 
@@ -434,3 +444,42 @@ ul.nav li.dropdown ul.dropdown-menu .divider {
 .save-button-text {
   pointer-events: none;
 }
+
+#content_archives .panel-body {
+  padding: 0;
+}
+
+#content_archives .panel-body .form-group {
+  padding: 15px;
+}
+
+#content_archives .panel-body th, #content_archives .panel-body td {
+  padding: 8px 15px;
+}
+
+#content_archives table {
+  border-top: 1px solid #ddd;
+  border-bottom: 1px solid #ddd;
+}
+
+tr.gray-tr {
+  background-color: #f5f5f5;
+}
+
+table .action-menu {
+  text-align: right;
+  width: 90px;
+}
+
+.dropdown-toggle span.glyphicon-option-vertical {
+  font-size: 110%;
+  cursor: pointer;
+  border-radius: 50%;
+  background-color: #F5F5F5;
+  padding: 4px 4px 4px 6px;
+}
+
+.dropdown.open span.glyphicon-option-vertical {
+  background-color: #337AB7;
+  color: white;
+}

domain-server/resources/web/footer.html

@@ -1,4 +1,5 @@
 </div>
 <script src='/js/jquery-2.1.4.min.js'></script>
 <script src='/js/bootstrap.min.js'></script>
+<script src='/js/shared.js'></script>
 <script src='/js/domain-server.js'></script>

domain-server/resources/web/header.html

@@ -9,6 +9,7 @@
     <link href="/css/style.css" rel="stylesheet" media="screen">
     <link href="/css/sweetalert.css" rel="stylesheet" media="screen">
     <link href="/css/bootstrap-switch.min.css" rel="stylesheet" media="screen">
+    <link href="/css/bootstrap-sortable.css" rel="stylesheet" media="screen">
 
     <script src='/js/sweetalert.min.js'></script>
   </head>

domain-server/resources/web/js/base-settings.js

@@ -106,8 +106,12 @@ function reloadSettings(callback) {
   $.getJSON(Settings.endpoint, function(data){
     _.extend(data, viewHelpers);
 
-    for (var spliceIndex in Settings.extraGroups) {
-      data.descriptions.splice(spliceIndex, 0, Settings.extraGroups[spliceIndex]);
+    for (var spliceIndex in Settings.extraGroupsAtIndex) {
+      data.descriptions.splice(spliceIndex, 0, Settings.extraGroupsAtIndex[spliceIndex]);
+    }
+
+    for (var endGroupIndex in Settings.extraGroupsAtEnd) {
+      data.descriptions.push(Settings.extraGroupsAtEnd[endGroupIndex]);
     }
 
     $('#panels').html(Settings.panelsTemplate(data));
@@ -122,6 +126,8 @@ function reloadSettings(callback) {
 
     $('[data-toggle="tooltip"]').tooltip();
 
+    Settings.pendingChanges = 0;
+
     // call the callback now that settings are loaded
     callback(true);
   }).fail(function() {
@@ -257,7 +263,7 @@ $(document).ready(function(){
     }
   });
 
-  $('#' + Settings.FORM_ID).on('change keyup paste', '.' + Settings.TRIGGER_CHANGE_CLASS , function(e){
+  $('#' + Settings.FORM_ID).on('change input propertychange', '.' + Settings.TRIGGER_CHANGE_CLASS , function(e){
     // this input was changed, add the changed data attribute to it
     $(this).attr('data-changed', true);
 
@@ -676,11 +682,11 @@ function makeTableHiddenInputs(setting, initialValues, categoryValue) {
     } else {
       html +=
         "<td " + (col.hidden ? "style='display: none;'" : "") + " class='" + Settings.DATA_COL_CLASS + "' " +
-            "name='" + col.name + "'>" +
-          "<input type='text' style='display: none;' class='form-control' placeholder='" + (col.placeholder ? col.placeholder : "") + "' " +
-                 "value='" + (defaultValue || "") + "' data-default='" + (defaultValue || "") + "'" +
-                 (col.readonly ? " readonly" : "") + ">" +
-        "</td>";
+          "name='" + col.name + "'>" +
+          "<input type='text' style='display: none;' class='form-control " + Settings.TRIGGER_CHANGE_CLASS +
+          "' placeholder='" + (col.placeholder ? col.placeholder : "") + "' " +
+          "value='" + (defaultValue || "") + "' data-default='" + (defaultValue || "") + "'" +
+          (col.readonly ? " readonly" : "") + ">" + "</td>";
     }
   })
 
@@ -801,6 +807,8 @@ function badgeForDifferences(changedElement) {
     }
   });
 
+  Settings.pendingChanges = totalChanges;
+
   if (totalChanges == 0) {
     totalChanges = ""
   }
@@ -830,7 +838,7 @@ function addTableRow(row) {
     var keyInput = row.children(".key").children("input");
 
     // whenever the keyInput changes, re-badge for differences
-    keyInput.on('change keyup paste', function(e){
+    keyInput.on('change input propertychange', function(e){
       // update siblings in the row to have the correct name
       var currentKey = $(this).val();
 

domain-server/resources/web/js/domain-server.js

@@ -28,7 +28,7 @@ function settingsGroupAnchor(base, html_id) {
 }
 
 $(document).ready(function(){
-  var url = window.location;
+  var url = location.protocol + '//' + location.host+location.pathname;
 
   // Will only work if string in href matches with location
   $('ul.nav a[href="'+ url +'"]').parent().addClass('active');
@@ -39,22 +39,49 @@ $(document).ready(function(){
   }).parent().addClass('active');
 
   $('body').on('click', '#restart-server', function(e) {
-    swal( {
-      title: "Are you sure?",
-      text: "This will restart your domain server, causing your domain to be briefly offline.",
-      type: "warning",
-      html: true,
-      showCancelButton: true
-    }, function() {
-      $.get("/restart");
-      showRestartModal();
-    });
+    swalAreYouSure(
+      "This will restart your domain server, causing your domain to be briefly offline.",
+      "Restart",
+      function() {
+        swal.close();
+        $.get("/restart");
+        showRestartModal();
+      }
+    )
     return false;
   });
 
   var $contentDropdown = $('#content-settings-nav-dropdown');
   var $settingsDropdown = $('#domain-settings-nav-dropdown');
 
+  // define extra groups to add to setting panels, with their splice index
+  Settings.extraContentGroupsAtIndex = {
+    0: {
+      html_id: Settings.CONTENT_ARCHIVES_PANEL_ID,
+      label: 'Content Archives'
+    },
+    1: {
+      html_id: Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID,
+      label: 'Upload Content'
+    }
+  };
+
+  Settings.extraContentGroupsAtEnd = [];
+
+  Settings.extraDomainGroupsAtIndex = {
+    1: {
+      html_id: 'places',
+      label: 'Places'
+    }
+  }
+
+  Settings.extraDomainGroupsAtEnd = [
+    {
+      html_id: 'settings_backup',
+      label: 'Settings Backup / Restore'
+    }
+  ]
+
   // for pages that have the settings dropdowns
   if ($contentDropdown.length && $settingsDropdown.length) {
     // make a JSON request to get the dropdown menus for content and settings
@@ -65,6 +92,15 @@ $(document).ready(function(){
         return "<li class='setting-group'><a href='" + settingsGroupAnchor(base, html_id) + "'>" + group.label + "<span class='badge'></span></a></li>";
       }
 
+      // add the dummy settings groups that get populated via JS
+      for (var spliceIndex in Settings.extraContentGroupsAtIndex) {
+        data.content_settings.splice(spliceIndex, 0, Settings.extraContentGroupsAtIndex[spliceIndex]);
+      }
+
+      for (var endIndex in Settings.extraContentGroupsAtEnd) {
+        data.content_settings.push(Settings.extraContentGroupsAtEnd[endIndex]);
+      }
+
       $.each(data.content_settings, function(index, group){
         if (index > 0) {
           $contentDropdown.append("<li role='separator' class='divider'></li>");
@@ -73,25 +109,22 @@ $(document).ready(function(){
         $contentDropdown.append(makeGroupDropdownElement(group, "/content/"));
       });
 
+      // add the dummy settings groups that get populated via JS
+      for (var spliceIndex in Settings.extraDomainGroupsAtIndex) {
+        data.domain_settings.splice(spliceIndex, 0, Settings.extraDomainGroupsAtIndex[spliceIndex]);
+      }
+
+      for (var endIndex in Settings.extraDomainGroupsAtEnd) {
+        data.domain_settings.push(Settings.extraDomainGroupsAtEnd[endIndex]);
+      }
+
       $.each(data.domain_settings, function(index, group){
         if (index > 0) {
           $settingsDropdown.append("<li role='separator' class='divider'></li>");
         }
 
         $settingsDropdown.append(makeGroupDropdownElement(group, "/settings/"));
-
-        // for domain settings, we add a dummy "Places" group that we fill
-        // via the API - add it to the dropdown menu in the right spot
-        // which is after "Metaverse / Networking"
-        if (group.name == "metaverse") {
-          $settingsDropdown.append("<li role='separator' class='divider'></li>");
-          $settingsDropdown.append(makeGroupDropdownElement({ html_id: 'places', label: 'Places' }, "/settings/"));
-        }
       });
-
-      // append a link for the "Settings Backup" panel
-      $settingsDropdown.append("<li role='separator' class='divider'></li>");
-      $settingsDropdown.append(makeGroupDropdownElement({ html_id: 'settings_backup', label: 'Settings Backup'}, "/settings"));
     });
   }
 });

domain-server/resources/web/js/shared.js

@@ -42,7 +42,9 @@ Object.assign(Settings, {
   ADD_PLACE_BTN_ID: 'add-place-btn',
   FORM_ID: 'settings-form',
   INVALID_ROW_CLASS: 'invalid-input',
-  DATA_ROW_INDEX: 'data-row-index'
+  DATA_ROW_INDEX: 'data-row-index',
+  CONTENT_ARCHIVES_PANEL_ID: 'content_archives',
+  UPLOAD_CONTENT_BACKUP_PANEL_ID: 'upload_content'
 });
 
 var URLs = {
@@ -96,6 +98,17 @@ var DOMAIN_ID_TYPE_TEMP = 1;
 var DOMAIN_ID_TYPE_FULL = 2;
 var DOMAIN_ID_TYPE_UNKNOWN = 3;
 
+function swalAreYouSure(text, confirmButtonText, callback) {
+  swal({
+    title: "Are you sure?",
+    text: text,
+    type: "warning",
+    showCancelButton: true,
+    confirmButtonText: confirmButtonText,
+    closeOnConfirm: false
+  }, callback);
+}
+
 function domainIDIsSet() {
   if (typeof Settings.data.values.metaverse !== 'undefined' &&
     typeof Settings.data.values.metaverse.id !== 'undefined') {
@@ -164,7 +177,7 @@ function getDomainFromAPI(callback) {
   if (callback === undefined) {
     callback = function() {};
   }
-  
+
   if (!domainIDIsSet()) {
     callback({ status: 'fail' });
     return null;

domain-server/resources/web/settings/js/settings.js

@@ -14,17 +14,8 @@ $(document).ready(function(){
       return b;
   })(window.location.search.substr(1).split('&'));
 
-  // define extra groups to add to description, with their splice index
-  Settings.extraGroups = {
-    1: {
-      html_id: 'places',
-      label: 'Places'
-    },
-    "-1": {
-      html_id: 'settings_backup',
-      label: 'Settings Backup'
-    }
-  }
+  Settings.extraGroupsAtEnd = Settings.extraDomainGroupsAtEnd;
+  Settings.extraGroupsAtIndex = Settings.extraDomainGroupsAtIndex;
 
   Settings.afterReloadActions = function() {
     // append the domain selection modal
@@ -103,20 +94,17 @@ $(document).ready(function(){
       var password = formJSON["security"]["http_password"];
 
       if ((password == sha256_digest("")) && (username == undefined || (username && username.length != 0))) {
-        swal({
-          title: "Are you sure?",
-          text: "You have entered a blank password with a non-blank username. Are you sure you want to require a blank password?",
-          type: "warning",
-          showCancelButton: true,
-          confirmButtonColor: "#5cb85c",
-          confirmButtonText: "Yes!",
-          closeOnConfirm: true
-        },
-        function () {
+        swalAreYouSure(
+          "You have entered a blank password with a non-blank username. Are you sure you want to require a blank password?",
+          "Use blank password",
+          function() {
+            swal.close();
+            
             formJSON["security"]["http_password"] = "";
 
             postSettings(formJSON);
-        });
+          }
+        );
 
         return;
       }
@@ -643,7 +631,6 @@ $(document).ready(function(){
     autoNetworkingEl.after(form);
   }
 
-
   function setupPlacesTable() {
     // create a dummy table using our view helper
     var placesTableSetting = {
@@ -1043,32 +1030,38 @@ $(document).ready(function(){
   $('body').on('click', '#' + RESTORE_SETTINGS_UPLOAD_ID, function(e){
     e.preventDefault();
 
-    var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
+    swalAreYouSure(
+      "Your domain settings will be replaced by the uploaded settings",
+      "Restore settings",
+      function() {
+        var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
 
-    var fileFormData = new FormData();
-    fileFormData.append('restore-file', files[0]);
+        var fileFormData = new FormData();
+        fileFormData.append('restore-file', files[0]);
 
-    showSpinnerAlert("Restoring Settings");
+        showSpinnerAlert("Restoring Settings");
 
-    $.ajax({
-      url: '/settings/restore',
-      type: 'POST',
-      processData: false,
-      contentType: false,
-      dataType: 'json',
-      data: fileFormData
-    }).done(function(data, textStatus, jqXHR) {
-      swal.close();
-      showRestartModal();
-    }).fail(function(jqXHR, textStatus, errorThrown) {
-      showErrorMessage(
-        "Error",
-        "There was a problem restoring domain settings.\n"
-        + "Please ensure that your current domain settings are valid and try again."
-      );
+        $.ajax({
+          url: '/settings/restore',
+          type: 'POST',
+          processData: false,
+          contentType: false,
+          dataType: 'json',
+          data: fileFormData
+        }).done(function(data, textStatus, jqXHR) {
+          swal.close();
+          showRestartModal();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain settings.\n"
+            + "Please ensure that your current domain settings are valid and try again."
+          );
 
-      reloadSettings();
-    });
+          reloadSettings();
+        });
+      }
+    );
   });
 
   $('body').on('change', '#' + RESTORE_SETTINGS_FILE_ID, function() {
@@ -1089,7 +1082,7 @@ $(document).ready(function(){
     html += "<div class='form-group'>";
     html += "<label class='control-label'>Upload a Settings Configuration</label>";
     html += "<span class='help-block'>Upload a settings configuration to quickly configure this domain";
-    html += "<br/>Note: Your domain's settings will be replaced by the settings you upload</span>";
+    html += "<br/>Note: Your domain settings will be replaced by the settings you upload</span>";
 
     html += "<input id='restore-settings-file' name='restore-settings' type='file'>";
     html += "<button type='button' id='" + RESTORE_SETTINGS_UPLOAD_ID + "' disabled='true' class='btn btn-primary'>Upload Domain Settings</button>";
@@ -1097,8 +1090,5 @@ $(document).ready(function(){
     html += "</div>";
 
     $('#settings_backup .panel-body').html(html);
-
-    // add an upload button to the footer to kick off the upload form
-
   }
 });

domain-server/resources/web/wizard/index.shtml

@@ -261,6 +261,5 @@
 <script src='/js/underscore-min.js'></script>
 <script src='/js/bootbox.min.js'></script>
 <script src='/js/sha256.js'></script>
-<script src='/js/shared.js'></script>
 <script src='js/wizard.js'></script>
 <!--#include virtual="page-end.html"-->

domain-server/resources/web/wizard/js/wizard.js

@@ -396,10 +396,12 @@ function savePermissions() {
 
   var admins = $('#admin-usernames').val().split(',');
 
-  var existingAdmins = Settings.data.values.security.permissions.map(function(value) {
-    return value.permissions_id;
-  });
-  admins = admins.concat(existingAdmins);
+  if (Settings.data.values.security.permissions) {
+    var existingAdmins = Settings.data.values.security.permissions.map(function(value) {
+      return value.permissions_id;
+    });
+    admins = admins.concat(existingAdmins);
+  }
 
   // Filter out unique values
   admins = _.uniq(admins.map(function(username) {

domain-server/src/AssetsBackupHandler.cpp

@@ -0,0 +1,605 @@
+//
+//  AssetsBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Clement Brisset on 1/12/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "AssetsBackupHandler.h"
+
+#include <QJsonDocument>
+#include <QDate>
+#include <QtCore/QLoggingCategory>
+
+#include <quazip5/quazipfile.h>
+#include <quazip5/quazipdir.h>
+
+#include <AssetClient.h>
+#include <AssetRequest.h>
+#include <AssetUpload.h>
+#include <MappingRequest.h>
+#include <PathUtils.h>
+
+using namespace std;
+
+static const QString ASSETS_DIR { "/assets/" };
+static const QString MAPPINGS_FILE { "mappings.json" };
+static const QString ZIP_ASSETS_FOLDER { "files" };
+static const chrono::minutes MAX_REFRESH_TIME { 5 };
+
+Q_DECLARE_LOGGING_CATEGORY(asset_backup)
+Q_LOGGING_CATEGORY(asset_backup, "hifi.asset-backup");
+
+AssetsBackupHandler::AssetsBackupHandler(const QString& backupDirectory) :
+    _assetsDirectory(backupDirectory + ASSETS_DIR)
+{
+    // Make sure the asset directory exists.
+    QDir(_assetsDirectory).mkpath(".");
+
+    refreshAssetsOnDisk();
+
+    setupRefreshTimer();
+}
+
+void AssetsBackupHandler::setupRefreshTimer() {
+    _mappingsRefreshTimer.setTimerType(Qt::CoarseTimer);
+    _mappingsRefreshTimer.setSingleShot(true);
+    QObject::connect(&_mappingsRefreshTimer, &QTimer::timeout, this, &AssetsBackupHandler::refreshMappings);
+
+    auto nodeList = DependencyManager::get<LimitedNodeList>();
+    QObject::connect(nodeList.data(), &LimitedNodeList::nodeActivated, this, [this](SharedNodePointer node) {
+        if (node->getType() == NodeType::AssetServer) {
+            // run immediately for the first time.
+            _mappingsRefreshTimer.start(0);
+        }
+    });
+    QObject::connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, [this](SharedNodePointer node) {
+        if (node->getType() == NodeType::AssetServer) {
+            _mappingsRefreshTimer.stop();
+        }
+    });
+}
+
+void AssetsBackupHandler::refreshAssetsOnDisk() {
+    QDir assetsDir { _assetsDirectory };
+    auto assetNames = assetsDir.entryList(QDir::Files);
+
+    // store all valid hashes
+    copy_if(begin(assetNames), end(assetNames),
+            inserter(_assetsOnDisk, begin(_assetsOnDisk)),
+            AssetUtils::isValidHash);
+
+}
+
+void AssetsBackupHandler::refreshAssetsInBackups() {
+    _assetsInBackups.clear();
+    for (const auto& backup : _backups) {
+        for (const auto& mapping : backup.mappings) {
+            _assetsInBackups.insert(mapping.second);
+        }
+    }
+}
+
+void AssetsBackupHandler::checkForMissingAssets() {
+    vector<AssetUtils::AssetHash> missingAssets;
+    set_difference(begin(_assetsInBackups), end(_assetsInBackups),
+                   begin(_assetsOnDisk), end(_assetsOnDisk),
+                   back_inserter(missingAssets));
+    if (missingAssets.size() > 0) {
+        qCWarning(asset_backup) << "Found" << missingAssets.size() << "backup assets missing from disk.";
+    }
+}
+
+void AssetsBackupHandler::checkForAssetsToDelete() {
+    vector<AssetUtils::AssetHash> deprecatedAssets;
+    set_difference(begin(_assetsOnDisk), end(_assetsOnDisk),
+                   begin(_assetsInBackups), end(_assetsInBackups),
+                   back_inserter(deprecatedAssets));
+
+    if (deprecatedAssets.size() > 0) {
+        qCDebug(asset_backup) << "Found" << deprecatedAssets.size() << "backup assets to delete from disk.";
+        const auto noCorruptedBackups = none_of(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+            return backup.corruptedBackup;
+        });
+        if (noCorruptedBackups) {
+            for (const auto& hash : deprecatedAssets) {
+                auto success = QFile::remove(_assetsDirectory + hash);
+                if (success) {
+                    _assetsOnDisk.erase(hash);
+                } else {
+                    qCWarning(asset_backup) << "Could not delete asset:" << hash;
+                }
+            }
+        } else {
+            qCWarning(asset_backup) << "Some backups did not load properly, aborting delete operation for safety.";
+        }
+    }
+}
+
+bool AssetsBackupHandler::isCorruptedBackup(const QString& backupName) {
+    auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& value) {
+        return value.name == backupName;
+    });
+
+    if (it == end(_backups)) {
+        return false;
+    }
+
+    return it->corruptedBackup;
+}
+
+std::pair<bool, float> AssetsBackupHandler::isAvailable(const QString& backupName) {
+    const auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        return { true, 1.0f };
+    }
+
+    int mappingsMissing = 0;
+    for (const auto& mapping : it->mappings) {
+        if (_assetsLeftToRequest.find(mapping.second) != end(_assetsLeftToRequest)) {
+            ++mappingsMissing;
+        }
+    }
+
+    if (mappingsMissing == 0) {
+        return { true, 1.0f };
+    }
+
+    float progress = (float)it->mappings.size();
+    progress -= (float)mappingsMissing;
+    progress /= it->mappings.size();
+
+    return { false, progress };
+}
+
+std::pair<bool, float> AssetsBackupHandler::getRecoveryStatus() {
+    if (_assetsLeftToUpload.empty() &&
+        _mappingsLeftToSet.empty() &&
+        _mappingsLeftToDelete.empty() &&
+        _mappingRequestsInFlight == 0) {
+        return { false, 1.0f };
+    }
+
+    float progress = (float)_numRestoreOperations;
+    progress -= (float)_assetsLeftToUpload.size();
+    progress -= (float)_mappingRequestsInFlight;
+    progress /= (float)_numRestoreOperations;
+
+    return { true, progress };
+}
+
+void AssetsBackupHandler::loadBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    _backups.emplace_back(backupName, AssetUtils::Mappings(), false);
+    auto& backup = _backups.back();
+
+    if (!zip.setCurrentFile(MAPPINGS_FILE)) {
+        qCCritical(asset_backup) << "Failed to find" << MAPPINGS_FILE << "while loading backup";
+        qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QFile::ReadOnly)) {
+        qCCritical(asset_backup) << "Could not unzip backup file for load:" << MAPPINGS_FILE;
+        qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    QJsonParseError error;
+    auto document = QJsonDocument::fromJson(zipFile.readAll(), &error);
+    if (document.isNull() || !document.isObject()) {
+        qCCritical(asset_backup) << "Could not parse backup file to JSON object for load:" << MAPPINGS_FILE;
+        qCCritical(asset_backup) << "    Error:" << error.errorString();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    auto jsonObject = document.object();
+    for (auto it = begin(jsonObject); it != end(jsonObject); ++it) {
+        const auto& assetPath = it.key();
+        const auto& assetHash = it.value().toString();
+
+        if (!AssetUtils::isValidHash(assetHash)) {
+            qCCritical(asset_backup) << "Corrupted mapping in loading backup file" << backupName << ":" << it.key();
+            backup.corruptedBackup = true;
+            continue;
+        }
+
+        backup.mappings[assetPath] = assetHash;
+        _assetsInBackups.insert(assetHash);
+    }
+}
+
+void AssetsBackupHandler::loadingComplete() {
+    checkForMissingAssets();
+    checkForAssetsToDelete();
+}
+
+void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is already an operation in progress.";
+        return;
+    }
+
+    if (_lastMappingsRefresh.time_since_epoch().count() == 0) {
+        qCWarning(asset_backup) << "Current mappings not yet loaded.";
+        return;
+    }
+
+    if ((p_high_resolution_clock::now() - _lastMappingsRefresh) > MAX_REFRESH_TIME) {
+        qCWarning(asset_backup) << "Backing up asset mappings that might be stale.";
+    }
+
+    AssetUtils::Mappings mappings;
+
+    QJsonObject jsonObject;
+    for (const auto& mapping : _currentMappings) {
+        mappings[mapping.first] = mapping.second;
+        _assetsInBackups.insert(mapping.second);
+        jsonObject.insert(mapping.first, mapping.second);
+    }
+    QJsonDocument document(jsonObject);
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(MAPPINGS_FILE))) {
+        qCDebug(asset_backup) << "Could not open zip file:" << zipFile.getZipError();
+        return;
+    }
+    zipFile.write(document.toJson());
+    zipFile.close();
+    if (zipFile.getZipError() != UNZ_OK) {
+        qCDebug(asset_backup) << "Could not close zip file: " << zipFile.getZipError();
+        return;
+    }
+    _backups.emplace_back(backupName, mappings, false);
+    qDebug() << "Created asset backup:" << backupName;
+}
+
+void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is already a backup/restore in progress.";
+        return;
+    }
+
+    if (_lastMappingsRefresh.time_since_epoch().count() == 0) {
+        qCWarning(asset_backup) << "Current mappings not yet loaded.";
+        return;
+    }
+
+    if ((p_high_resolution_clock::now() - _lastMappingsRefresh) > MAX_REFRESH_TIME) {
+        qCWarning(asset_backup) << "Recovering while current asset mappings might be stale.";
+    }
+
+    auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        loadBackup(backupName, zip);
+
+        QuaZipDir zipDir { &zip, ZIP_ASSETS_FOLDER };
+
+        auto assetNames = zipDir.entryList(QDir::Files);
+        for (const auto& asset : assetNames) {
+            if (AssetUtils::isValidHash(asset)) {
+                if (!zip.setCurrentFile(zipDir.filePath(asset))) {
+                    qCCritical(asset_backup) << "Failed to find" << asset << "while recovering backup";
+                    qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+                    continue;
+                }
+
+                QuaZipFile zipFile { &zip };
+                if (!zipFile.open(QFile::ReadOnly)) {
+                    qCCritical(asset_backup) << "Could not unzip asset file:" << asset;
+                    qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+                    continue;
+                }
+
+                writeAssetFile(asset, zipFile.readAll());
+            }
+        }
+
+        // iterator is end() and has been invalidated in the `loadBackup` call
+        // grab the new iterator
+        it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+            return backup.name == backupName;
+        });
+
+        if (it == end(_backups)) {
+            qCCritical(asset_backup) << "Failed to recover backup:" << backupName;
+            return;
+        }
+    }
+
+    const auto& newMappings = it->mappings;
+    computeServerStateDifference(_currentMappings, newMappings);
+
+    restoreAllAssets();
+}
+
+void AssetsBackupHandler::deleteBackup(const QString& backupName) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is a backup/restore in progress.";
+        return;
+    }
+
+    const auto it = remove_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        qCDebug(asset_backup) << "Could not find backup" << backupName << "to delete.";
+        return;
+    }
+
+    _backups.erase(it, end(_backups));
+
+    refreshAssetsInBackups();
+    checkForAssetsToDelete();
+    qDebug() << "Deleted asset backup:" << backupName;
+}
+
+void AssetsBackupHandler::consolidateBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is a backup/restore in progress.";
+        return;
+    }
+
+    const auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        qCDebug(asset_backup) << "Could not find backup" << backupName << "to consolidate.";
+        return;
+    }
+
+    for (const auto& mapping : it->mappings) {
+        const auto& hash = mapping.second;
+
+        QDir assetsDir { _assetsDirectory };
+        QFile file { assetsDir.filePath(hash) };
+        if (!file.open(QFile::ReadOnly)) {
+            qCCritical(asset_backup) << "Could not open asset file" << file.fileName();
+            continue;
+        }
+
+        QuaZipFile zipFile { &zip };
+        if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(ZIP_ASSETS_FOLDER + "/" + hash))) {
+            qCDebug(asset_backup) << "Could not open zip file:" << zipFile.getZipError();
+            continue;
+        }
+        zipFile.write(file.readAll());
+        zipFile.close();
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCDebug(asset_backup) << "Could not close zip file: " << zipFile.getZipError();
+            continue;
+        }
+    }
+
+}
+
+void AssetsBackupHandler::refreshMappings() {
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto request = assetClient->createGetAllMappingsRequest();
+
+    QObject::connect(request, &GetAllMappingsRequest::finished, this, [this](GetAllMappingsRequest* request) {
+        if (request->getError() == MappingRequest::NoError) {
+            const auto& mappings = request->getMappings();
+
+            // Clear existing mappings
+            _currentMappings.clear();
+
+            // Set new mapping, but ignore baked assets
+            for (const auto& mapping : mappings) {
+                if (!mapping.first.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
+                    _currentMappings.insert({ mapping.first, mapping.second.hash });
+                }
+            }
+            _lastMappingsRefresh = p_high_resolution_clock::now();
+
+            downloadMissingFiles(_currentMappings);
+        } else {
+            qCCritical(asset_backup) << "Could not refresh asset server mappings.";
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        request->deleteLater();
+
+        // Launch next mappings request
+        static constexpr int MAPPINGS_REFRESH_INTERVAL = 30 * 1000;
+        _mappingsRefreshTimer.start(MAPPINGS_REFRESH_INTERVAL);
+    });
+
+    request->start();
+}
+
+void AssetsBackupHandler::downloadMissingFiles(const AssetUtils::Mappings& mappings) {
+    auto wasEmpty = _assetsLeftToRequest.empty();
+
+    for (const auto& mapping : mappings) {
+        const auto& hash = mapping.second;
+        if (_assetsOnDisk.find(hash) == end(_assetsOnDisk)) {
+            _assetsLeftToRequest.insert(hash);
+        }
+    }
+
+    // If we were empty, that means no download chain was already going, start one.
+    if (wasEmpty) {
+        downloadNextMissingFile();
+    }
+}
+
+void AssetsBackupHandler::downloadNextMissingFile() {
+    if (_assetsLeftToRequest.empty()) {
+        return;
+    }
+    auto hash = *begin(_assetsLeftToRequest);
+
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto assetRequest = assetClient->createRequest(hash);
+
+    QObject::connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) {
+        if (request->getError() == AssetRequest::NoError) {
+            qCDebug(asset_backup) << "Backing up asset" << request->getHash();
+
+            bool success = writeAssetFile(request->getHash(), request->getData());
+            if (!success) {
+                qCCritical(asset_backup) << "Failed to write asset file" << request->getHash();
+            }
+        } else {
+            qCCritical(asset_backup) << "Failed to backup asset" << request->getHash();
+        }
+
+        _assetsLeftToRequest.erase(request->getHash());
+        downloadNextMissingFile();
+
+        request->deleteLater();
+    });
+
+    assetRequest->start();
+}
+
+bool AssetsBackupHandler::writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data) {
+    QDir assetsDir { _assetsDirectory };
+    QFile file { assetsDir.filePath(hash) };
+    if (!file.open(QFile::WriteOnly)) {
+        qCCritical(asset_backup) << "Could not open asset file for write:" << file.fileName();
+        return false;
+    }
+
+    auto bytesWritten = file.write(data);
+    if (bytesWritten != data.size()) {
+        qCCritical(asset_backup) << "Could not write data to file" << file.fileName();
+        file.remove();
+        return false;
+    }
+
+    _assetsOnDisk.insert(hash);
+
+    return true;
+}
+
+void AssetsBackupHandler::computeServerStateDifference(const AssetUtils::Mappings& currentMappings,
+                                                       const AssetUtils::Mappings& newMappings) {
+    _mappingsLeftToSet.reserve((int)newMappings.size());
+    _assetsLeftToUpload.reserve((int)newMappings.size());
+    _mappingsLeftToDelete.reserve((int)currentMappings.size());
+
+    set<AssetUtils::AssetHash> currentAssets;
+    for (const auto& currentMapping : currentMappings) {
+        const auto& currentPath = currentMapping.first;
+        const auto& currentHash = currentMapping.second;
+
+        if (newMappings.find(currentPath) == end(newMappings)) {
+            _mappingsLeftToDelete.push_back(currentPath);
+        }
+        currentAssets.insert(currentHash);
+    }
+
+    for (const auto& newMapping : newMappings) {
+        const auto& newPath = newMapping.first;
+        const auto& newHash = newMapping.second;
+
+        auto it = currentMappings.find(newPath);
+        if (it == end(currentMappings) || it->second != newHash) {
+            _mappingsLeftToSet.push_back({ newPath, newHash });
+        }
+        if (currentAssets.find(newHash) == end(currentAssets)) {
+            _assetsLeftToUpload.push_back(newHash);
+        }
+    }
+
+    _numRestoreOperations = (int)_assetsLeftToUpload.size() + (int)_mappingsLeftToSet.size();
+    if (!_mappingsLeftToDelete.empty()) {
+        ++_numRestoreOperations;
+    }
+
+    qCDebug(asset_backup) << "Mappings to set:" << _mappingsLeftToSet.size();
+    qCDebug(asset_backup) << "Mappings to del:" << _mappingsLeftToDelete.size();
+    qCDebug(asset_backup) << "Assets to upload:" << _assetsLeftToUpload.size();
+}
+
+void AssetsBackupHandler::restoreAllAssets() {
+    restoreNextAsset();
+}
+
+void AssetsBackupHandler::restoreNextAsset() {
+    if (_assetsLeftToUpload.empty()) {
+        updateMappings();
+        return;
+    }
+
+    auto hash = _assetsLeftToUpload.back();
+    _assetsLeftToUpload.pop_back();
+
+    auto assetFilename = _assetsDirectory + hash;
+
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto request = assetClient->createUpload(assetFilename);
+
+    QObject::connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) {
+        if (request->getError() != AssetUpload::NoError) {
+            qCCritical(asset_backup) << "Failed to restore asset:" << request->getFilename();
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        restoreNextAsset();
+
+        request->deleteLater();
+    });
+
+    request->start();
+}
+
+void AssetsBackupHandler::updateMappings() {
+    auto assetClient = DependencyManager::get<AssetClient>();
+    for (const auto& mapping : _mappingsLeftToSet) {
+        auto request = assetClient->createSetMappingRequest(mapping.first, mapping.second);
+        QObject::connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) {
+            if (request->getError() != MappingRequest::NoError) {
+                qCCritical(asset_backup) << "Failed to set mapping:" << request->getPath();
+                qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+            }
+
+            --_mappingRequestsInFlight;
+
+            request->deleteLater();
+        });
+
+        request->start();
+        ++_mappingRequestsInFlight;
+    }
+    _mappingsLeftToSet.clear();
+
+    auto request = assetClient->createDeleteMappingsRequest(_mappingsLeftToDelete);
+    QObject::connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) {
+        if (request->getError() != MappingRequest::NoError) {
+            qCCritical(asset_backup) << "Failed to delete mappings";
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        --_mappingRequestsInFlight;
+
+        request->deleteLater();
+    });
+    _mappingsLeftToDelete.clear();
+
+    request->start();
+    ++_mappingRequestsInFlight;
+}

domain-server/src/AssetsBackupHandler.h

@@ -0,0 +1,98 @@
+//
+//  AssetsBackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 1/12/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AssetsBackupHandler_h
+#define hifi_AssetsBackupHandler_h
+
+#include <set>
+#include <map>
+
+#include <QObject>
+#include <QTimer>
+#include <QJsonDocument>
+#include <QJsonObject>
+
+#include <AssetUtils.h>
+#include <ReceivedMessage.h>
+#include <PortableHighResolutionClock.h>
+
+#include "BackupHandler.h"
+
+class AssetsBackupHandler : public QObject, public BackupHandlerInterface {
+    Q_OBJECT
+
+public:
+    AssetsBackupHandler(const QString& backupDirectory);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override;
+    std::pair<bool, float> getRecoveryStatus() override;
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override;
+    void loadingComplete() override;
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+    void deleteBackup(const QString& backupName) override;
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override;
+    bool isCorruptedBackup(const QString& backupName) override;
+
+    bool operationInProgress() { return getRecoveryStatus().first; }
+
+private:
+    void setupRefreshTimer();
+    void refreshMappings();
+
+    void refreshAssetsInBackups();
+    void refreshAssetsOnDisk();
+    void checkForMissingAssets();
+    void checkForAssetsToDelete();
+
+    void downloadMissingFiles(const AssetUtils::Mappings& mappings);
+    void downloadNextMissingFile();
+    bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data);
+
+    void computeServerStateDifference(const AssetUtils::Mappings& currentMappings,
+                                      const AssetUtils::Mappings& newMappings);
+    void restoreAllAssets();
+    void restoreNextAsset();
+    void updateMappings();
+
+    QString _assetsDirectory;
+
+    QTimer _mappingsRefreshTimer;
+    p_high_resolution_clock::time_point _lastMappingsRefresh;
+    AssetUtils::Mappings _currentMappings;
+
+    struct AssetServerBackup {
+        AssetServerBackup(const QString& pName, AssetUtils::Mappings pMappings, bool pCorruptedBackup) :
+            name(pName), mappings(pMappings), corruptedBackup(pCorruptedBackup) {}
+
+        QString name;
+        AssetUtils::Mappings mappings;
+        bool corruptedBackup;
+    };
+
+    // Internal storage for backups on disk
+    std::vector<AssetServerBackup> _backups;
+    std::set<AssetUtils::AssetHash> _assetsInBackups;
+    std::set<AssetUtils::AssetHash> _assetsOnDisk;
+
+    // Internal storage for backup in progress
+    std::set<AssetUtils::AssetHash> _assetsLeftToRequest;
+
+    // Internal storage for restore in progress
+    std::vector<AssetUtils::AssetHash> _assetsLeftToUpload;
+    std::vector<std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash>> _mappingsLeftToSet;
+    AssetUtils::AssetPathList _mappingsLeftToDelete;
+    int _mappingRequestsInFlight { 0 };
+    int _numRestoreOperations { 0 }; // Used to compute a restore progress.
+};
+
+#endif /* hifi_AssetsBackupHandler_h */

domain-server/src/BackupHandler.h

@@ -0,0 +1,40 @@
+//
+//  BackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/5/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_BackupHandler_h
+#define hifi_BackupHandler_h
+
+#include <memory>
+
+#include <QString>
+
+class QuaZip;
+
+class BackupHandlerInterface {
+public:
+    virtual ~BackupHandlerInterface() = default;
+
+    virtual std::pair<bool, float> isAvailable(const QString& backupName) = 0;
+
+    // Returns whether a recovery is ongoing and a progress between 0 and 1 if one is.
+    virtual std::pair<bool, float> getRecoveryStatus() = 0;
+
+    virtual void loadBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void loadingComplete() = 0;
+    virtual void createBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void recoverBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void deleteBackup(const QString& backupName) = 0;
+    virtual void consolidateBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual bool isCorruptedBackup(const QString& backupName) = 0;
+};
+using BackupHandlerPointer = std::unique_ptr<BackupHandlerInterface>;
+
+#endif /* hifi_BackupHandler_h */

domain-server/src/BackupSupervisor.cpp

@@ -1,400 +0,0 @@
-//
-//  BackupSupervisor.cpp
-//  domain-server/src
-//
-//  Created by Clement Brisset on 1/12/18.
-//  Copyright 2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#include "BackupSupervisor.h"
-
-#include <QJsonDocument>
-#include <QDate>
-
-#include <AssetClient.h>
-#include <AssetRequest.h>
-#include <AssetUpload.h>
-#include <MappingRequest.h>
-#include <PathUtils.h>
-
-const QString BACKUPS_DIR = "backups/";
-const QString ASSETS_DIR = "files/";
-const QString MAPPINGS_PREFIX = "mappings-";
-
-using namespace std;
-
-BackupSupervisor::BackupSupervisor() {
-    _backupsDirectory = PathUtils::getAppDataPath() + BACKUPS_DIR;
-    QDir backupDir { _backupsDirectory };
-    if (!backupDir.exists()) {
-        backupDir.mkpath(".");
-    }
-
-    _assetsDirectory = PathUtils::getAppDataPath() + BACKUPS_DIR + ASSETS_DIR;
-    QDir assetsDir { _assetsDirectory };
-    if (!assetsDir.exists()) {
-        assetsDir.mkpath(".");
-    }
-
-    loadAllBackups();
-}
-
-void BackupSupervisor::loadAllBackups() {
-    _backups.clear();
-    _assetsInBackups.clear();
-    _assetsOnDisk.clear();
-    _allBackupsLoadedSuccessfully = true;
-
-    QDir assetsDir { _assetsDirectory };
-    auto assetNames = assetsDir.entryList(QDir::Files);
-    qDebug() << "Loading" << assetNames.size() << "assets.";
-
-    // store all valid hashes
-    copy_if(begin(assetNames), end(assetNames),
-            inserter(_assetsOnDisk, begin(_assetsOnDisk)), AssetUtils::isValidHash);
-
-    QDir backupsDir { _backupsDirectory };
-    auto files = backupsDir.entryList({ MAPPINGS_PREFIX + "*.json" }, QDir::Files);
-    qDebug() << "Loading" << files.size() << "backups.";
-
-    for (const auto& fileName : files) {
-        auto filePath = backupsDir.filePath(fileName);
-        auto success = loadBackup(filePath);
-        if (!success) {
-            qCritical() << "Failed to load backup file" << filePath;
-            _allBackupsLoadedSuccessfully = false;
-        }
-    }
-
-    vector<AssetUtils::AssetHash> missingAssets;
-    set_difference(begin(_assetsInBackups), end(_assetsInBackups),
-                   begin(_assetsOnDisk), end(_assetsOnDisk),
-                   back_inserter(missingAssets));
-    if (missingAssets.size() > 0) {
-        qWarning() << "Found" << missingAssets.size() << "assets missing.";
-    }
-
-    vector<AssetUtils::AssetHash> deprecatedAssets;
-    set_difference(begin(_assetsOnDisk), end(_assetsOnDisk),
-                   begin(_assetsInBackups), end(_assetsInBackups),
-                   back_inserter(deprecatedAssets));
-
-    if (deprecatedAssets.size() > 0) {
-        qDebug() << "Found" << deprecatedAssets.size() << "assets to delete.";
-        if (_allBackupsLoadedSuccessfully) {
-            for (const auto& hash : deprecatedAssets) {
-                QFile::remove(_assetsDirectory + hash);
-            }
-        } else {
-            qWarning() << "Some backups did not load properly, aborting deleting for safety.";
-        }
-    }
-}
-
-bool BackupSupervisor::loadBackup(const QString& backupFile) {
-    _backups.push_back({ backupFile.toStdString(), {}, false });
-    auto& backup = _backups.back();
-
-    QFile file { backupFile };
-    if (!file.open(QFile::ReadOnly)) {
-        qCritical() << "Could not open backup file:" << backupFile;
-        backup.corruptedBackup = true;
-        return false;
-    }
-    QJsonParseError error;
-    auto document = QJsonDocument::fromJson(file.readAll(), &error);
-    if (document.isNull() || !document.isObject()) {
-        qCritical() << "Could not parse backup file to JSON object:" << backupFile;
-        qCritical() << "    Error:" << error.errorString();
-        backup.corruptedBackup = true;
-        return false;
-    }
-
-    auto jsonObject = document.object();
-    for (auto it = begin(jsonObject); it != end(jsonObject); ++it) {
-        const auto& assetPath = it.key();
-        const auto& assetHash = it.value().toString();
-
-        if (!AssetUtils::isValidHash(assetHash)) {
-            qCritical() << "Corrupted mapping in backup file" << backupFile << ":" << it.key();
-            backup.corruptedBackup = true;
-            return false;
-        }
-
-        backup.mappings[assetPath] = assetHash;
-        _assetsInBackups.insert(assetHash);
-    }
-
-    _backups.push_back(backup);
-    return true;
-}
-
-void BackupSupervisor::backupAssetServer() {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is already a backup/restore in progress.";
-        return;
-    }
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createGetAllMappingsRequest();
-
-    connect(request, &GetAllMappingsRequest::finished, this, [this](GetAllMappingsRequest* request) {
-        qDebug() << "Got" << request->getMappings().size() << "mappings!";
-
-        if (request->getError() != MappingRequest::NoError) {
-            qCritical() << "Could not complete backup.";
-            qCritical() << "    Error:" << request->getErrorString();
-            finishBackup();
-            request->deleteLater();
-            return;
-        }
-
-        if (!writeBackupFile(request->getMappings())) {
-            finishBackup();
-            request->deleteLater();
-            return;
-        }
-
-        assert(!_backups.empty());
-        const auto& mappings = _backups.back().mappings;
-        backupMissingFiles(mappings);
-
-        request->deleteLater();
-    });
-
-    startBackup();
-    request->start();
-}
-
-void BackupSupervisor::backupMissingFiles(const AssetUtils::Mappings& mappings) {
-    _assetsLeftToRequest.reserve(mappings.size());
-    for (auto& mapping : mappings) {
-        const auto& hash = mapping.second;
-        if (_assetsOnDisk.find(hash) == end(_assetsOnDisk)) {
-            _assetsLeftToRequest.push_back(hash);
-        }
-    }
-
-    backupNextMissingFile();
-}
-
-void BackupSupervisor::backupNextMissingFile() {
-    if (_assetsLeftToRequest.empty()) {
-        finishBackup();
-        return;
-    }
-
-    auto hash = _assetsLeftToRequest.back();
-    _assetsLeftToRequest.pop_back();
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto assetRequest = assetClient->createRequest(hash);
-
-    connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) {
-        if (request->getError() == AssetRequest::NoError) {
-            qDebug() << "Got" << request->getHash();
-
-            bool success = writeAssetFile(request->getHash(), request->getData());
-            if (!success) {
-                qCritical() << "Failed to write asset file" << request->getHash();
-            }
-        } else {
-            qCritical() << "Failed to backup asset" << request->getHash();
-        }
-
-        backupNextMissingFile();
-
-        request->deleteLater();
-    });
-
-    assetRequest->start();
-}
-
-bool BackupSupervisor::writeBackupFile(const AssetUtils::AssetMappings& mappings) {
-    auto filename = MAPPINGS_PREFIX + QDateTime::currentDateTimeUtc().toString(Qt::ISODate) + ".json";
-    QFile file { PathUtils::getAppDataPath() + BACKUPS_DIR + filename };
-    if (!file.open(QFile::WriteOnly)) {
-        qCritical() << "Could not open backup file" << file.fileName();
-        return false;
-    }
-
-    AssetServerBackup backup;
-    QJsonObject jsonObject;
-    for (auto& mapping : mappings) {
-        backup.mappings[mapping.first] = mapping.second.hash;
-        _assetsInBackups.insert(mapping.second.hash);
-        jsonObject.insert(mapping.first, mapping.second.hash);
-    }
-
-    QJsonDocument document(jsonObject);
-    file.write(document.toJson());
-
-    backup.filePath = file.fileName().toStdString();
-    _backups.push_back(backup);
-
-    return true;
-}
-
-bool BackupSupervisor::writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data) {
-    QDir assetsDir { _assetsDirectory };
-    QFile file { assetsDir.filePath(hash) };
-    if (!file.open(QFile::WriteOnly)) {
-        qCritical() << "Could not open backup file" << file.fileName();
-        return false;
-    }
-
-    file.write(data);
-
-    _assetsOnDisk.insert(hash);
-
-    return true;
-}
-
-void BackupSupervisor::restoreAssetServer(int backupIndex) {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is already a backup/restore in progress.";
-        return;
-    }
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createGetAllMappingsRequest();
-
-    connect(request, &GetAllMappingsRequest::finished, this, [this, backupIndex](GetAllMappingsRequest* request) {
-        if (request->getError() == MappingRequest::NoError) {
-            const auto& newMappings = _backups.at(backupIndex).mappings;
-            computeServerStateDifference(request->getMappings(), newMappings);
-
-            restoreAllAssets();
-        } else {
-            finishRestore();
-        }
-
-        request->deleteLater();
-    });
-
-    startRestore();
-    request->start();
-}
-
-void BackupSupervisor::computeServerStateDifference(const AssetUtils::AssetMappings& currentMappings,
-                                                    const AssetUtils::Mappings& newMappings) {
-    _mappingsLeftToSet.reserve((int)newMappings.size());
-    _assetsLeftToUpload.reserve((int)newMappings.size());
-    _mappingsLeftToDelete.reserve((int)currentMappings.size());
-
-    set<AssetUtils::AssetHash> currentAssets;
-    for (const auto& currentMapping : currentMappings) {
-        const auto& currentPath = currentMapping.first;
-        const auto& currentHash = currentMapping.second.hash;
-
-        if (newMappings.find(currentPath) == end(newMappings)) {
-            _mappingsLeftToDelete.push_back(currentPath);
-        }
-        currentAssets.insert(currentHash);
-    }
-
-    for (const auto& newMapping : newMappings) {
-        const auto& newPath = newMapping.first;
-        const auto& newHash = newMapping.second;
-
-        auto it = currentMappings.find(newPath);
-        if (it == end(currentMappings) || it->second.hash != newHash) {
-            _mappingsLeftToSet.push_back({ newPath, newHash });
-        }
-        if (currentAssets.find(newHash) == end(currentAssets)) {
-            _assetsLeftToUpload.push_back(newHash);
-        }
-    }
-
-    qDebug() << "Mappings to set:" << _mappingsLeftToSet.size();
-    qDebug() << "Mappings to del:" << _mappingsLeftToDelete.size();
-    qDebug() << "Assets to upload:" << _assetsLeftToUpload.size();
-}
-
-void BackupSupervisor::restoreAllAssets() {
-    restoreNextAsset();
-}
-
-void BackupSupervisor::restoreNextAsset() {
-    if (_assetsLeftToUpload.empty()) {
-        updateMappings();
-        return;
-    }
-
-    auto hash = _assetsLeftToUpload.back();
-    _assetsLeftToUpload.pop_back();
-
-    auto assetFilename = _assetsDirectory + hash;
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createUpload(assetFilename);
-
-    connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) {
-        if (request->getError() != AssetUpload::NoError) {
-            qCritical() << "Failed to restore asset:" << request->getFilename();
-            qCritical() << "    Error:" << request->getErrorString();
-        }
-
-        restoreNextAsset();
-
-        request->deleteLater();
-    });
-
-    request->start();
-}
-
-void BackupSupervisor::updateMappings() {
-    auto assetClient = DependencyManager::get<AssetClient>();
-    for (const auto& mapping : _mappingsLeftToSet) {
-        auto request = assetClient->createSetMappingRequest(mapping.first, mapping.second);
-        connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) {
-            if (request->getError() != MappingRequest::NoError) {
-                qCritical() << "Failed to set mapping:" << request->getPath();
-                qCritical() << "    Error:" << request->getErrorString();
-            }
-
-            if (--_mappingRequestsInFlight == 0) {
-                finishRestore();
-            }
-
-            request->deleteLater();
-        });
-
-        request->start();
-        ++_mappingRequestsInFlight;
-    }
-    _mappingsLeftToSet.clear();
-
-    auto request = assetClient->createDeleteMappingsRequest(_mappingsLeftToDelete);
-    connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) {
-        if (request->getError() != MappingRequest::NoError) {
-            qCritical() << "Failed to delete mappings";
-            qCritical() << "    Error:" << request->getErrorString();
-        }
-
-        if (--_mappingRequestsInFlight == 0) {
-            finishRestore();
-        }
-
-        request->deleteLater();
-    });
-    _mappingsLeftToDelete.clear();
-
-    request->start();
-    ++_mappingRequestsInFlight;
-}
-bool BackupSupervisor::deleteBackup(int backupIndex) {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is a backup/restore in progress.";
-        return false;
-    }
-    const auto& filePath = _backups.at(backupIndex).filePath;
-    auto success = QFile::remove(filePath.c_str());
-
-    loadAllBackups();
-
-    return success;
-}

domain-server/src/BackupSupervisor.h

@@ -1,85 +0,0 @@
-//
-//  BackupSupervisor.h
-//  domain-server/src
-//
-//  Created by Clement Brisset on 1/12/18.
-//  Copyright 2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#ifndef hifi_BackupSupervisor_h
-#define hifi_BackupSupervisor_h
-
-#include <set>
-#include <map>
-
-#include <QObject>
-
-#include <AssetUtils.h>
-
-#include <ReceivedMessage.h>
-
-struct AssetServerBackup {
-    std::string filePath;
-    AssetUtils::Mappings mappings;
-    bool corruptedBackup;
-};
-
-class BackupSupervisor : public QObject {
-    Q_OBJECT
-
-public:
-    BackupSupervisor();
-
-    void backupAssetServer();
-    void restoreAssetServer(int backupIndex);
-    bool deleteBackup(int backupIndex);
-
-    const std::vector<AssetServerBackup>& getBackups() const { return _backups; };
-
-    bool backupInProgress() const { return _backupInProgress; }
-    bool restoreInProgress() const { return _restoreInProgress; }
-
-private:
-    void loadAllBackups();
-    bool loadBackup(const QString& backupFile);
-
-    void startBackup() { _backupInProgress = true; }
-    void finishBackup() { _backupInProgress = false; }
-    void backupMissingFiles(const AssetUtils::Mappings& mappings);
-    void backupNextMissingFile();
-    bool writeBackupFile(const AssetUtils::AssetMappings& mappings);
-    bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data);
-
-    void startRestore() { _restoreInProgress = true; }
-    void finishRestore() { _restoreInProgress = false; }
-    void computeServerStateDifference(const AssetUtils::AssetMappings& currentMappings,
-                                      const AssetUtils::Mappings& newMappings);
-    void restoreAllAssets();
-    void restoreNextAsset();
-    void updateMappings();
-
-    QString _backupsDirectory;
-    QString _assetsDirectory;
-
-    // Internal storage for backups on disk
-    bool _allBackupsLoadedSuccessfully { false };
-    std::vector<AssetServerBackup> _backups;
-    std::set<AssetUtils::AssetHash> _assetsInBackups;
-    std::set<AssetUtils::AssetHash> _assetsOnDisk;
-
-    // Internal storage for backup in progress
-    bool _backupInProgress { false };
-    std::vector<AssetUtils::AssetHash> _assetsLeftToRequest;
-
-    // Internal storage for restore in progress
-    bool _restoreInProgress { false };
-    std::vector<AssetUtils::AssetHash> _assetsLeftToUpload;
-    std::vector<std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash>> _mappingsLeftToSet;
-    AssetUtils::AssetPathList _mappingsLeftToDelete;
-    int _mappingRequestsInFlight { 0 };
-};
-
-#endif /* hifi_BackupSupervisor_h */

domain-server/src/ContentSettingsBackupHandler.cpp

@@ -0,0 +1,74 @@
+//
+//  ContentSettingsBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Stephen Birarda on 2/15/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "ContentSettingsBackupHandler.h"
+
+#include <quazip5/quazip.h>
+#include <quazip5/quazipfile.h>
+
+ContentSettingsBackupHandler::ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager) :
+    _settingsManager(domainServerSettingsManager)
+{
+}
+
+static const QString CONTENT_SETTINGS_BACKUP_FILENAME = "content-settings.json";
+
+void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+
+    // grab the content settings as JSON, excluding default values and values hidden from backup
+    QJsonObject contentSettingsJSON = _settingsManager.settingsResponseObjectForType(
+        "", // include all settings types
+        DomainServerSettingsManager::Authenticated, DomainServerSettingsManager::NoDomainSettings,
+        DomainServerSettingsManager::IncludeContentSettings, DomainServerSettingsManager::NoDefaultSettings,
+        DomainServerSettingsManager::ForBackup
+    );
+
+    // make a QJsonDocument using the object
+    QJsonDocument contentSettingsDocument { contentSettingsJSON };
+
+    QuaZipFile zipFile { &zip };
+
+    if (zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(CONTENT_SETTINGS_BACKUP_FILENAME))) {
+        if (zipFile.write(contentSettingsDocument.toJson()) == -1) {
+            qCritical().nospace() << "Failed to write to " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+
+        zipFile.close();
+
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCritical().nospace() << "Failed to zip " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+    } else {
+        qCritical().nospace() << "Failed to open " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+    }
+}
+
+void ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    if (!zip.setCurrentFile(CONTENT_SETTINGS_BACKUP_FILENAME)) {
+        qWarning() << "Failed to find" << CONTENT_SETTINGS_BACKUP_FILENAME << "while recovering backup";
+        return;
+    }
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::ReadOnly)) {
+        qCritical() << "Failed to open" << CONTENT_SETTINGS_BACKUP_FILENAME << "in backup";
+        return;
+    }
+
+    auto rawData = zipFile.readAll();
+    zipFile.close();
+
+    QJsonDocument jsonDocument = QJsonDocument::fromJson(rawData);
+
+    if (!_settingsManager.restoreSettingsFromObject(jsonDocument.object(), ContentSettings)) {
+        qCritical() << "Failed to restore settings from" << CONTENT_SETTINGS_BACKUP_FILENAME << "in content archive";
+    }
+}

domain-server/src/ContentSettingsBackupHandler.h

@@ -0,0 +1,43 @@
+//
+//  ContentSettingsBackupHandler.h
+//  domain-server/src
+//
+//  Created by Stephen Birarda on 2/15/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_ContentSettingsBackupHandler_h
+#define hifi_ContentSettingsBackupHandler_h
+
+#include "BackupHandler.h"
+#include "DomainServerSettingsManager.h"
+
+class ContentSettingsBackupHandler : public BackupHandlerInterface {
+public:
+    ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override { return { true, 1.0f }; }
+    std::pair<bool, float> getRecoveryStatus() override { return { false, 1.0f }; }
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override {}
+
+    void loadingComplete() override {}
+
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+
+    void deleteBackup(const QString& backupName) override {}
+
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override {}
+
+    bool isCorruptedBackup(const QString& backupName) override { return false; }
+
+private:
+    DomainServerSettingsManager& _settingsManager;
+};
+
+#endif // hifi_ContentSettingsBackupHandler_h

domain-server/src/DomainContentBackupManager.cpp

@@ -0,0 +1,599 @@
+//
+//  DomainContentBackupManager.cpp
+//  libraries/domain-server/src
+//
+//  Created by Ryan Huffman on 1/01/18.
+//  Adapted from OctreePersistThread
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "DomainContentBackupManager.h"
+
+#include <chrono>
+#include <thread>
+
+#include <cstdio>
+#include <fstream>
+#include <time.h>
+
+#include <QBuffer>
+#include <QDateTime>
+#include <QDebug>
+#include <QDir>
+#include <QDirIterator>
+#include <QFile>
+#include <QJsonArray>
+#include <QJsonObject>
+#include <QJsonDocument>
+
+#include <quazip5/quazip.h>
+
+#include <NumericalConstants.h>
+#include <PerfStat.h>
+#include <PathUtils.h>
+#include <shared/QtHelpers.h>
+
+#include "DomainServer.h"
+
+const std::chrono::seconds DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL { 30 };
+
+// Backup format looks like: daily_backup-TIMESTAMP.zip
+static const QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
+static const QString DATETIME_FORMAT_RE { "\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}" };
+static const QString AUTOMATIC_BACKUP_PREFIX { "autobackup-" };
+static const QString MANUAL_BACKUP_PREFIX { "backup-" };
+static const QString MANUAL_BACKUP_NAME_RE { "[a-zA-Z0-9\\-_ ]+" };
+
+void DomainContentBackupManager::addBackupHandler(BackupHandlerPointer handler) {
+    _backupHandlers.push_back(std::move(handler));
+}
+
+DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory,
+                                                       const QVariantList& backupRules,
+                                                       std::chrono::milliseconds persistInterval,
+                                                       bool debugTimestampNow) :
+    _backupDirectory(backupDirectory), _persistInterval(persistInterval), _lastCheck(p_high_resolution_clock::now())
+{
+
+    setObjectName("DomainContentBackupManager");
+
+    // Make sure the backup directory exists.
+    QDir(_backupDirectory).mkpath(".");
+
+    parseBackupRules(backupRules);
+}
+
+void DomainContentBackupManager::parseBackupRules(const QVariantList& backupRules) {
+    qCDebug(domain_server) << "BACKUP RULES:";
+
+    for (const QVariant& value : backupRules) {
+        QVariantMap map = value.toMap();
+
+        int interval = map["backupInterval"].toInt();
+        int count = map["maxBackupVersions"].toInt();
+        auto name = map["Name"].toString();
+        auto format = name.toLower();
+        QRegExp matchDisallowedCharacters { "[^a-zA-Z0-9\\-_]+" };
+        format.replace(matchDisallowedCharacters, "_");
+
+        qCDebug(domain_server) << "    Name:" << name;
+        qCDebug(domain_server) << "        format:" << format;
+        qCDebug(domain_server) << "        interval:" << interval;
+        qCDebug(domain_server) << "        count:" << count;
+
+        BackupRule newRule = { name, interval, format, count, 0 };
+
+        newRule.lastBackupSeconds = getMostRecentBackupTimeInSecs(format);
+
+        if (newRule.lastBackupSeconds > 0) {
+            auto now = QDateTime::currentSecsSinceEpoch();
+            auto sinceLastBackup = now - newRule.lastBackupSeconds;
+            qCDebug(domain_server).noquote() << "        lastBackup:" <<  formatSecTime(sinceLastBackup) << "ago";
+        } else {
+            qCDebug(domain_server) << "        lastBackup: NEVER";
+        }
+
+        _backupRules.push_back(newRule);
+    }
+}
+
+void DomainContentBackupManager::refreshBackupRules() {
+    for (auto& backup : _backupRules) {
+        backup.lastBackupSeconds = getMostRecentBackupTimeInSecs(backup.extensionFormat);
+    }
+}
+
+int64_t DomainContentBackupManager::getMostRecentBackupTimeInSecs(const QString& format) {
+    int64_t mostRecentBackupInSecs = 0;
+
+    QString mostRecentBackupFileName;
+    QDateTime mostRecentBackupTime;
+
+    bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
+
+    if (recentBackup) {
+        mostRecentBackupInSecs = mostRecentBackupTime.toSecsSinceEpoch();
+    }
+
+    return mostRecentBackupInSecs;
+}
+
+void DomainContentBackupManager::setup() {
+    auto backups = getAllBackups();
+    for (auto& backup : backups) {
+        QFile backupFile { backup.absolutePath };
+        if (!backupFile.open(QIODevice::ReadOnly)) {
+            qCritical() << "Could not open file:" << backup.absolutePath;
+            qCritical() << "    ERROR:" << backupFile.errorString();
+            continue;
+        }
+
+        QuaZip zip { &backupFile };
+        if (!zip.open(QuaZip::mdUnzip)) {
+            qCritical() << "Could not open backup archive:" << backup.absolutePath;
+            qCritical() << "    ERROR:" << zip.getZipError();
+            continue;
+        }
+
+        for (auto& handler : _backupHandlers) {
+            handler->loadBackup(backup.id, zip);
+        }
+
+        zip.close();
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->loadingComplete();
+    }
+}
+
+bool DomainContentBackupManager::process() {
+    if (isStillRunning()) {
+        constexpr int64_t MSECS_TO_USECS = 1000;
+        constexpr int64_t USECS_TO_SLEEP = 10 * MSECS_TO_USECS;  // every 10ms
+        std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
+
+        if (_isRecovering) {
+            bool isStillRecovering = any_of(begin(_backupHandlers), end(_backupHandlers), [](const BackupHandlerPointer& handler) {
+                return handler->getRecoveryStatus().first;
+            });
+
+            if (!isStillRecovering) {
+                _isRecovering = false;
+                _recoveryFilename = "";
+                emit recoveryCompleted();
+            }
+        }
+
+        auto now = p_high_resolution_clock::now();
+        auto sinceLastSave = now - _lastCheck;
+        if (sinceLastSave > _persistInterval) {
+            _lastCheck = now;
+
+            if (!_isRecovering) {
+                backup();
+            }
+        }
+    }
+
+    return isStillRunning();
+}
+
+void DomainContentBackupManager::shutdown() {
+    // Destroy handlers on the correct thread so that they can cleanup timers
+    _backupHandlers.clear();
+}
+
+void DomainContentBackupManager::aboutToFinish() {
+    _stopThread = true;
+}
+
+bool DomainContentBackupManager::getMostRecentBackup(const QString& format,
+                                                     QString& mostRecentBackupFileName,
+                                                     QDateTime& mostRecentBackupTime) {
+    QRegExp formatRE { AUTOMATIC_BACKUP_PREFIX + QRegExp::escape(format) + "\\-(" + DATETIME_FORMAT_RE + ")" + "\\.zip" };
+
+    QStringList filters;
+    filters << AUTOMATIC_BACKUP_PREFIX + format + "*.zip";
+
+    bool bestBackupFound = false;
+    QString bestBackupFile;
+    QDateTime bestBackupFileTime;
+
+    // Iterate over all of the backup files in the persist location
+    QDirIterator dirIterator(_backupDirectory, filters, QDir::Files | QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
+    while (dirIterator.hasNext()) {
+        dirIterator.next();
+        auto fileName = dirIterator.fileInfo().fileName();
+
+        if (formatRE.exactMatch(fileName)) {
+            auto datetime = formatRE.cap(1);
+            auto createdAt = QDateTime::fromString(datetime, DATETIME_FORMAT);
+
+            if (!createdAt.isValid()) {
+                qDebug() << "Skipping backup with invalid timestamp: " << datetime;
+                continue;
+            }
+
+            qDebug() << "Checking " << dirIterator.fileInfo().filePath();
+
+            // Based on last modified date, track the most recently modified file as the best backup
+            if (createdAt > bestBackupFileTime) {
+                bestBackupFound = true;
+                bestBackupFile = dirIterator.filePath();
+                bestBackupFileTime = createdAt;
+            }
+        } else {
+            qDebug() << "NO match: " << fileName << formatRE;
+        }
+    }
+
+    // If we found a backup then return the results
+    if (bestBackupFound) {
+        mostRecentBackupFileName = bestBackupFile;
+        mostRecentBackupTime = bestBackupFileTime;
+    }
+    return bestBackupFound;
+}
+
+void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, const QString& backupName) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "deleteBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, backupName));
+        return;
+    }
+
+    if (_isRecovering && backupName == _recoveryFilename) {
+        promise->resolve({
+            { "success", false }
+        });
+        return;
+    }
+
+    QDir backupDir { _backupDirectory };
+    QFile backupFile { backupDir.filePath(backupName) };
+    auto success = backupFile.remove();
+
+    refreshBackupRules();
+
+    for (auto& handler : _backupHandlers) {
+        handler->deleteBackup(backupName);
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip) {
+    if (!zip.open(QuaZip::Mode::mdUnzip)) {
+        qWarning() << "Failed to unzip file: " << backupName;
+        return false;
+    } else {
+        _isRecovering = true;
+        _recoveryFilename = backupName;
+
+        for (auto& handler : _backupHandlers) {
+            handler->recoverBackup(backupName, zip);
+        }
+
+        qDebug() << "Successfully started recovering from " << backupName;
+        return true;
+    }
+}
+
+void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, const QString& backupName) {
+    if (_isRecovering) {
+        promise->resolve({
+            { "success", false }
+        });
+        return;
+    };
+
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "recoverFromBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, backupName));
+        return;
+    }
+
+    qDebug() << "Recovering from" << backupName;
+
+    bool success { false };
+    QDir backupDir { _backupDirectory };
+    auto backupFilePath { backupDir.filePath(backupName) };
+    QFile backupFile { backupFilePath };
+    if (backupFile.open(QIODevice::ReadOnly)) {
+        QuaZip zip { &backupFile };
+
+        success = recoverFromBackupZip(backupName, zip);
+
+        backupFile.close();
+    } else {
+        success = false;
+        qWarning() << "Failed to open backup file for reading: " << backupFilePath;
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup) {
+
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "recoverFromUploadedBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(QByteArray, uploadedBackup));
+        return;
+    }
+
+    qDebug() << "Recovering from uploaded content archive";
+
+    // create a buffer and then a QuaZip from that buffer
+    QBuffer uploadedBackupBuffer { &uploadedBackup };
+    QuaZip uploadedZip { &uploadedBackupBuffer };
+
+    QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
+    bool success = recoverFromBackupZip(backupName, uploadedZip);
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
+
+    QDir backupDir { _backupDirectory };
+    auto matchingFiles =
+            backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + "*.zip", MANUAL_BACKUP_PREFIX + "*.zip" },
+                                    QDir::Files | QDir::NoSymLinks, QDir::Name);
+    QString prefixFormat = "(" + QRegExp::escape(AUTOMATIC_BACKUP_PREFIX) + "|" + QRegExp::escape(MANUAL_BACKUP_PREFIX) + ")";
+    QString nameFormat = "(.+)";
+    QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
+    QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
+
+    std::vector<BackupItemInfo> backups;
+
+    for (const auto& fileInfo : matchingFiles) {
+        auto fileName = fileInfo.fileName();
+        if (backupNameFormat.exactMatch(fileName)) {
+            auto type = backupNameFormat.cap(1);
+            auto name = backupNameFormat.cap(2);
+            auto dateTime = backupNameFormat.cap(3);
+            auto createdAt = QDateTime::fromString(dateTime, DATETIME_FORMAT);
+            if (!createdAt.isValid()) {
+                qDebug().nospace() << "Skipping backup (" << fileName << ") with invalid timestamp: " << dateTime;
+                continue;
+            }
+
+            backups.emplace_back(fileInfo.fileName(), name, fileInfo.absoluteFilePath(), createdAt,
+                                 type == MANUAL_BACKUP_PREFIX);
+        }
+    }
+
+    return backups;
+}
+
+void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise promise) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "getAllBackupsAndStatus", Q_ARG(MiniPromise::Promise, promise));
+        return;
+    }
+
+    auto backups = getAllBackups();
+
+    QVariantList variantBackups;
+
+    for (auto& backup : backups) {
+        bool isAvailable { true };
+        bool isCorrupted { false };
+        float availabilityProgress { 0.0f };
+        for (auto& handler : _backupHandlers) {
+            bool handlerIsAvailable { true };
+            float progress { 0.0f };
+            std::tie(handlerIsAvailable, progress) = handler->isAvailable(backup.id);
+            isAvailable &= handlerIsAvailable;
+            availabilityProgress += progress / _backupHandlers.size();
+
+            isCorrupted = isCorrupted || handler->isCorruptedBackup(backup.id);
+        }
+        variantBackups.push_back(QVariantMap({
+            { "id", backup.id },
+            { "name", backup.name },
+            { "createdAtMillis", backup.createdAt.toMSecsSinceEpoch() },
+            { "isAvailable", isAvailable },
+            { "availabilityProgress", availabilityProgress },
+            { "isManualBackup", backup.isManualBackup },
+            { "isCorrupted", isCorrupted }
+        }));
+    }
+
+    float recoveryProgress = 0.0f;
+    bool isRecovering = _isRecovering.load();
+    if (_isRecovering) {
+        for (auto& handler : _backupHandlers) {
+            float progress = handler->getRecoveryStatus().second;
+            recoveryProgress += progress / _backupHandlers.size();
+        }
+    }
+
+    QVariantMap status { 
+        { "isRecovering", isRecovering },
+        { "recoveringBackupId", _recoveryFilename },
+        { "recoveryProgress", recoveryProgress }
+    };
+
+    QVariantMap info {
+        { "backups", variantBackups },
+        { "status", status }
+    };
+
+    promise->resolve(info);
+}
+
+void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
+    QDir backupDir { _backupDirectory };
+    if (backupDir.exists() && rule.maxBackupVersions > 0) {
+        qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name;
+
+        auto matchingFiles =
+                backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
+
+        int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions;
+        if (backupsToDelete <= 0) {
+            qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, no backups need to be deleted";
+        } else {
+            qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, deleting " << backupsToDelete << "backup(s)";
+            for (int i = 0; i < backupsToDelete; ++i) {
+                auto fileInfo = matchingFiles[i].absoluteFilePath();
+                QFile backupFile(fileInfo);
+                if (backupFile.remove()) {
+                    qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName();
+                } else {
+                    qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName();
+                }
+            }
+            qCDebug(domain_server) << "Done removing old backup versions";
+        }
+    } else {
+        qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "."
+                                << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
+                                << " No need to roll backups";
+    }
+}
+
+void DomainContentBackupManager::backup() {
+    auto nowDateTime = QDateTime::currentDateTime();
+    auto nowSeconds = nowDateTime.toSecsSinceEpoch();
+
+    for (BackupRule& rule : _backupRules) {
+        auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds;
+
+        qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup
+                                << "] "
+                                << "compared to backup interval [" << rule.intervalSeconds << "]...";
+
+        if (secondsSinceLastBackup > rule.intervalSeconds) {
+            qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name
+                                    << "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now...";
+
+            bool success;
+            QString path;
+            std::tie(success, path) =  createBackup(AUTOMATIC_BACKUP_PREFIX, rule.extensionFormat);
+            if (!success) {
+                qCWarning(domain_server) << "Failed to create backup for" << rule.name << "at" << path;
+                continue;
+            }
+
+            qDebug() << "Created backup: " << path;
+
+            rule.lastBackupSeconds = nowSeconds;
+
+            removeOldBackupVersions(rule);
+        } else {
+            qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]...";
+        }
+    }
+}
+
+void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, QString fileName) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "consolidateBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(QString, fileName));
+        return;
+    }
+
+    QDir backupDir { _backupDirectory };
+    if (!backupDir.exists()) {
+        qCritical() << "Backup directory does not exist, bailing consolidation of backup";
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    auto filePath = backupDir.absoluteFilePath(fileName);
+
+    auto copyFilePath = QDir::tempPath() + "/" + fileName;
+
+    {
+        QFile copyFile(copyFilePath);
+        copyFile.remove();
+        copyFile.close();
+    }
+    auto copySuccess = QFile::copy(filePath, copyFilePath);
+    if (!copySuccess) {
+        qCritical() << "Failed to create copy of backup.";
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    QuaZip zip(copyFilePath);
+    if (!zip.open(QuaZip::mdAdd)) {
+        qCritical() << "Could not open backup archive:" << filePath;
+        qCritical() << "    ERROR:" << zip.getZipError();
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->consolidateBackup(fileName, zip);
+    }
+
+    zip.close();
+
+    if (zip.getZipError() != UNZ_OK) {
+        qCritical() << "Failed to consolidate backup: " << zip.getZipError();
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    promise->resolve({
+        { "success", true },
+        { "backupFilePath", copyFilePath }
+    });
+}
+
+void DomainContentBackupManager::createManualBackup(MiniPromise::Promise promise, const QString& name) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "createManualBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, name));
+        return;
+    }
+
+
+    QRegExp nameRE { MANUAL_BACKUP_NAME_RE };
+    bool success;
+
+    if (!nameRE.exactMatch(name)) {
+        qDebug() << "Cannot create manual backup with invalid name: " << name;
+        success = false;
+    } else {
+        QString path;
+        std::tie(success, path) = createBackup(MANUAL_BACKUP_PREFIX, name);
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+std::pair<bool, QString> DomainContentBackupManager::createBackup(const QString& prefix, const QString& name) {
+    auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT);
+    auto fileName = prefix + name + "-" + timestamp + ".zip";
+    auto path = _backupDirectory + "/" + fileName;
+    QuaZip zip(path);
+    if (!zip.open(QuaZip::mdAdd)) {
+        qCWarning(domain_server) << "Failed to open zip file at " << path;
+        qCWarning(domain_server) << "    ERROR:" << zip.getZipError();
+        return { false, path };
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->createBackup(fileName, zip);
+    }
+
+    zip.close();
+
+    return { true, path };
+}

domain-server/src/DomainContentBackupManager.h

@@ -0,0 +1,106 @@
+//
+//  DomainContentBackupManager.h
+//  libraries/domain-server/src
+//
+//  Created by Ryan Huffman on 1/01/18.
+//  Adapted from OctreePersistThread
+//  Copyright 2018 High Fidelity, Inc.
+//
+//
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_DomainContentBackupManager_h
+#define hifi_DomainContentBackupManager_h
+
+#include <QString>
+#include <QVector>
+#include <QDateTime>
+
+#include <GenericThread.h>
+
+#include "BackupHandler.h"
+
+#include <shared/MiniPromises.h>
+
+#include <PortableHighResolutionClock.h>
+
+struct BackupItemInfo {
+    BackupItemInfo(QString pId, QString pName, QString pAbsolutePath, QDateTime pCreatedAt, bool pIsManualBackup) :
+        id(pId), name(pName), absolutePath(pAbsolutePath), createdAt(pCreatedAt), isManualBackup(pIsManualBackup) { };
+
+    QString id;
+    QString name;
+    QString absolutePath;
+    QDateTime createdAt;
+    bool isManualBackup;
+};
+
+class DomainContentBackupManager : public GenericThread {
+    Q_OBJECT
+public:
+    class BackupRule {
+    public:
+        QString name;
+        int intervalSeconds;
+        QString extensionFormat;
+        int maxBackupVersions;
+        qint64 lastBackupSeconds;
+    };
+
+    static const std::chrono::seconds DEFAULT_PERSIST_INTERVAL;
+
+    DomainContentBackupManager(const QString& rootBackupDirectory,
+                               const QVariantList& settings,
+                               std::chrono::milliseconds persistInterval = DEFAULT_PERSIST_INTERVAL,
+                               bool debugTimestampNow = false);
+
+    std::vector<BackupItemInfo> getAllBackups();
+    void addBackupHandler(BackupHandlerPointer handler);
+    void aboutToFinish();  /// call this to inform the persist thread that the owner is about to finish to support final persist
+    void replaceData(QByteArray data);
+
+public slots:
+    void getAllBackupsAndStatus(MiniPromise::Promise promise);
+    void createManualBackup(MiniPromise::Promise promise, const QString& name);
+    void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
+    void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
+    void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
+    void consolidateBackup(MiniPromise::Promise promise, QString fileName);
+
+signals:
+    void loadCompleted();
+    void recoveryCompleted();
+
+protected:
+    /// Implements generic processing behavior for this thread.
+    virtual void setup() override;
+    virtual bool process() override;
+    virtual void shutdown() override;
+
+    void backup();
+    void removeOldBackupVersions(const BackupRule& rule);
+    void refreshBackupRules();
+    bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
+    int64_t getMostRecentBackupTimeInSecs(const QString& format);
+    void parseBackupRules(const QVariantList& backupRules);
+
+    std::pair<bool, QString> createBackup(const QString& prefix, const QString& name);
+
+    bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip);
+
+private:
+    const QString _backupDirectory;
+    std::vector<BackupHandlerPointer> _backupHandlers;
+    std::chrono::milliseconds _persistInterval { 0 };
+
+    std::atomic<bool> _isRecovering { false };
+    QString _recoveryFilename { };
+
+    p_high_resolution_clock::time_point _lastCheck;
+    std::vector<BackupRule> _backupRules;
+};
+
+#endif  // hifi_DomainContentBackupManager_h

domain-server/src/DomainGatekeeper.cpp

@@ -435,10 +435,11 @@ SharedNodePointer DomainGatekeeper::processAgentConnectRequest(const NodeConnect
     if (!userPerms.can(NodePermissions::Permission::canConnectPastMaxCapacity) && !isWithinMaxCapacity()) {
         // we can't allow this user to connect because we are at max capacity
         QString redirectOnMaxCapacity;
-        const QVariant* redirectOnMaxCapacityVariant =
-            valueForKeyPath(_server->_settingsManager.getSettingsMap(), MAXIMUM_USER_CAPACITY_REDIRECT_LOCATION);
-        if (redirectOnMaxCapacityVariant && redirectOnMaxCapacityVariant->canConvert<QString>()) {
-            redirectOnMaxCapacity = redirectOnMaxCapacityVariant->toString();
+
+        QVariant redirectOnMaxCapacityVariant =
+            _server->_settingsManager.valueForKeyPath(MAXIMUM_USER_CAPACITY_REDIRECT_LOCATION);
+        if (redirectOnMaxCapacityVariant.canConvert<QString>()) {
+            redirectOnMaxCapacity = redirectOnMaxCapacityVariant.toString();
             qDebug() << "Redirection domain:" << redirectOnMaxCapacity;
         }
 
@@ -610,9 +611,9 @@ bool DomainGatekeeper::verifyUserSignature(const QString& username,
 
 bool DomainGatekeeper::isWithinMaxCapacity() {
     // find out what our maximum capacity is
-    const QVariant* maximumUserCapacityVariant =
-        valueForKeyPath(_server->_settingsManager.getSettingsMap(), MAXIMUM_USER_CAPACITY);
-    unsigned int maximumUserCapacity = maximumUserCapacityVariant ? maximumUserCapacityVariant->toUInt() : 0;
+    QVariant maximumUserCapacityVariant =
+        _server->_settingsManager.valueForKeyPath(MAXIMUM_USER_CAPACITY);
+    unsigned int maximumUserCapacity = maximumUserCapacityVariant.isValid() ? maximumUserCapacityVariant.toUInt() : 0;
 
     if (maximumUserCapacity > 0) {
         unsigned int connectedUsers = _server->countConnectedUsers();

domain-server/src/DomainMetadata.cpp

@@ -84,21 +84,22 @@ void DomainMetadata::descriptorsChanged() {
     // get descriptors
     assert(_metadata[DESCRIPTORS].canConvert<QVariantMap>());
     auto& state = *static_cast<QVariantMap*>(_metadata[DESCRIPTORS].data());
-    auto& settings = static_cast<DomainServer*>(parent())->_settingsManager.getSettingsMap();
-    auto& descriptors = static_cast<DomainServer*>(parent())->_settingsManager.getDescriptorsMap();
+
+    static const QString DESCRIPTORS_GROUP_KEYPATH = "descriptors";
+    auto descriptorsMap = static_cast<DomainServer*>(parent())->_settingsManager.valueForKeyPath(DESCRIPTORS).toMap();
 
     // copy simple descriptors (description/maturity)
-    state[Descriptors::DESCRIPTION] = descriptors[Descriptors::DESCRIPTION]; 
-    state[Descriptors::MATURITY] = descriptors[Descriptors::MATURITY];
+    state[Descriptors::DESCRIPTION] = descriptorsMap[Descriptors::DESCRIPTION];
+    state[Descriptors::MATURITY] = descriptorsMap[Descriptors::MATURITY];
 
     // copy array descriptors (hosts/tags)
-    state[Descriptors::HOSTS] = descriptors[Descriptors::HOSTS].toList();
-    state[Descriptors::TAGS] = descriptors[Descriptors::TAGS].toList();
+    state[Descriptors::HOSTS] = descriptorsMap[Descriptors::HOSTS].toList();
+    state[Descriptors::TAGS] = descriptorsMap[Descriptors::TAGS].toList();
 
     // parse capacity
     static const QString CAPACITY = "security.maximum_user_capacity";
-    const QVariant* capacityVariant = valueForKeyPath(settings, CAPACITY);
-    unsigned int capacity = capacityVariant ? capacityVariant->toUInt() : 0;
+    QVariant capacityVariant = static_cast<DomainServer*>(parent())->_settingsManager.valueForKeyPath(CAPACITY);
+    unsigned int capacity = capacityVariant.isValid() ? capacityVariant.toUInt() : 0;
     state[Descriptors::CAPACITY] = capacity;
 
 #if DEV_BUILD || PR_BUILD

domain-server/src/DomainServer.cpp

@@ -24,6 +24,7 @@
 #include <QTimer>
 #include <QUrlQuery>
 #include <QCommandLineParser>
+#include <QUuid>
 
 #include <AccountManager.h>
 #include <AssetClient.h>
@@ -44,10 +45,20 @@
 #include <Trace.h>
 #include <StatTracker.h>
 
+#include "AssetsBackupHandler.h"
+#include "ContentSettingsBackupHandler.h"
 #include "DomainServerNodeData.h"
+#include "EntitiesBackupHandler.h"
 #include "NodeConnectionData.h"
 
+#include <Gzip.h>
+
+#include <OctreeDataUtils.h>
+
+Q_LOGGING_CATEGORY(domain_server, "hifi.domain_server")
+
 const QString ACCESS_TOKEN_KEY_PATH = "metaverse.access_token";
+const QString DomainServer::REPLACEMENT_FILE_EXTENSION = ".replace";
 
 int const DomainServer::EXIT_CODE_REBOOT = 234923;
 
@@ -64,8 +75,8 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection,
                                               std::initializer_list<QString> optionalData,
                                               bool requireAccessToken) {
 
-    auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-    if (accessTokenVariant == nullptr && requireAccessToken) {
+    auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+    if (!accessTokenVariant.isValid() && requireAccessToken) {
         connection->respond(HTTPConnection::StatusCode400, "User access token has not been set");
         return true;
     }
@@ -101,8 +112,8 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection,
     req.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
     req.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
 
-    if (accessTokenVariant != nullptr) {
-        auto accessTokenHeader = QString("Bearer ") + accessTokenVariant->toString();
+    if (accessTokenVariant.isValid()) {
+        auto accessTokenHeader = QString("Bearer ") + accessTokenVariant.toString();
         req.setRawHeader("Authorization", accessTokenHeader.toLatin1());
     }
 
@@ -280,6 +291,27 @@ DomainServer::DomainServer(int argc, char* argv[]) :
             qDebug() << "Ignoring subnet in whitelist, invalid ip portion: " << subnet;
         }
     }
+
+    if (QDir(getEntitiesDirPath()).mkpath(".")) {
+        qCDebug(domain_server) << "Created entities data directory";
+    }
+    maybeHandleReplacementEntityFile();
+
+
+    static const QString BACKUP_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
+    auto backupRulesVariant = _settingsManager.valueOrDefaultValueForKeyPath(BACKUP_RULES_KEYPATH);
+
+    _contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), backupRulesVariant.toList()));
+
+    connect(_contentManager.get(), &DomainContentBackupManager::started, _contentManager.get(), [this](){
+        _contentManager->addBackupHandler(BackupHandlerPointer(new EntitiesBackupHandler(getEntitiesFilePath(), getEntitiesReplacementFilePath())));
+        _contentManager->addBackupHandler(BackupHandlerPointer(new AssetsBackupHandler(getContentBackupDir())));
+        _contentManager->addBackupHandler(BackupHandlerPointer(new ContentSettingsBackupHandler(_settingsManager)));
+    });
+
+    _contentManager->initialize(true);
+
+    connect(_contentManager.get(), &DomainContentBackupManager::recoveryCompleted, this, &DomainServer::restart);
 }
 
 void DomainServer::parseCommandLine() {
@@ -345,6 +377,11 @@ void DomainServer::parseCommandLine() {
 DomainServer::~DomainServer() {
     qInfo() << "Domain Server is shutting down.";
 
+    if (_contentManager) {
+        _contentManager->aboutToFinish();
+        _contentManager->terminate();
+    }
+
     // cleanup the AssetClient thread
     DependencyManager::destroy<AssetClient>();
     _assetClientThread.quit();
@@ -377,8 +414,8 @@ bool DomainServer::optionallyReadX509KeyAndCertificate() {
     const QString X509_PRIVATE_KEY_OPTION = "key";
     const QString X509_KEY_PASSPHRASE_ENV = "DOMAIN_SERVER_KEY_PASSPHRASE";
 
-    QString certPath = _settingsManager.getSettingsMap().value(X509_CERTIFICATE_OPTION).toString();
-    QString keyPath = _settingsManager.getSettingsMap().value(X509_PRIVATE_KEY_OPTION).toString();
+    QString certPath = _settingsManager.valueForKeyPath(X509_CERTIFICATE_OPTION).toString();
+    QString keyPath = _settingsManager.valueForKeyPath(X509_PRIVATE_KEY_OPTION).toString();
 
     if (!certPath.isEmpty() && !keyPath.isEmpty()) {
         // the user wants to use the following cert and key for HTTPS
@@ -421,8 +458,7 @@ bool DomainServer::optionallySetupOAuth() {
     const QString OAUTH_CLIENT_SECRET_ENV = "DOMAIN_SERVER_CLIENT_SECRET";
     const QString REDIRECT_HOSTNAME_OPTION = "hostname";
 
-    const QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-    _oauthProviderURL = QUrl(settingsMap.value(OAUTH_PROVIDER_URL_OPTION).toString());
+    _oauthProviderURL = QUrl(_settingsManager.valueForKeyPath(OAUTH_PROVIDER_URL_OPTION).toString());
 
     // if we don't have an oauth provider URL then we default to the default node auth url
     if (_oauthProviderURL.isEmpty()) {
@@ -432,9 +468,9 @@ bool DomainServer::optionallySetupOAuth() {
     auto accountManager = DependencyManager::get<AccountManager>();
     accountManager->setAuthURL(_oauthProviderURL);
 
-    _oauthClientID = settingsMap.value(OAUTH_CLIENT_ID_OPTION).toString();
+    _oauthClientID = _settingsManager.valueForKeyPath(OAUTH_CLIENT_ID_OPTION).toString();
     _oauthClientSecret = QProcessEnvironment::systemEnvironment().value(OAUTH_CLIENT_SECRET_ENV);
-    _hostname = settingsMap.value(REDIRECT_HOSTNAME_OPTION).toString();
+    _hostname = _settingsManager.valueForKeyPath(REDIRECT_HOSTNAME_OPTION).toString();
 
     if (!_oauthClientID.isEmpty()) {
         if (_oauthProviderURL.isEmpty()
@@ -459,11 +495,11 @@ static const QString METAVERSE_DOMAIN_ID_KEY_PATH = "metaverse.id";
 
 void DomainServer::getTemporaryName(bool force) {
     // check if we already have a domain ID
-    const QVariant* idValueVariant = valueForKeyPath(_settingsManager.getSettingsMap(), METAVERSE_DOMAIN_ID_KEY_PATH);
+    QVariant idValueVariant = _settingsManager.valueForKeyPath(METAVERSE_DOMAIN_ID_KEY_PATH);
 
     qInfo() << "Requesting temporary domain name";
-    if (idValueVariant) {
-        qDebug() << "A domain ID is already present in domain-server settings:" << idValueVariant->toString();
+    if (idValueVariant.isValid()) {
+        qDebug() << "A domain ID is already present in domain-server settings:" << idValueVariant.toString();
         if (force) {
             qDebug() << "Requesting temporary domain name to replace current ID:" << getID();
         } else {
@@ -503,9 +539,6 @@ void DomainServer::handleTempDomainSuccess(QNetworkReply& requestReply) {
         auto settingsDocument = QJsonDocument::fromJson(newSettingsJSON.toUtf8());
         _settingsManager.recurseJSONObjectAndOverwriteSettings(settingsDocument.object(), DomainSettings);
 
-        // store the new ID and auto networking setting on disk
-        _settingsManager.persistToFile();
-
         // store the new token to the account info
         auto accountManager = DependencyManager::get<AccountManager>();
         accountManager->setTemporaryDomain(id, key);
@@ -607,8 +640,6 @@ void DomainServer::setupNodeListAndAssignments() {
     QVariant localPortValue = _settingsManager.valueOrDefaultValueForKeyPath(CUSTOM_LOCAL_PORT_OPTION);
     int domainServerPort = localPortValue.toInt();
 
-    QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-
     int domainServerDTLSPort = INVALID_PORT;
 
     if (_isUsingDTLS) {
@@ -616,8 +647,9 @@ void DomainServer::setupNodeListAndAssignments() {
 
         const QString CUSTOM_DTLS_PORT_OPTION = "dtls-port";
 
-        if (settingsMap.contains(CUSTOM_DTLS_PORT_OPTION)) {
-            domainServerDTLSPort = (unsigned short) settingsMap.value(CUSTOM_DTLS_PORT_OPTION).toUInt();
+        auto dtlsPortVariant = _settingsManager.valueForKeyPath(CUSTOM_DTLS_PORT_OPTION);
+        if (dtlsPortVariant.isValid()) {
+            domainServerDTLSPort = (unsigned short) dtlsPortVariant.toUInt();
         }
     }
 
@@ -647,9 +679,9 @@ void DomainServer::setupNodeListAndAssignments() {
         nodeList->setSessionUUID(_overridingDomainID);
         isMetaverseDomain = true; // assume metaverse domain
     } else {
-        const QVariant* idValueVariant = valueForKeyPath(settingsMap, METAVERSE_DOMAIN_ID_KEY_PATH);
-        if (idValueVariant) {
-            nodeList->setSessionUUID(idValueVariant->toString());
+        QVariant idValueVariant = _settingsManager.valueForKeyPath(METAVERSE_DOMAIN_ID_KEY_PATH);
+        if (idValueVariant.isValid()) {
+            nodeList->setSessionUUID(idValueVariant.toString());
             isMetaverseDomain = true; // if we have an ID, we'll assume we're a metaverse domain
         } else {
             nodeList->setSessionUUID(QUuid::createUuid()); // Use random UUID
@@ -691,13 +723,18 @@ void DomainServer::setupNodeListAndAssignments() {
     packetReceiver.registerListener(PacketType::ICEServerHeartbeatDenied, this, "processICEServerHeartbeatDenialPacket");
     packetReceiver.registerListener(PacketType::ICEServerHeartbeatACK, this, "processICEServerHeartbeatACK");
 
+    packetReceiver.registerListener(PacketType::OctreeDataFileRequest, this, "processOctreeDataRequestMessage");
+    packetReceiver.registerListener(PacketType::OctreeDataPersist, this, "processOctreeDataPersistMessage");
+
+    packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacementRequest");
+    packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURLRequest");
+
     // set a custom packetVersionMatch as the verify packet operator for the udt::Socket
     nodeList->setPacketFilterOperator(&DomainServer::isPacketVerified);
 
     _assetClientThread.setObjectName("AssetClient Thread");
     auto assetClient = DependencyManager::set<AssetClient>();
     assetClient->moveToThread(&_assetClientThread);
-    QObject::connect(&_assetClientThread, &QThread::started, assetClient.data(), &AssetClient::init);
     _assetClientThread.start();
 
     // add whatever static assignments that have been parsed to the queue
@@ -712,10 +749,10 @@ bool DomainServer::resetAccountManagerAccessToken() {
         QString accessToken = QProcessEnvironment::systemEnvironment().value(ENV_ACCESS_TOKEN_KEY);
 
         if (accessToken.isEmpty()) {
-            const QVariant* accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
+            QVariant accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
 
-            if (accessTokenVariant && accessTokenVariant->canConvert(QMetaType::QString)) {
-                accessToken = accessTokenVariant->toString();
+            if (accessTokenVariant.canConvert(QMetaType::QString)) {
+                accessToken = accessTokenVariant.toString();
             } else {
                 qWarning() << "No access token is present. Some operations that use the metaverse API will fail.";
                 qDebug() << "Set an access token via the web interface, in your user config"
@@ -846,31 +883,26 @@ void DomainServer::updateICEServerAddresses() {
 }
 
 void DomainServer::parseAssignmentConfigs(QSet<Assignment::Type>& excludedTypes) {
-    const QString ASSIGNMENT_CONFIG_REGEX_STRING = "config-([\\d]+)";
-    QRegExp assignmentConfigRegex(ASSIGNMENT_CONFIG_REGEX_STRING);
-
-    const QVariantMap& settingsMap = _settingsManager.getSettingsMap();
+    const QString ASSIGNMENT_CONFIG_PREFIX = "config-";
 
     // scan for assignment config keys
-    QStringList variantMapKeys = settingsMap.keys();
-    int configIndex = variantMapKeys.indexOf(assignmentConfigRegex);
+    for (int i = 0; i < Assignment::AllTypes; ++i) {
+        QVariant assignmentConfigVariant = _settingsManager.valueOrDefaultValueForKeyPath(ASSIGNMENT_CONFIG_PREFIX + QString::number(i));
 
-    while (configIndex != -1) {
-        // figure out which assignment type this matches
-        Assignment::Type assignmentType = (Assignment::Type) assignmentConfigRegex.cap(1).toInt();
+        if (assignmentConfigVariant.isValid()) {
+            // figure out which assignment type this matches
+            Assignment::Type assignmentType = static_cast<Assignment::Type>(i);
 
-        if (assignmentType < Assignment::AllTypes && !excludedTypes.contains(assignmentType)) {
-            QVariant mapValue = settingsMap[variantMapKeys[configIndex]];
-            QVariantList assignmentList = mapValue.toList();
+            if (!excludedTypes.contains(assignmentType)) {
+                QVariantList assignmentList = assignmentConfigVariant.toList();
 
-            if (assignmentType != Assignment::AgentType) {
-                createStaticAssignmentsForType(assignmentType, assignmentList);
+                if (assignmentType != Assignment::AgentType) {
+                    createStaticAssignmentsForType(assignmentType, assignmentList);
+                }
+
+                excludedTypes.insert(assignmentType);
             }
-
-            excludedTypes.insert(assignmentType);
         }
-
-        configIndex = variantMapKeys.indexOf(assignmentConfigRegex, configIndex + 1);
     }
 }
 
@@ -882,10 +914,10 @@ void DomainServer::addStaticAssignmentToAssignmentHash(Assignment* newAssignment
 
 void DomainServer::populateStaticScriptedAssignmentsFromSettings() {
     const QString PERSISTENT_SCRIPTS_KEY_PATH = "scripts.persistent_scripts";
-    const QVariant* persistentScriptsVariant = valueForKeyPath(_settingsManager.getSettingsMap(), PERSISTENT_SCRIPTS_KEY_PATH);
+    QVariant persistentScriptsVariant = _settingsManager.valueOrDefaultValueForKeyPath(PERSISTENT_SCRIPTS_KEY_PATH);
 
-    if (persistentScriptsVariant) {
-        QVariantList persistentScriptsList = persistentScriptsVariant->toList();
+    if (persistentScriptsVariant.isValid()) {
+        QVariantList persistentScriptsList = persistentScriptsVariant.toList();
         foreach(const QVariant& persistentScriptVariant, persistentScriptsList) {
             QVariantMap persistentScript = persistentScriptVariant.toMap();
 
@@ -1695,10 +1727,96 @@ void DomainServer::sendHeartbeatToIceServer() {
     } else {
         qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server.";
         qDebug() << "Waiting for" << _iceServerAddr << "host lookup response";
-
     }
 }
 
+void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message) {
+    qDebug() << "Received octree data persist message";
+    auto data = message->readAll();
+    auto filePath = getEntitiesFilePath();
+
+    QDir dir(getEntitiesDirPath());
+    if (!dir.exists()) {
+        qCDebug(domain_server) << "Creating entities content directory:" << dir.absolutePath();
+        dir.mkpath(".");
+    }
+
+    QFile f(filePath);
+    if (f.open(QIODevice::WriteOnly)) {
+        f.write(data);
+        OctreeUtils::RawEntityData entityData;
+        if (entityData.readOctreeDataInfoFromData(data)) {
+            qCDebug(domain_server) << "Wrote new entities file" << entityData.id << entityData.version;
+        } else {
+            qCDebug(domain_server) << "Failed to read new octree data info";
+        }
+    } else {
+        qCDebug(domain_server) << "Failed to write new entities file:" << filePath;
+    }
+}
+
+QString DomainServer::getContentBackupDir() {
+    return PathUtils::getAppDataFilePath("backups");
+}
+
+QString DomainServer::getEntitiesDirPath() {
+    return PathUtils::getAppDataFilePath("entities");
+}
+
+QString DomainServer::getEntitiesFilePath() {
+    return PathUtils::getAppDataFilePath("entities/models.json.gz");
+}
+
+QString DomainServer::getEntitiesReplacementFilePath() {
+    return getEntitiesFilePath().append(REPLACEMENT_FILE_EXTENSION);
+}
+
+void DomainServer::processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message) {
+    qDebug() << "Got request for octree data from " << message->getSenderSockAddr();
+
+    maybeHandleReplacementEntityFile();
+
+    bool remoteHasExistingData { false };
+    QUuid id;
+    int version;
+    message->readPrimitive(&remoteHasExistingData);
+    if (remoteHasExistingData) {
+        constexpr size_t UUID_SIZE_BYTES = 16;
+        auto idData = message->read(UUID_SIZE_BYTES);
+        id = QUuid::fromRfc4122(idData);
+        message->readPrimitive(&version);
+        qCDebug(domain_server) << "Entity server does have existing data: ID(" << id << ") DataVersion(" << version << ")";
+    } else {
+        qCDebug(domain_server) << "Entity server does not have existing data";
+    }
+    auto entityFilePath = getEntitiesFilePath();
+
+    auto reply = NLPacketList::create(PacketType::OctreeDataFileReply, QByteArray(), true, true);
+    OctreeUtils::RawEntityData data;
+    if (data.readOctreeDataInfoFromFile(entityFilePath)) {
+        if (data.id == id && data.version <= version) {
+            qCDebug(domain_server) << "ES has sufficient octree data, not sending data";
+            reply->writePrimitive(false);
+        } else {
+            qCDebug(domain_server) << "Sending newer octree data to ES: ID(" << data.id << ") DataVersion(" << data.version << ")";
+            QFile file(entityFilePath);
+            if (file.open(QIODevice::ReadOnly)) {
+                reply->writePrimitive(true);
+                reply->write(file.readAll());
+            } else {
+                qCDebug(domain_server) << "Unable to load entity file";
+                reply->writePrimitive(false);
+            }
+        }
+    } else {
+        qCDebug(domain_server) << "Domain server does not have valid octree data";
+        reply->writePrimitive(false);
+    }
+
+    auto nodeList = DependencyManager::get<LimitedNodeList>();
+    nodeList->sendPacketList(std::move(reply), message->getSenderSockAddr());
+}
+
 void DomainServer::processNodeJSONStatsPacket(QSharedPointer<ReceivedMessage> packetList, SharedNodePointer sendingNode) {
     auto nodeData = static_cast<DomainServerNodeData*>(sendingNode->getLinkedData());
     if (nodeData) {
@@ -1808,23 +1926,25 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
     const QString URI_ASSIGNMENT = "/assignment";
     const QString URI_NODES = "/nodes";
     const QString URI_SETTINGS = "/settings";
-    const QString URI_ENTITY_FILE_UPLOAD = "/content/upload";
+    const QString URI_CONTENT_UPLOAD = "/content/upload";
     const QString URI_RESTART = "/restart";
     const QString URI_API_PLACES = "/api/places";
     const QString URI_API_DOMAINS = "/api/domains";
     const QString URI_API_DOMAINS_ID = "/api/domains/";
+    const QString URI_API_BACKUPS = "/api/backups";
+    const QString URI_API_BACKUPS_ID = "/api/backups/";
+    const QString URI_API_BACKUPS_RECOVER = "/api/backups/recover/";
 
     const QString UUID_REGEX_STRING = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
 
     auto nodeList = DependencyManager::get<LimitedNodeList>();
 
-    auto getSetting = [this](QString keyPath, QVariant& value) -> bool {
-        QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-        QVariant* var = valueForKeyPath(settingsMap, keyPath);
-        if (var == nullptr) {
+    auto getSetting = [this](QString keyPath, QVariant value) -> bool {
+
+        value = _settingsManager.valueForKeyPath(keyPath);
+        if (!value.isValid()) {
             return false;
         }
-        value = *var;
         return true;
     };
 
@@ -1892,8 +2012,8 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
     if (connection->requestOperation() == QNetworkAccessManager::GetOperation) {
         const QString URI_WIZARD = "/wizard/";
         const QString WIZARD_COMPLETED_ONCE_KEY_PATH = "wizard.completed_once";
-        const QVariant* wizardCompletedOnce = valueForKeyPath(_settingsManager.getSettingsMap(), WIZARD_COMPLETED_ONCE_KEY_PATH);
-        const bool completedOnce = wizardCompletedOnce && wizardCompletedOnce->toBool();
+        QVariant wizardCompletedOnce = _settingsManager.valueForKeyPath(WIZARD_COMPLETED_ONCE_KEY_PATH);
+        const bool completedOnce = wizardCompletedOnce.isValid() && wizardCompletedOnce.toBool();
 
         if (url.path() != URI_WIZARD && url.path().endsWith('/') && !completedOnce) {
             // First visit, redirect to the wizard
@@ -1997,6 +2117,37 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             // send the response
             connection->respond(HTTPConnection::StatusCode200, nodesDocument.toJson(), qPrintable(JSON_MIME_TYPE));
 
+            return true;
+        } else if (url.path() == URI_API_BACKUPS) {
+            auto deferred = makePromise("getAllBackupsAndStatus");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
+
+                connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->getAllBackupsAndStatus(deferred);
+            return true;
+        } else if (url.path().startsWith(URI_API_BACKUPS_ID)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
+            auto deferred = makePromise("consolidateBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                if (success) {
+                    auto path = result["backupFilePath"].toString();
+                    auto file { std::unique_ptr<QFile>(new QFile(path)) };
+                    if (file->open(QIODevice::ReadOnly)) {
+                        connection->respond(HTTPConnection::StatusCode200, std::move(file));
+                    } else {
+                        qCritical(domain_server) << "Unable to load consolidated backup at:" << path << result;
+                        connection->respond(HTTPConnection::StatusCode500, "Error opening backup");
+                    }
+                } else {
+                    connection->respond(HTTPConnection::StatusCode400);
+                }
+            });
+            _contentManager->consolidateBackup(deferred, id);
+
             return true;
         } else if (url.path() == URI_RESTART) {
             connection->respond(HTTPConnection::StatusCode200);
@@ -2084,17 +2235,52 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             connection->respond(HTTPConnection::StatusCode200);
 
             return true;
-        } else if (url.path() == URI_ENTITY_FILE_UPLOAD) {
+        } else if (url.path() == URI_CONTENT_UPLOAD) {
             // this is an entity file upload, ask the HTTPConnection to parse the data
             QList<FormData> formData = connection->parseFormData();
 
             if (formData.size() > 0 && formData[0].second.size() > 0) {
-                // invoke our method to hand the new octree file off to the octree server
-                QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
-                                          Qt::QueuedConnection, Q_ARG(QByteArray, formData[0].second));
+                auto& firstFormData = formData[0];
+
+                // check the file extension to see what kind of file this is
+                // to make sure we handle this filetype for a content restore
+                auto dispositionValue = QString(firstFormData.first.value("Content-Disposition"));
+                auto formDataFilenameRegex = QRegExp("filename=\"(.+)\"");
+                auto matchIndex = formDataFilenameRegex.indexIn(dispositionValue);
+
+                QString uploadedFilename = "";
+                if (matchIndex != -1) {
+                    uploadedFilename = formDataFilenameRegex.cap(1);
+                }
+
+                if (uploadedFilename.endsWith(".json", Qt::CaseInsensitive)
+                    || uploadedFilename.endsWith(".json.gz", Qt::CaseInsensitive)) {
+                    // invoke our method to hand the new octree file off to the octree server
+                    QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
+                                              Qt::QueuedConnection, Q_ARG(QByteArray, firstFormData.second));
+
+                    // respond with a 200 for success
+                    connection->respond(HTTPConnection::StatusCode200);
+                } else if (uploadedFilename.endsWith(".zip", Qt::CaseInsensitive)) {
+                    auto deferred = makePromise("recoverFromUploadedBackup");
+
+                    deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                        QJsonObject rootJSON;
+                        auto success = result["success"].toBool();
+                        rootJSON["success"] = success;
+                        QJsonDocument docJSON(rootJSON);
+                        connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                            JSON_MIME_TYPE.toUtf8());
+                    });
+
+                    _contentManager->recoverFromUploadedBackup(deferred, firstFormData.second);
+
+                    return true;
+                } else {
+                    // we don't have handling for this filetype, send back a 400 for failure
+                    connection->respond(HTTPConnection::StatusCode400);
+                }
 
-                // respond with a 200 for success
-                connection->respond(HTTPConnection::StatusCode200);
             } else {
                 // respond with a 400 for failure
                 connection->respond(HTTPConnection::StatusCode400);
@@ -2102,16 +2288,50 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
             return true;
 
+        } else if (url.path() == URI_API_BACKUPS) {
+            auto params = connection->parseUrlEncodedForm();
+            auto it = params.find("name");
+            if (it == params.end()) {
+                connection->respond(HTTPConnection::StatusCode400, "Bad request, missing `name`");
+                return true;
+            }
+
+            auto deferred = makePromise("createManualBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->createManualBackup(deferred, it.value());
+
+            return true;
+
         } else if (url.path() == "/domain_settings") {
-            auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-            if (!accessTokenVariant) {
+            auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+            if (!accessTokenVariant.isValid()) {
                 connection->respond(HTTPConnection::StatusCode400);
                 return true;
             }
 
         } else if (url.path() == URI_API_DOMAINS) {
-
             return forwardMetaverseAPIRequest(connection, "/api/v1/domains", "domain", { "label" });
+
+        } else if (url.path().startsWith(URI_API_BACKUPS_RECOVER)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_RECOVER).length());
+            auto deferred = makePromise("recoverFromBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->recoverFromBackup(deferred, id);
+            return true;
         }
     } else if (connection->requestOperation() == QNetworkAccessManager::PutOperation) {
         if (url.path() == URI_API_DOMAINS) {
@@ -2124,8 +2344,8 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             return forwardMetaverseAPIRequest(connection, "/api/v1/domains/" + domainID, "domain",
                                               { }, { "network_address", "network_port", "label" });
         }  else if (url.path() == URI_API_PLACES) {
-            auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-            if (!accessTokenVariant->isValid()) {
+            auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+            if (!accessTokenVariant.isValid()) {
                 connection->respond(HTTPConnection::StatusCode400, "User access token has not been set");
                 return true;
             }
@@ -2173,7 +2393,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
             QUrl url { NetworkingConstants::METAVERSE_SERVER_URL().toString() + "/api/v1/places/" + place_id };
 
-            url.setQuery("access_token=" + accessTokenVariant->toString());
+            url.setQuery("access_token=" + accessTokenVariant.toString());
 
             QNetworkRequest req(url);
             req.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
@@ -2200,7 +2420,22 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
         QRegExp allNodesDeleteRegex(ALL_NODE_DELETE_REGEX_STRING);
         QRegExp nodeDeleteRegex(NODE_DELETE_REGEX_STRING);
 
-        if (nodeDeleteRegex.indexIn(url.path()) != -1) {
+        if (url.path().startsWith(URI_API_BACKUPS_ID)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
+            auto deferred = makePromise("deleteBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->deleteBackup(deferred, id);
+
+            return true;
+
+        } else if (nodeDeleteRegex.indexIn(url.path()) != -1) {
             // this is a request to DELETE one node by UUID
 
             // pull the captured string, if it exists
@@ -2353,10 +2588,11 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
 
     const QByteArray UNAUTHENTICATED_BODY = "You do not have permission to access this domain-server.";
 
-    QVariantMap& settingsMap = _settingsManager.getSettingsMap();
+    QVariant adminUsersVariant = _settingsManager.valueForKeyPath(ADMIN_USERS_CONFIG_KEY);
+    QVariant adminRolesVariant = _settingsManager.valueForKeyPath(ADMIN_ROLES_CONFIG_KEY);
 
     if (!_oauthProviderURL.isEmpty()
-        && (settingsMap.contains(ADMIN_USERS_CONFIG_KEY) || settingsMap.contains(ADMIN_ROLES_CONFIG_KEY))) {
+        && (adminUsersVariant.isValid() || adminRolesVariant.isValid())) {
         QString cookieString = connection->requestHeaders().value(HTTP_COOKIE_HEADER_KEY);
 
         const QString COOKIE_UUID_REGEX_STRING = HIFI_SESSION_COOKIE_KEY + "=([\\d\\w-]+)($|;)";
@@ -2367,7 +2603,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             cookieUUID = cookieUUIDRegex.cap(1);
         }
 
-        if (valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)) {
+        if (_settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).isValid()) {
             qDebug() << "Config file contains web admin settings for OAuth and basic HTTP authentication."
                 << "These cannot be combined - using OAuth for authentication.";
         }
@@ -2377,13 +2613,13 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             DomainServerWebSessionData sessionData = _cookieSessionHash.value(cookieUUID);
             QString profileUsername = sessionData.getUsername();
 
-            if (settingsMap.value(ADMIN_USERS_CONFIG_KEY).toStringList().contains(profileUsername)) {
+            if (_settingsManager.valueForKeyPath(ADMIN_USERS_CONFIG_KEY).toStringList().contains(profileUsername)) {
                 // this is an authenticated user
                 return true;
             }
 
             // loop the roles of this user and see if they are in the admin-roles array
-            QStringList adminRolesArray = settingsMap.value(ADMIN_ROLES_CONFIG_KEY).toStringList();
+            QStringList adminRolesArray = _settingsManager.valueForKeyPath(ADMIN_ROLES_CONFIG_KEY).toStringList();
 
             if (!adminRolesArray.isEmpty()) {
                 foreach(const QString& userRole, sessionData.getRoles()) {
@@ -2428,7 +2664,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             // we don't know about this user yet, so they are not yet authenticated
             return false;
         }
-    } else if (valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)) {
+    } else if (_settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).isValid()) {
         // config file contains username and password combinations for basic auth
         const QByteArray BASIC_AUTH_HEADER_KEY = "Authorization";
 
@@ -2447,10 +2683,10 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
                     QString headerPassword = credentialList[1];
 
                     // we've pulled a username and password - now check if there is a match in our basic auth hash
-                    QString settingsUsername = valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)->toString();
-                    const QVariant* settingsPasswordVariant = valueForKeyPath(settingsMap, BASIC_AUTH_PASSWORD_KEY_PATH);
+                    QString settingsUsername = _settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).toString();
+                    QVariant settingsPasswordVariant = _settingsManager.valueForKeyPath(BASIC_AUTH_PASSWORD_KEY_PATH);
 
-                    QString settingsPassword = settingsPasswordVariant ? settingsPasswordVariant->toString() : "";
+                    QString settingsPassword = settingsPasswordVariant.isValid() ? settingsPasswordVariant.toString() : "";
                     QString hexHeaderPassword = headerPassword.isEmpty() ?
                         "" : QCryptographicHash::hash(headerPassword.toUtf8(), QCryptographicHash::Sha256).toHex();
                         
@@ -2587,13 +2823,14 @@ ReplicationServerInfo serverInformationFromSettings(QVariantMap serverMap, Repli
 }
 
 void DomainServer::updateReplicationNodes(ReplicationServerDirection direction) {
-    auto settings = _settingsManager.getSettingsMap();
 
-    if (settings.contains(BROADCASTING_SETTINGS_KEY)) {
+    auto broadcastSettingsVariant = _settingsManager.valueForKeyPath(BROADCASTING_SETTINGS_KEY);
+
+    if (broadcastSettingsVariant.isValid()) {
         auto nodeList = DependencyManager::get<LimitedNodeList>();
         std::vector<HifiSockAddr> replicationNodesInSettings;
 
-        auto replicationSettings = settings.value(BROADCASTING_SETTINGS_KEY).toMap();
+        auto replicationSettings = broadcastSettingsVariant.toMap();
 
         QString serversKey = direction == Upstream ? "upstream_servers" : "downstream_servers";
         QString replicationDirection = direction == Upstream ? "upstream" : "downstream";
@@ -2669,13 +2906,12 @@ void DomainServer::updateUpstreamNodes() {
 
 void DomainServer::updateReplicatedNodes() {
     // Make sure we have downstream nodes in our list
-    auto settings = _settingsManager.getSettingsMap();
-
     static const QString REPLICATED_USERS_KEY = "users";
     _replicatedUsernames.clear();
-    
-    if (settings.contains(BROADCASTING_SETTINGS_KEY)) {
-        auto replicationSettings = settings.value(BROADCASTING_SETTINGS_KEY).toMap();
+
+    auto replicationVariant = _settingsManager.valueForKeyPath(BROADCASTING_SETTINGS_KEY);
+    if (replicationVariant.isValid()) {
+        auto replicationSettings = replicationVariant.toMap();
         if (replicationSettings.contains(REPLICATED_USERS_KEY)) {
             auto usersSettings = replicationSettings.value(REPLICATED_USERS_KEY).toList();
             for (auto& username : usersSettings) {
@@ -2863,17 +3099,17 @@ void DomainServer::processPathQueryPacket(QSharedPointer<ReceivedMessage> messag
 
         // check out paths in the _configMap to see if we have a match
         auto keypath = QString(PATHS_SETTINGS_KEYPATH_FORMAT).arg(SETTINGS_PATHS_KEY).arg(pathQuery);
-        const QVariant* pathMatch = valueForKeyPath(_settingsManager.getSettingsMap(), keypath);
+        QVariant pathMatch = _settingsManager.valueForKeyPath(keypath);
 
-        if (pathMatch || pathQuery == INDEX_PATH) {
+        if (pathMatch.isValid() || pathQuery == INDEX_PATH) {
             // we got a match, respond with the resulting viewpoint
             auto nodeList = DependencyManager::get<LimitedNodeList>();
 
             QString responseViewpoint;
 
             // if we didn't match the path BUT this is for the index path then send back our default
-            if (pathMatch) {
-                responseViewpoint = pathMatch->toMap()[PATH_VIEWPOINT_KEY].toString();
+            if (pathMatch.isValid()) {
+                responseViewpoint = pathMatch.toMap()[PATH_VIEWPOINT_KEY].toString();
             } else {
                 const QString DEFAULT_INDEX_PATH = "/0,0,0/0,0,0,1";
                 responseViewpoint = DEFAULT_INDEX_PATH;
@@ -3105,19 +3341,101 @@ void DomainServer::setupGroupCacheRefresh() {
     }
 }
 
-void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
-    // enumerate the nodes and find any octree type servers with active sockets
+void DomainServer::maybeHandleReplacementEntityFile() {
+    const auto replacementFilePath = getEntitiesReplacementFilePath();
+    OctreeUtils::RawEntityData data;
+    if (!data.readOctreeDataInfoFromFile(replacementFilePath)) {
+        qCWarning(domain_server) << "Replacement file could not be read, it either doesn't exist or is invalid.";
+    } else {
+        qCDebug(domain_server) << "Replacing existing entity date with replacement file";
 
-    auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
-    limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
-        return node->getType() == NodeType::EntityServer && node->getActiveSocket();
-    }, [&octreeFile, limitedNodeList](const SharedNodePointer& octreeNode) {
-        // setup a packet to send to this octree server with the new octree file data
-        auto octreeFilePacketList = NLPacketList::create(PacketType::OctreeFileReplacement, QByteArray(), true, true);
-        octreeFilePacketList->write(octreeFile);
+        QFile replacementFile(replacementFilePath);
+        if (!replacementFile.remove()) {
+            // If we can't remove the replacement file, we are at risk of getting into a state where
+            // we continually replace the primary entity file with the replacement entity file.
+            qCWarning(domain_server) << "Unable to remove replacement file, bailing";
+        } else {
+            data.resetIdAndVersion();
+            auto gzippedData = data.toGzippedByteArray();
 
-        qDebug() << "Sending an octree file replacement of" << octreeFile.size() << "bytes to" << octreeNode;
-
-        limitedNodeList->sendPacketList(std::move(octreeFilePacketList), *octreeNode);
-    });
+            QFile currentFile(getEntitiesFilePath());
+            if (!currentFile.open(QIODevice::WriteOnly)) {
+                qCWarning(domain_server)
+                    << "Failed to update entities data file with replacement file, unable to open entities file for writing";
+            } else {
+                currentFile.write(gzippedData);
+            }
+        }
+    }
+}
+
+void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
+    //Assume we have compressed data
+    auto compressedOctree = octreeFile;
+    QByteArray jsonOctree;
+
+    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
+    if (!wasCompressed) {
+        // the source was not compressed, assume we were sent regular JSON data
+        jsonOctree = compressedOctree;
+    }
+
+    OctreeUtils::RawEntityData data;
+    if (data.readOctreeDataInfoFromData(jsonOctree)) {
+        data.resetIdAndVersion();
+
+        gzip(data.toByteArray(), compressedOctree);
+
+        // write the compressed octree data to a special file
+        auto replacementFilePath = getEntitiesReplacementFilePath();
+        QFile replacementFile(replacementFilePath);
+        if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
+            // we've now written our replacement file, time to take the server down so it can
+            // process it when it comes back up
+            qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
+
+            QMetaObject::invokeMethod(this, "restart", Qt::QueuedConnection);
+        } else {
+            qWarning() << "Could not write replacement octree data to file - refusing to process";
+        }
+    } else {
+        qDebug() << "Received replacement octree file that is invalid - refusing to process";
+    }
+}
+
+void DomainServer::handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message) {
+    qInfo() << "Received request to replace content from a url";
+    auto node = DependencyManager::get<LimitedNodeList>()->findNodeWithAddr(message->getSenderSockAddr());
+    if (node) {
+        qDebug() << "Found node: " << node->getCanReplaceContent();
+    }
+    if (node->getCanReplaceContent()) {
+        // Convert message data into our URL
+        QString url(message->getMessage());
+        QUrl modelsURL = QUrl(url, QUrl::StrictMode);
+        QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+        QNetworkRequest request(modelsURL);
+        QNetworkReply* reply = networkAccessManager.get(request);
+
+        qDebug() << "Downloading JSON from: " << modelsURL;
+
+        connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
+            QNetworkReply::NetworkError networkError = reply->error();
+            if (networkError == QNetworkReply::NoError) {
+                handleOctreeFileReplacement(reply->readAll());
+            } else {
+                qDebug() << "Error downloading JSON from specified file: " << modelsURL;
+            }
+        });
+    }
+}
+
+
+
+
+void DomainServer::handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message) {
+    auto node = DependencyManager::get<NodeList>()->nodeWithUUID(message->getSourceID());
+    if (node->getCanReplaceContent()) {
+        handleOctreeFileReplacement(message->readAll());
+    }
 }

domain-server/src/DomainServer.h

@@ -26,15 +26,20 @@
 #include <HTTPSConnection.h>
 #include <LimitedNodeList.h>
 
-#include "BackupSupervisor.h"
+#include "AssetsBackupHandler.h"
 #include "DomainGatekeeper.h"
 #include "DomainMetadata.h"
 #include "DomainServerSettingsManager.h"
 #include "DomainServerWebSessionData.h"
 #include "WalletTransaction.h"
+#include "DomainContentBackupManager.h"
 
 #include "PendingAssignedNodeData.h"
 
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(domain_server)
+
 typedef QSharedPointer<Assignment> SharedAssignmentPointer;
 typedef QMultiHash<QUuid, WalletTransaction*> TransactionHash;
 
@@ -65,6 +70,8 @@ public:
     bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
     bool handleHTTPSRequest(HTTPSConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
 
+    static const QString REPLACEMENT_FILE_EXTENSION;
+
 public slots:
     /// Called by NodeList to inform us a node has been added
     void nodeAdded(SharedNodePointer node);
@@ -84,6 +91,13 @@ private slots:
     void processICEServerHeartbeatDenialPacket(QSharedPointer<ReceivedMessage> message);
     void processICEServerHeartbeatACK(QSharedPointer<ReceivedMessage> message);
 
+    void handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeFileReplacement(QByteArray octreeFile);
+
+    void processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message);
+    void processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message);
+
     void setupPendingAssignmentCredits();
     void sendPendingTransactionsToServer();
 
@@ -91,8 +105,7 @@ private slots:
     void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); }
     void sendHeartbeatToIceServer();
 
-    void handleConnectedNode(SharedNodePointer newNode);
-
+    void handleConnectedNode(SharedNodePointer newNode); 
     void handleTempDomainSuccess(QNetworkReply& requestReply);
     void handleTempDomainError(QNetworkReply& requestReply);
 
@@ -109,8 +122,6 @@ private slots:
     void handleSuccessfulICEServerAddressUpdate(QNetworkReply& requestReply);
     void handleFailedICEServerAddressUpdate(QNetworkReply& requestReply);
 
-    void handleOctreeFileReplacement(QByteArray octreeFile);
-
     void updateReplicatedNodes();
     void updateDownstreamNodes();
     void updateUpstreamNodes();
@@ -127,6 +138,13 @@ private:
     const QUuid& getID();
     void parseCommandLine();
 
+    QString getContentBackupDir();
+    QString getEntitiesDirPath();
+    QString getEntitiesFilePath();
+    QString getEntitiesReplacementFilePath();
+
+    void maybeHandleReplacementEntityFile();
+
     void setupNodeListAndAssignments();
     bool optionallySetupOAuth();
     bool optionallyReadX509KeyAndCertificate();
@@ -252,6 +270,8 @@ private:
     bool _sendICEServerAddressToMetaverseAPIInProgress { false };
     bool _sendICEServerAddressToMetaverseAPIRedo { false };
 
+    std::unique_ptr<DomainContentBackupManager> _contentManager { nullptr };
+
     QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
 
     QThread _assetClientThread;

domain-server/src/DomainServerSettingsManager.cpp

@@ -19,6 +19,7 @@
 #include <QtCore/QJsonArray>
 #include <QtCore/QJsonObject>
 #include <QtCore/QStandardPaths>
+#include <QtCore/QThread>
 #include <QtCore/QUrl>
 #include <QtCore/QUrlQuery>
 
@@ -32,9 +33,13 @@
 #include <SettingHelpers.h>
 #include <AvatarData.h> //for KillAvatarReason
 #include <FingerprintUtils.h>
+
 #include "DomainServerNodeData.h"
 
 const QString SETTINGS_DESCRIPTION_RELATIVE_PATH = "/resources/describe-settings.json";
+const QString SETTINGS_PATH = "/settings";
+const QString SETTINGS_PATH_JSON = SETTINGS_PATH + ".json";
+const QString CONTENT_SETTINGS_PATH_JSON = "/content-settings.json";
 
 const QString DESCRIPTION_SETTINGS_KEY = "settings";
 const QString SETTING_DEFAULT_KEY = "default";
@@ -187,6 +192,9 @@ void DomainServerSettingsManager::processSettingsRequestPacket(QSharedPointer<Re
 }
 
 void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList) {
+    // since we're called from the DomainServerSettingsManager constructor, we don't take a write lock here
+    // even though we change the underlying config map
+
     _argumentList = argumentList;
 
     _configMap.loadConfig(_argumentList);
@@ -390,6 +398,7 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
             _standardAgentPermissions[NodePermissions::standardNameLocalhost]->set(NodePermissions::Permission::canRezTemporaryCertifiedEntities);
             packPermissions();
         }
+
         if (oldVersion < 2.0) {
             const QString WIZARD_COMPLETED_ONCE = "wizard.completed_once";
 
@@ -397,6 +406,7 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
 
             *wizardCompletedOnce = QVariant(true);
         }
+
         if (oldVersion < 2.1) {
             // convert old avatar scale settings into avatar height.
 
@@ -418,6 +428,21 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
             }
         }
 
+        if (oldVersion < 2.2) {
+            // migrate entity server rolling backup intervals to new location for automatic content archive intervals
+
+            const QString ENTITY_SERVER_BACKUPS_KEYPATH = "entity_server_settings.backups";
+            const QString AUTO_CONTENT_ARCHIVES_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
+
+            QVariant* previousBackupsVariant = _configMap.valueForKeyPath(ENTITY_SERVER_BACKUPS_KEYPATH);
+
+            if (previousBackupsVariant) {
+                auto migratedBackupsVariant = _configMap.valueForKeyPath(AUTO_CONTENT_ARCHIVES_RULES_KEYPATH, true);
+                *migratedBackupsVariant = *previousBackupsVariant;
+            }
+        }
+
+
         // write the current description version to our settings
         *versionVariant = _descriptionVersion;
 
@@ -428,17 +453,6 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
     unpackPermissions();
 }
 
-QVariantMap& DomainServerSettingsManager::getDescriptorsMap() {
-    static const QString DESCRIPTORS{ "descriptors" };
-
-    auto& settingsMap = getSettingsMap();
-    if (!getSettingsMap().contains(DESCRIPTORS)) {
-        settingsMap.insert(DESCRIPTORS, QVariantMap());
-    }
-
-    return *static_cast<QVariantMap*>(getSettingsMap()[DESCRIPTORS].data());
-}
-
 void DomainServerSettingsManager::initializeGroupPermissions(NodePermissionsMap& permissionsRows,
                                                              QString groupName, NodePermissionsPointer perms) {
     // this is called when someone has used the domain-settings webpage to add a group.  They type the group's name
@@ -467,6 +481,9 @@ void DomainServerSettingsManager::initializeGroupPermissions(NodePermissionsMap&
 void DomainServerSettingsManager::packPermissionsForMap(QString mapName,
                                                         NodePermissionsMap& permissionsRows,
                                                         QString keyPath) {
+    // grab a write lock on the settings mutex since we're about to change the config map
+    QWriteLocker locker(&_settingsLock);
+
     // find (or create) the "security" section of the settings map
     QVariant* security = _configMap.valueForKeyPath("security", true);
     if (!security->canConvert(QMetaType::QVariantMap)) {
@@ -556,15 +573,20 @@ bool DomainServerSettingsManager::unpackPermissionsForKeypath(const QString& key
 
     mapPointer->clear();
 
-    QVariant* permissions = _configMap.valueForKeyPath(keyPath, true);
-    if (!permissions->canConvert(QMetaType::QVariantList)) {
+    QVariant permissions = valueOrDefaultValueForKeyPath(keyPath);
+
+    if (!permissions.isValid()) {
+        // we don't have a permissions object to unpack for this keypath, bail
+        return false;
+    }
+
+    if (!permissions.canConvert(QMetaType::QVariantList)) {
         qDebug() << "Failed to extract permissions for key path" << keyPath << "from settings.";
-        (*permissions) = QVariantList();
     }
 
     bool needPack = false;
 
-    QList<QVariant> permissionsList = permissions->toList();
+    QList<QVariant> permissionsList = permissions.toList();
     foreach (QVariant permsHash, permissionsList) {
         NodePermissionsPointer perms { new NodePermissions(permsHash.toMap()) };
         QString id = perms->getID();
@@ -591,6 +613,11 @@ bool DomainServerSettingsManager::unpackPermissionsForKeypath(const QString& key
 void DomainServerSettingsManager::unpackPermissions() {
     // transfer details from _configMap to _agentPermissions
 
+    // NOTE: Defaults for standard permissions (anonymous, friends, localhost, logged-in) used
+    // to be set here and then immediately persisted to the config JSON file.
+    // They have since been moved to describe-settings.json as the default value for AGENT_STANDARD_PERMISSIONS_KEYPATH.
+    // In order to change the default standard permissions you must change the default value in describe-settings.json.
+
     bool needPack = false;
 
     needPack |= unpackPermissionsForKeypath(AGENT_STANDARD_PERMISSIONS_KEYPATH, &_standardAgentPermissions);
@@ -650,57 +677,39 @@ void DomainServerSettingsManager::unpackPermissions() {
             }
     });
 
-    // if any of the standard names are missing, add them
-    foreach(const QString& standardName, NodePermissions::standardNames) {
-        NodePermissionsKey standardKey { standardName, 0 };
-        if (!_standardAgentPermissions.contains(standardKey)) {
-            // we don't have permissions for one of the standard groups, so we'll add them now
-            NodePermissionsPointer perms { new NodePermissions(standardKey) };
-
-            if (standardKey == NodePermissions::standardNameLocalhost) {
-                // the localhost user is granted all permissions by default
-                perms->setAll(true);
-            } else {
-                // anonymous, logged in, and friend users get connect permissions by default
-                perms->set(NodePermissions::Permission::canConnectToDomain);
-                perms->set(NodePermissions::Permission::canRezTemporaryCertifiedEntities);
-            }
-
-            // add the permissions to the standard map
-            _standardAgentPermissions[standardKey] = perms;
-
-            // this will require a packing of permissions
-            needPack = true;
-        }
-    }
-
     needPack |= ensurePermissionsForGroupRanks();
 
     if (needPack) {
         packPermissions();
     }
 
-    #ifdef WANT_DEBUG
+#ifdef WANT_DEBUG
     qDebug() << "--------------- permissions ---------------------";
-    QList<QHash<NodePermissionsKey, NodePermissionsPointer>> permissionsSets;
-    permissionsSets << _standardAgentPermissions.get() << _agentPermissions.get()
-                    << _groupPermissions.get() << _groupForbiddens.get()
-                    << _ipPermissions.get() << _macPermissions.get()
-                    << _machineFingerprintPermissions.get();
+    std::array<NodePermissionsMap*, 7> permissionsSets {{
+        &_standardAgentPermissions, &_agentPermissions,
+        &_groupPermissions, &_groupForbiddens,
+        &_ipPermissions, &_macPermissions,
+        &_machineFingerprintPermissions
+    }};
 
     foreach (auto permissionSet, permissionsSets) {
-        QHashIterator<NodePermissionsKey, NodePermissionsPointer> i(permissionSet);
-        while (i.hasNext()) {
-            i.next();
-            NodePermissionsPointer perms = i.value();
+        auto& permissionKeyMap = permissionSet->get();
+        auto it = permissionKeyMap.begin();
+
+        while (it != permissionKeyMap.end()) {
+
+            NodePermissionsPointer perms = it->second;
             if (perms->isGroup()) {
-                qDebug() << i.key() << perms->getGroupID() << perms;
+                qDebug() << it->first << perms->getGroupID() << perms;
             } else {
-                qDebug() << i.key() << perms;
+                qDebug() << it->first << perms;
             }
+
+            ++it;
         }
     }
-    #endif
+#endif
+
 }
 
 bool DomainServerSettingsManager::ensurePermissionsForGroupRanks() {
@@ -1048,12 +1057,22 @@ NodePermissions DomainServerSettingsManager::getForbiddensForGroup(const QUuid&
     return getForbiddensForGroup(groupKey.first, groupKey.second);
 }
 
+QVariant DomainServerSettingsManager::valueForKeyPath(const QString& keyPath) {
+    QReadLocker locker(&_settingsLock);
+    auto foundValue = _configMap.valueForKeyPath(keyPath);
+    return foundValue ? *foundValue : QVariant();
+}
+
 QVariant DomainServerSettingsManager::valueOrDefaultValueForKeyPath(const QString& keyPath) {
+    QReadLocker locker(&_settingsLock);
     const QVariant* foundValue = _configMap.valueForKeyPath(keyPath);
 
     if (foundValue) {
         return *foundValue;
     } else {
+        // we don't need the settings lock anymore since we're done reading from the config map
+        locker.unlock();
+
         int dotIndex = keyPath.indexOf('.');
 
         QString groupKey = keyPath.mid(0, dotIndex);
@@ -1092,9 +1111,6 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
             // we recurse one level deep below each group for the appropriate setting
             bool restartRequired = recurseJSONObjectAndOverwriteSettings(postedObject, endpointType);
 
-            // store whatever the current _settingsMap is to file
-            persistToFile();
-
             // return success to the caller
             QString jsonSuccess = "{\"status\": \"success\"}";
             connection->respond(HTTPConnection::StatusCode200, jsonSuccess.toUtf8(), "application/json");
@@ -1152,17 +1168,20 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
 
             QJsonObject rootObject;
 
-            bool forDomainSettings = (url.path() == SETTINGS_PATH_JSON);
-            bool forContentSettings = (url.path() == CONTENT_SETTINGS_PATH_JSON);;
+            DomainSettingsInclusion domainSettingsInclusion = (url.path() == SETTINGS_PATH_JSON)
+                ? IncludeDomainSettings : NoDomainSettings;
+            ContentSettingsInclusion contentSettingsInclusion = (url.path() == CONTENT_SETTINGS_PATH_JSON)
+                ? IncludeContentSettings : NoContentSettings;
 
-            rootObject[SETTINGS_RESPONSE_DESCRIPTION_KEY] = forDomainSettings
+            rootObject[SETTINGS_RESPONSE_DESCRIPTION_KEY] = (url.path() == SETTINGS_PATH_JSON)
                 ? _domainSettingsDescription : _contentSettingsDescription;
 
             // grab a domain settings object for all types, filtered for the right class of settings
             // and exclude default values
-            rootObject[SETTINGS_RESPONSE_VALUE_KEY] = settingsResponseObjectForType("", true,
-                                                                                    forDomainSettings, forContentSettings,
-                                                                                    true);
+            rootObject[SETTINGS_RESPONSE_VALUE_KEY] = settingsResponseObjectForType("", Authenticated,
+                                                                                    domainSettingsInclusion,
+                                                                                    contentSettingsInclusion,
+                                                                                    IncludeDefaultSettings);
 
             connection->respond(HTTPConnection::StatusCode200, QJsonDocument(rootObject).toJson(), "application/json");
 
@@ -1174,7 +1193,8 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
         } else if (url.path() == SETTINGS_BACKUP_PATH) {
             // grab the settings backup as an authenticated user
             // for the domain settings type only, excluding hidden and default values
-            auto currentDomainSettingsJSON = settingsResponseObjectForType("", true, true, false, false, true);
+            auto currentDomainSettingsJSON = settingsResponseObjectForType("", Authenticated, IncludeDomainSettings,
+                                                                           NoContentSettings, NoDefaultSettings, ForBackup);
 
             // setup headers that tell the client to download the file wth a special name
             Headers downloadHeaders;
@@ -1196,6 +1216,10 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
 }
 
 bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType) {
+
+    // grab a write lock since we're about to change the settings map
+    QWriteLocker locker(&_settingsLock);
+
     QJsonArray* filteredDescriptionArray = settingsType == DomainSettings
         ? &_domainSettingsDescription : &_contentSettingsDescription;
 
@@ -1277,6 +1301,9 @@ bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settings
                     }
                 } else {
                     // we have a value to restore, use update setting to set it
+                    // but clear the existing value first so that no merging between the restored settings
+                    // and existing settings occurs
+                    variantValue->clear();
 
                     // we might need to re-grab config group map in case it didn't exist when we looked for it before
                     // but was created by the call to valueForKeyPath before
@@ -1310,18 +1337,24 @@ bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settings
     } else {
         // restore completed, persist the new settings
         qDebug() << "Restore completed, persisting restored settings to file";
+
+        // let go of the write lock since we're done making changes to the config map
+        locker.unlock();
+
         persistToFile();
         return true;
     }
 }
 
-QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QString& typeValue, bool isAuthenticated,
-                                                                       bool includeDomainSettings,
-                                                                       bool includeContentSettings,
-                                                                       bool includeDefaults, bool isForBackup) {
+QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QString& typeValue,
+                                                                       SettingsRequestAuthentication authentication,
+                                                                       DomainSettingsInclusion domainSettingsInclusion,
+                                                                       ContentSettingsInclusion contentSettingsInclusion,
+                                                                       DefaultSettingsInclusion defaultSettingsInclusion,
+                                                                       SettingsBackupFlag settingsBackupFlag) {
     QJsonObject responseObject;
 
-    if (!typeValue.isEmpty() || isAuthenticated) {
+    if (!typeValue.isEmpty() || authentication == Authenticated) {
         // convert the string type value to a QJsonValue
         QJsonValue queryType = typeValue.isEmpty() ? QJsonValue() : QJsonValue(typeValue.toInt());
 
@@ -1329,9 +1362,10 @@ QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QSt
 
         // only enumerate the requested settings type (domain setting or content setting)
         QJsonArray* filteredDescriptionArray = &_descriptionArray;
-        if (includeDomainSettings && !includeContentSettings) {
+
+        if (domainSettingsInclusion == IncludeDomainSettings && contentSettingsInclusion != IncludeContentSettings) {
             filteredDescriptionArray = &_domainSettingsDescription;
-        } else if (includeContentSettings && !includeDomainSettings) {
+        } else if (contentSettingsInclusion == IncludeContentSettings && domainSettingsInclusion != IncludeDomainSettings) {
             filteredDescriptionArray = &_contentSettingsDescription;
         }
 
@@ -1354,35 +1388,35 @@ QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QSt
                 bool includedInBackups = !settingObject.contains(DESCRIPTION_BACKUP_FLAG_KEY)
                     || settingObject[DESCRIPTION_BACKUP_FLAG_KEY].toBool();
 
-                if (!settingObject[VALUE_HIDDEN_FLAG_KEY].toBool() && (!isForBackup || includedInBackups)) {
+                if (!settingObject[VALUE_HIDDEN_FLAG_KEY].toBool() && (settingsBackupFlag != ForBackup || includedInBackups)) {
                     QJsonArray affectedTypesArray = settingObject[AFFECTED_TYPES_JSON_KEY].toArray();
                     if (affectedTypesArray.isEmpty()) {
                         affectedTypesArray = groupObject[AFFECTED_TYPES_JSON_KEY].toArray();
                     }
 
                     if (affectedTypesArray.contains(queryType) ||
-                        (queryType.isNull() && isAuthenticated)) {
+                        (queryType.isNull() && authentication == Authenticated)) {
                         QString settingName = settingObject[DESCRIPTION_NAME_KEY].toString();
 
                         // we need to check if the settings map has a value for this setting
                         QVariant variantValue;
 
                         if (!groupKey.isEmpty()) {
-                             QVariant settingsMapGroupValue = _configMap.value(groupKey);
+                             QVariant settingsMapGroupValue = valueForKeyPath(groupKey);
 
                             if (!settingsMapGroupValue.isNull()) {
                                 variantValue = settingsMapGroupValue.toMap().value(settingName);
                             }
                         } else {
-                            variantValue = _configMap.value(settingName);
+                            variantValue = valueForKeyPath(settingName);
                         }
 
                         // final check for inclusion
                         // either we include default values or we don't but this isn't a default value
-                        if (includeDefaults || !variantValue.isNull()) {
+                        if ((defaultSettingsInclusion == IncludeDefaultSettings) || variantValue.isValid()) {
                             QJsonValue result;
 
-                            if (variantValue.isNull()) {
+                            if (!variantValue.isValid()) {
                                 // no value for this setting, pass the default
                                 if (settingObject.contains(SETTING_DEFAULT_KEY)) {
                                     result = settingObject[SETTING_DEFAULT_KEY];
@@ -1521,6 +1555,10 @@ QJsonObject DomainServerSettingsManager::settingDescriptionFromGroup(const QJson
 
 bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject,
                                                                         SettingsType settingsType) {
+
+    // take a write lock since we're about to overwrite settings in the config map
+    QWriteLocker locker(&_settingsLock);
+
     static const QString SECURITY_ROOT_KEY = "security";
     static const QString AC_SUBNET_WHITELIST_KEY = "ac_subnet_whitelist";
     static const QString BROADCASTING_KEY = "broadcasting";
@@ -1618,6 +1656,12 @@ bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJ
         }
     }
 
+    // we're done making changes to the config map, let go of our read lock
+    locker.unlock();
+
+    // store whatever the current config map is to file
+    persistToFile();
+
     return needRestart;
 }
 
@@ -1644,6 +1688,9 @@ bool permissionVariantLessThan(const QVariant &v1, const QVariant &v2) {
 }
 
 void DomainServerSettingsManager::sortPermissions() {
+    // take a write lock since we're about to change the config map data
+    QWriteLocker locker(&_settingsLock);
+
     // sort the permission-names
     QVariant* standardPermissions = _configMap.valueForKeyPath(AGENT_STANDARD_PERMISSIONS_KEYPATH);
     if (standardPermissions && standardPermissions->canConvert(QMetaType::QVariantList)) {
@@ -1680,11 +1727,15 @@ void DomainServerSettingsManager::persistToFile() {
     QFile settingsFile(_configMap.getUserConfigFilename());
 
     if (settingsFile.open(QIODevice::WriteOnly)) {
+        // take a read lock so we can grab the config and write it to file
+        QReadLocker locker(&_settingsLock);
         settingsFile.write(QJsonDocument::fromVariant(_configMap.getConfig()).toJson());
     } else {
         qCritical("Could not write to JSON settings file. Unable to persist settings.");
 
         // failed to write, reload whatever the current config state is
+        // with a write lock since we're about to overwrite the config map
+        QWriteLocker locker(&_settingsLock);
         _configMap.loadConfig(_argumentList);
     }
 }

domain-server/src/DomainServerSettingsManager.h

@@ -27,9 +27,6 @@
 
 const QString SETTINGS_PATHS_KEY = "paths";
 
-const QString SETTINGS_PATH = "/settings";
-const QString SETTINGS_PATH_JSON = SETTINGS_PATH + ".json";
-const QString CONTENT_SETTINGS_PATH_JSON = "/content-settings.json";
 const QString AGENT_STANDARD_PERMISSIONS_KEYPATH = "security.standard_permissions";
 const QString AGENT_PERMISSIONS_KEYPATH = "security.permissions";
 const QString IP_PERMISSIONS_KEYPATH = "security.ip_permissions";
@@ -37,6 +34,7 @@ const QString MAC_PERMISSIONS_KEYPATH = "security.mac_permissions";
 const QString MACHINE_FINGERPRINT_PERMISSIONS_KEYPATH = "security.machine_fingerprint_permissions";
 const QString GROUP_PERMISSIONS_KEYPATH = "security.group_permissions";
 const QString GROUP_FORBIDDENS_KEYPATH = "security.group_forbiddens";
+const QString AUTOMATIC_CONTENT_ARCHIVES_GROUP = "automatic_content_archives";
 
 using GroupByUUIDKey = QPair<QUuid, QUuid>; // groupID, rankID
 
@@ -52,11 +50,12 @@ public:
     bool handleAuthenticatedHTTPRequest(HTTPConnection* connection, const QUrl& url);
 
     void setupConfigMap(const QStringList& argumentList);
+
+    // each of the three methods in this group takes a read lock of _settingsLock
+    // and cannot be called when the a write lock is held by the same thread
     QVariant valueOrDefaultValueForKeyPath(const QString& keyPath);
-
-    QVariantMap& getSettingsMap() { return _configMap.getConfig(); }
-
-    QVariantMap& getDescriptorsMap();
+    QVariant valueForKeyPath(const QString& keyPath);
+    bool containsKeyPath(const QString& keyPath) { return valueForKeyPath(keyPath).isValid(); }
 
     // these give access to anonymous/localhost/logged-in settings from the domain-server settings page
     bool haveStandardPermissionsForName(const QString& name) const { return _standardAgentPermissions.contains(name, 0); }
@@ -111,6 +110,24 @@ public:
 
     void debugDumpGroupsState();
 
+    enum SettingsRequestAuthentication { NotAuthenticated, Authenticated };
+    enum DomainSettingsInclusion { NoDomainSettings, IncludeDomainSettings };
+    enum ContentSettingsInclusion { NoContentSettings, IncludeContentSettings };
+    enum DefaultSettingsInclusion { NoDefaultSettings, IncludeDefaultSettings };
+    enum SettingsBackupFlag { NotForBackup, ForBackup };
+
+    /// thread safe method to retrieve a JSON representation of settings
+    QJsonObject settingsResponseObjectForType(const QString& typeValue,
+                                              SettingsRequestAuthentication authentication = NotAuthenticated,
+                                              DomainSettingsInclusion domainSettingsInclusion = IncludeDomainSettings,
+                                              ContentSettingsInclusion contentSettingsInclusion = IncludeContentSettings,
+                                              DefaultSettingsInclusion defaultSettingsInclusion = IncludeDefaultSettings,
+                                              SettingsBackupFlag settingsBackupFlag = NotForBackup);
+    /// thread safe method to restore settings from a JSON object
+    Q_INVOKABLE bool restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType);
+
+    bool recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject, SettingsType settingsType);
+
 signals:
     void updateNodePermissions();
     void settingsUpdated();
@@ -130,21 +147,17 @@ private:
     QStringList _argumentList;
 
     QJsonArray filteredDescriptionArray(bool isContentSettings);
-    QJsonObject settingsResponseObjectForType(const QString& typeValue, bool isAuthenticated = false,
-                                              bool includeDomainSettings = true, bool includeContentSettings = true,
-                                              bool includeDefaults = true, bool isForBackup = false);
-    bool recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject, SettingsType settingsType);
-
     void updateSetting(const QString& key, const QJsonValue& newValue, QVariantMap& settingMap,
                        const QJsonObject& settingDescription);
     QJsonObject settingDescriptionFromGroup(const QJsonObject& groupObject, const QString& settingName);
     void sortPermissions();
+
+    // you cannot be holding the _settingsLock when persisting to file from the same thread
+    // since it may take either a read lock or write lock and recursive locking doesn't allow a change in type
     void persistToFile();
 
     void splitSettingsDescription();
 
-    bool restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType);
-
     double _descriptionVersion;
 
     QJsonArray _descriptionArray;
@@ -152,10 +165,10 @@ private:
     QJsonArray _contentSettingsDescription;
     QJsonObject _settingsMenuGroups;
 
+    // any method that calls valueForKeyPath on this _configMap must get a write lock it keeps until it
+    // is done with the returned QVariant*
     HifiConfigVariantMap _configMap;
 
-    friend class DomainServer;
-
     // these cause calls to metaverse's group api
     void apiGetGroupID(const QString& groupName);
     void apiGetGroupRanks(const QUuid& groupID);
@@ -189,6 +202,9 @@ private:
 
     // keep track of answers to api queries about which users are in which groups
     QHash<QString, QHash<QUuid, QUuid>> _groupMembership; // QHash<user-name, QHash<group-id, rank-id>>
+
+    /// guard read/write access from multiple threads to settings 
+    QReadWriteLock _settingsLock { QReadWriteLock::Recursive };
 };
 
 #endif // hifi_DomainServerSettingsManager_h

domain-server/src/EntitiesBackupHandler.cpp

@@ -0,0 +1,83 @@
+//
+//  EntitiesBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/14/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntitiesBackupHandler.h"
+
+#include <QDebug>
+
+#include <quazip5/quazip.h>
+#include <quazip5/quazipfile.h>
+
+#include <OctreeDataUtils.h>
+
+EntitiesBackupHandler::EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath) :
+    _entitiesFilePath(entitiesFilePath),
+    _entitiesReplacementFilePath(entitiesReplacementFilePath)
+{
+}
+
+static const QString ENTITIES_BACKUP_FILENAME = "models.json.gz";
+
+void EntitiesBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+    QFile entitiesFile { _entitiesFilePath };
+
+    if (entitiesFile.open(QIODevice::ReadOnly)) {
+        QuaZipFile zipFile { &zip };
+        if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(ENTITIES_BACKUP_FILENAME, _entitiesFilePath))) {
+            qCritical().nospace() << "Failed to open " << ENTITIES_BACKUP_FILENAME << " for writing in zip";
+            return;
+        }
+        auto entityData = entitiesFile.readAll();
+        if (zipFile.write(entityData) != entityData.size()) {
+            qCritical() << "Failed to write entities file to backup";
+            zipFile.close();
+            return;
+        }
+        zipFile.close();
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCritical().nospace() << "Failed to zip " << ENTITIES_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+    }
+}
+
+void EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    if (!zip.setCurrentFile(ENTITIES_BACKUP_FILENAME)) {
+        qWarning() << "Failed to find" << ENTITIES_BACKUP_FILENAME << "while recovering backup";
+        return;
+    }
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::ReadOnly)) {
+        qCritical() << "Failed to open" << ENTITIES_BACKUP_FILENAME << "in backup";
+        return;
+    }
+    auto rawData = zipFile.readAll();
+
+    zipFile.close();
+
+    OctreeUtils::RawEntityData data;
+    if (!data.readOctreeDataInfoFromData(rawData)) {
+        qCritical() << "Unable to parse octree data during backup recovery";
+        return;
+    }
+
+    data.resetIdAndVersion();
+
+    if (zipFile.getZipError() != UNZ_OK) {
+        qCritical().nospace() << "Failed to unzip " << ENTITIES_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        return;
+    }
+
+    QFile entitiesFile { _entitiesReplacementFilePath };
+
+    if (entitiesFile.open(QIODevice::WriteOnly)) {
+        entitiesFile.write(data.toGzippedByteArray());
+    }
+}

domain-server/src/EntitiesBackupHandler.h

@@ -0,0 +1,47 @@
+//
+//  EntitiesBackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/14/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_EntitiesBackupHandler_h
+#define hifi_EntitiesBackupHandler_h
+
+#include "BackupHandler.h"
+
+class EntitiesBackupHandler : public BackupHandlerInterface {
+public:
+    EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override { return { true, 1.0f }; }
+    std::pair<bool, float> getRecoveryStatus() override { return { false, 1.0f }; }
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override {}
+
+    void loadingComplete() override {}
+
+    // Create a skeleton backup
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+
+    // Recover from a full backup
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+
+    // Delete a skeleton backup
+    void deleteBackup(const QString& backupName) override {}
+
+    // Create a full backup
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override {}
+
+    bool isCorruptedBackup(const QString& backupName) override { return false; }
+
+private:
+    QString _entitiesFilePath;
+    QString _entitiesReplacementFilePath;
+};
+
+#endif /* hifi_EntitiesBackupHandler_h */

domain-server/src/main.cpp

@@ -22,22 +22,10 @@
 #include "DomainServer.h"
 
 int main(int argc, char* argv[]) {
-    disableQtBearerPoll(); // Fixes wifi ping spikes
-
-    QCoreApplication::setApplicationName(BuildInfo::DOMAIN_SERVER_NAME);
-    QCoreApplication::setOrganizationName(BuildInfo::MODIFIED_ORGANIZATION);
-    QCoreApplication::setOrganizationDomain(BuildInfo::ORGANIZATION_DOMAIN);
-    QCoreApplication::setApplicationVersion(BuildInfo::VERSION);
+    setupHifiApplication(BuildInfo::DOMAIN_SERVER_NAME);
 
     Setting::init();
 
-#ifndef WIN32
-    setvbuf(stdout, NULL, _IOLBF, 0);
-#endif
-
-    qInstallMessageHandler(LogHandler::verboseMessageHandler);
-    qInfo() << "Starting.";
-
     int currentExitCode = 0;
 
     // use a do-while to handle domain-server restart

ice-server/src/main.cpp

@@ -11,18 +11,13 @@
 
 #include <QtCore/QCoreApplication>
 
-#include <LogHandler.h>
+#include <SharedUtil.h>
 
 #include "IceServer.h"
 
 int main(int argc, char* argv[]) {
-#ifndef WIN32
-    setvbuf(stdout, NULL, _IOLBF, 0);
-#endif
-
-    qInstallMessageHandler(LogHandler::verboseMessageHandler);
-    qInfo() << "Starting.";
+    setupHifiApplication("Ice Server");
     
     IceServer iceServer(argc, argv);
     return iceServer.exec();
-}
+}

interface/CMakeLists.txt

@@ -204,7 +204,7 @@ endif()
 
 # link required hifi libraries
 link_hifi_libraries(
-  shared task octree ktx gpu gl procedural graphics render
+  shared task octree ktx gpu gl procedural graphics graphics-scripting render
   pointers
   recording fbx networking model-networking entities avatars trackers
   audio audio-client animation script-engine physics

interface/resources/qml/hifi/commerce/checkout/Checkout.qml

@@ -59,17 +59,17 @@ Rectangle {
                 if (root.activeView !== "needsLogIn") {
                     root.activeView = "needsLogIn";
                 }
-            } else if (walletStatus === 1) {
+            } else if ((walletStatus === 1) || (walletStatus === 2) || (walletStatus === 3)) {
                 if (root.activeView !== "notSetUp") {
                     root.activeView = "notSetUp";
                     notSetUpTimer.start();
                 }
-            } else if (walletStatus === 2) {
+            } else if (walletStatus === 4) {
                 if (root.activeView !== "passphraseModal") {
                     root.activeView = "passphraseModal";
                     UserActivityLogger.commercePassphraseEntry("marketplace checkout");
                 }
-            } else if (walletStatus === 3) {
+            } else if (walletStatus === 5) {
                 authSuccessStep();
             } else {
                 console.log("ERROR in Checkout.qml: Unknown wallet status: " + walletStatus);

interface/resources/qml/hifi/commerce/common/CommerceLightbox.qml

@@ -25,10 +25,13 @@ Rectangle {
     property string titleText;
     property string bodyImageSource;
     property string bodyText;
+    property string button1color: hifi.buttons.noneBorderlessGray;
     property string button1text;
     property string button1method;
+    property string button2color: hifi.buttons.noneBorderless;
     property string button2text;
     property string button2method;
+    property string buttonLayout: "leftright";
 
     readonly property string securityPicBodyText: "When you see your Security Pic, your actions and data are securely making use of your " +
         "Wallet's private keys.<br><br>You can change your Security Pic in your Wallet.";
@@ -39,6 +42,12 @@ Rectangle {
     color: Qt.rgba(0, 0, 0, 0.5);
     z: 999;
 
+    onVisibleChanged: {
+        if (!visible) {
+            resetLightbox();
+        }
+    }
+
     // This object is always used in a popup.
     // This MouseArea is used to prevent a user from being
     //     able to click on a button/mouseArea underneath the popup.
@@ -112,18 +121,21 @@ Rectangle {
             anchors.topMargin: 30;
             anchors.left: parent.left;
             anchors.right: parent.right;
-            height: 70;
+            height: root.buttonLayout === "leftright" ? 70 : 150;
 
             // Button 1
             HifiControlsUit.Button {
-                color: hifi.buttons.noneBorderlessGray;
+                id: button1;
+                color: root.button1color;
                 colorScheme: hifi.colorSchemes.light;
-                anchors.top: parent.top;
-                anchors.bottom: parent.bottom;
-                anchors.bottomMargin: 20;
+                anchors.top: root.buttonLayout === "leftright" ? parent.top : parent.top;
                 anchors.left: parent.left;
                 anchors.leftMargin: 10;
-                width: root.button2text ? parent.width/2 - anchors.leftMargin*2 : parent.width - anchors.leftMargin * 2;
+                anchors.right: root.buttonLayout === "leftright" ? undefined : parent.right;
+                anchors.rightMargin: root.buttonLayout === "leftright" ? undefined : 10;
+                width: root.buttonLayout === "leftright" ? (root.button2text ? parent.width/2 - anchors.leftMargin*2 : parent.width - anchors.leftMargin * 2) :
+                    (undefined);
+                height: 50;
                 text: root.button1text;
                 onClicked: {
                     eval(button1method);
@@ -132,15 +144,18 @@ Rectangle {
 
             // Button 2
             HifiControlsUit.Button {
+                id: button2;
                 visible: root.button2text;
-                color: hifi.buttons.noneBorderless;
+                color: root.button2color;
                 colorScheme: hifi.colorSchemes.light;
-                anchors.top: parent.top;
-                anchors.bottom: parent.bottom;
-                anchors.bottomMargin: 20;
+                anchors.top: root.buttonLayout === "leftright" ? parent.top : button1.bottom;
+                anchors.topMargin: root.buttonLayout === "leftright" ? undefined : 20;
+                anchors.left: root.buttonLayout === "leftright" ? undefined : parent.left;
+                anchors.leftMargin: root.buttonLayout === "leftright" ? undefined : 10;
                 anchors.right: parent.right;
                 anchors.rightMargin: 10;
-                width: parent.width/2 - anchors.rightMargin*2;
+                width: root.buttonLayout === "leftright" ? parent.width/2 - anchors.rightMargin*2 : undefined;
+                height: 50;
                 text: root.button2text;
                 onClicked: {
                     eval(button2method);
@@ -153,6 +168,19 @@ Rectangle {
     // FUNCTION DEFINITIONS START
     //
     signal sendToParent(var msg);
+
+    function resetLightbox() {
+        root.titleText = "";
+        root.bodyImageSource = "";
+        root.bodyText = "";
+        root.button1color = hifi.buttons.noneBorderlessGray;
+        root.button1text = "";
+        root.button1method = "";
+        root.button2color = hifi.buttons.noneBorderless;
+        root.button2text = "";
+        root.button2method = "";
+        root.buttonLayout = "leftright";
+    }
     //
     // FUNCTION DEFINITIONS END
     //

interface/resources/qml/hifi/commerce/common/EmulatedMarketplaceHeader.qml

@@ -37,9 +37,9 @@ Item {
         onWalletStatusResult: {
             if (walletStatus === 0) {
                 sendToParent({method: "needsLogIn"});
-            } else if (walletStatus === 3) {
+            } else if (walletStatus === 5) {
                 Commerce.getSecurityImage();
-            } else if (walletStatus > 3) {
+            } else if (walletStatus > 5) {
                 console.log("ERROR in EmulatedMarketplaceHeader.qml: Unknown wallet status: " + walletStatus);
             }
         }

interface/resources/qml/hifi/commerce/purchases/Purchases.qml

@@ -47,17 +47,17 @@ Rectangle {
                 if (root.activeView !== "needsLogIn") {
                     root.activeView = "needsLogIn";
                 }
-            } else if (walletStatus === 1) {
+            } else if ((walletStatus === 1) || (walletStatus === 2) || (walletStatus === 3)) {
                 if (root.activeView !== "notSetUp") {
                     root.activeView = "notSetUp";
                     notSetUpTimer.start();
                 }
-            } else if (walletStatus === 2) {
+            } else if (walletStatus === 4) {
                 if (root.activeView !== "passphraseModal") {
                     root.activeView = "passphraseModal";
                     UserActivityLogger.commercePassphraseEntry("marketplace purchases");
                 }
-            } else if (walletStatus === 3) {
+            } else if (walletStatus === 5) {
                 if ((Settings.getValue("isFirstUseOfPurchases", true) || root.isDebuggingFirstUseTutorial) && root.activeView !== "firstUseTutorial") {
                     root.activeView = "firstUseTutorial";
                 } else if (!Settings.getValue("isFirstUseOfPurchases", true) && root.activeView === "initialize") {

interface/resources/qml/hifi/commerce/wallet/Help.qml

@@ -60,48 +60,88 @@ Item {
 
         ListElement {
             isExpanded: false;
-            question: "How can I get HFC?"
-            answer: qsTr("High Fidelity commerce is in closed beta right now.<br><br>To request entry and get free HFC, <b>please contact info@highfidelity.com with your High Fidelity account username and the email address registered to that account.</b>");
+            question: "How can I get HFC?";
+            answer: "High Fidelity commerce is in open beta right now. Want more HFC? \
+Get it by going to <br><br><b><font color='#0093C5'><a href='#bank'>BankOfHighFidelity.</a></font></b> and meeting with the banker!";
         }
         ListElement {
             isExpanded: false;
-            question: "What are private keys?"
-            answer: qsTr("A private key is a secret piece of text that is used to prove ownership, unlock confidential information, and sign transactions.<br><br>In High Fidelity, <b>your private keys are used to securely access the contents of your Wallet and Purchases.</b>");
+            question: "What are private keys and where are they stored?";
+            answer: 
+                "A private key is a secret piece of text that is used to prove ownership, unlock confidential information, and sign transactions. \
+In High Fidelity, your private key is used to securely access the contents of your Wallet and Purchases. \
+After wallet setup, a hifikey file is stored on your computer in High Fidelity Interface's AppData directory. \
+Your hifikey file contains your private key and is protected by your wallet passphrase. \
+<br><br>It is very important to back up your hifikey file! \
+<b><font color='#0093C5'><a href='#privateKeyPath'>Tap here to open the folder where your HifiKeys are stored on your main display.</a></font></b>"
         }
         ListElement {
             isExpanded: false;
-            question: "Where are my private keys stored?"
-            answer: qsTr('By default, your private keys are <b>only stored on your hard drive</b> in High Fidelity Interface\'s AppData directory.<br><br><b><font color="#0093C5"><a href="#privateKeyPath">Tap here to open the folder where your HifiKeys are stored on your main display.</a></font></b>');
+            question: "How do I back up my private keys?";
+            answer: "You can back up your hifikey file (which contains your private key and is encrypted using your wallet passphrase) by copying it to a USB flash drive, or to a service like Dropbox or Google Drive. \
+Restore your hifikey file by replacing the file in Interface's AppData directory with your backup copy. \
+Others with access to your back up should not be able to spend your HFC without your passphrase. \
+<b><font color='#0093C5'><a href='#privateKeyPath'>Tap here to open the folder where your HifiKeys are stored on your main display.</a></font></b>";
         }
         ListElement {
             isExpanded: false;
-            question: "How can I backup my private keys?"
-            answer: qsTr('You may backup the file containing your private keys by copying it to a USB flash drive, or to a service like Dropbox or Google Drive.<br><br>Restore your backup by replacing the file in Interface\'s AppData directory with your backed-up copy.<br><br><b><font color="#0093C5"><a href="#privateKeyPath">Tap here to open the folder where your HifiKeys are stored on your main display.</a></font></b>');
+            question: "What happens if I lose my private keys?";
+            answer: "We cannot stress enough that you should keep a backup! For security reasons, High Fidelity does not keep a copy, and cannot restore it for you. \
+If you lose your private key, you will no longer have access to the contents of your Wallet or My Purchases. \
+Here are some things to try:<ul>\
+<li>If you have backed up your hifikey file before, search your backup location</li>\
+<li>Search your AppData directory in the last machine you used to set up the Wallet</li>\
+<li>If you are a developer and have installed multiple builds of High Fidelity, your hifikey file might be in another folder</li>\
+</ul><br><br>As a last resort, you can set up your Wallet again and generate a new hifikey file. \
+Unfortunately, this means you will start with 0 HFC and your purchased items will not be transferred over.";
         }
         ListElement {
             isExpanded: false;
-            question: "What happens if I lose my passphrase?"
-            answer: qsTr("Your passphrase is used to encrypt your private keys. If you lose your passphrase, you will no longer be able to decrypt your private key file. You will also no longer have access to the contents of your Wallet or My Purchases.<br><br><b>Nobody can help you recover your passphrase, including High Fidelity.</b> Please write it down and store it securely.");
+            question: "What if I forget my wallet passphrase?";
+            answer: "Your wallet passphrase is used to encrypt your private keys. Please write it down and store it securely! \
+<br><br>If you forget your passphrase, you will no longer be able to decrypt the hifikey file that the passphrase protects. \
+You will also no longer have access to the contents of your Wallet or My Purchases. \
+For security reasons, High Fidelity does not keep a copy of your passphrase, and can't restore it for you. \
+<br><br>If you still cannot remember your wallet passphrase, you can set up your Wallet again and generate a new hifikey file. \
+Unfortunately, this means you will start with 0 HFC and your purchased items will not be transferred over.";
         }
         ListElement {
             isExpanded: false;
-            question: "What is a 'Security Pic'?"
-            answer: qsTr("Your Security Pic is an encrypted image that you selected during Wallet Setup. <b>It acts as an extra layer of Wallet security.</b><br><br>When you see your Security Pic, you know that your actions and data are securely making use of your private keys.<br><br><b>If you don't see your Security Pic on a page that is asking you for your Wallet passphrase, someone untrustworthy may be trying to gain access to your Wallet.</b><br><br>The encrypted Pic is stored on your hard drive inside the same file as your private keys.");
+            question: "How do I send HFC to other people?";
+            answer: "You can send HFC to a High Fidelity connection (someone you've shaken hands with in-world) or somebody Nearby (currently in the same domain as you). \
+In your Wallet's Send Money tab, choose from your list of connections, or choose Nearby and select the glowing sphere of the person's avatar.";
         }
         ListElement {
             isExpanded: false;
-            question: "My HFC balance isn't what I expect it to be. Why?"
-            answer: qsTr('High Fidelity Coin (HFC) transactions are backed by a <b>blockchain</b>, which takes time to update. The status of a transaction usually updates within a few seconds.<br><br><b><font color="#0093C5"><a href="#blockchain">Tap here to learn more about the blockchain.</a></font></b>');
+            question: "What is a Security Pic?"
+            answer: "Your Security Pic is an encrypted image that you select during Wallet Setup. \
+It acts as an extra layer of Wallet security. \
+When you see your Security Pic, you know that your actions and data are securely making use of your private keys.\
+<br><br>Don't enter your passphrase anywhere that doesn't display your Security Pic! \
+If you don't see your Security Pic on a page that requests your Wallet passphrase, someone untrustworthy may be trying to access your Wallet.";
         }
         ListElement {
             isExpanded: false;
-            question: "Do I get charged money if a transaction fails?"
-            answer: qsTr("<b>No.</b> Your HFC balance only changes after a transaction is confirmed.");
+            question: "Why does my HFC balance not update instantly?";
+            answer: "HFC transations sometimes takes a few seconds to update as they are backed by a blockchain. \
+<br><br><b><font color='#0093C5'><a href='#blockchain'>Tap here to learn more about the blockchain.</a></font></b>";
+        }
+        ListElement {
+            isExpanded: false;
+            question: "Do I get charged money if a transaction fails?";
+            answer: "<b>No.</b> Your HFC balance only changes after a transaction is confirmed.";
         }
         ListElement {
             isExpanded: false;
             question: "How do I convert HFC to other currencies?"
-            answer: qsTr("We are still building the tools needed to support a vibrant economy in High Fidelity. <b>There is currently no way to convert HFC to other currencies.</b>");
+            answer: "We are hard at work building the tools needed to support a vibrant economy in High Fidelity. \
+At the moment, there is currently no way to convert HFC to other currencies. Stay tuned...";
+        }
+        ListElement {
+            isExpanded: false;
+            question: "Who can I reach out to with questions?";
+            answer: "Please email us if you have any issues or questions: \
+<b><font color='#0093C5'><a href='#support'>support@highfidelity.com</a></font></b>";
         }
     }
 
@@ -212,6 +252,10 @@ Item {
                             Qt.openUrlExternally("file:///" + root.keyFilePath.substring(0, root.keyFilePath.lastIndexOf('/')));
                         } else if (link === "#blockchain") {
                             Qt.openUrlExternally("https://docs.highfidelity.com/high-fidelity-commerce");
+                        } else if (link === "#bank") {
+                            Qt.openUrlExternally("hifi://BankOfHighFidelity");
+                        } else if (link === "#support") {
+                            Qt.openUrlExternally("mailto:support@highfidelity.com");
                         }
                     }
                 }

interface/resources/qml/hifi/commerce/wallet/Wallet.qml

@@ -47,20 +47,22 @@ Rectangle {
                 }
             } else if (walletStatus === 1) {
                 if (root.activeView !== "walletSetup") {
-                    root.activeView = "walletSetup";
-                    Commerce.resetLocalWalletOnly();
-                    var timestamp = new Date();
-                    walletSetup.startingTimestamp = timestamp;
-                    walletSetup.setupAttemptID = generateUUID();
-                    UserActivityLogger.commerceWalletSetupStarted(timestamp, setupAttemptID, walletSetup.setupFlowVersion, walletSetup.referrer ? walletSetup.referrer : "wallet app",
-                        (AddressManager.placename || AddressManager.hostname || '') + (AddressManager.pathname ? AddressManager.pathname.match(/\/[^\/]+/)[0] : ''));
+                    walletResetSetup();
                 }
             } else if (walletStatus === 2) {
+                if (root.activeView != "preexisting") {
+                    root.activeView = "preexisting";
+                }
+            } else if (walletStatus === 3) {
+                if (root.activeView != "conflicting") {
+                    root.activeView = "conflicting";
+                }
+            } else if (walletStatus === 4) {
                 if (root.activeView !== "passphraseModal") {
                     root.activeView = "passphraseModal";
                     UserActivityLogger.commercePassphraseEntry("wallet app");
                 }
-            } else if (walletStatus === 3) {
+            } else if (walletStatus === 5) {
                 if (root.activeView !== "walletSetup") {
                     root.activeView = "walletHome";
                     Commerce.getSecurityImage();
@@ -169,6 +171,25 @@ Rectangle {
     // TITLE BAR END
     //
 
+    WalletChoice {
+        id: walletChoice;
+        proceedFunction: function (isReset) {
+            console.log(isReset ? "Reset wallet." : "Trying again with new wallet.");
+            Commerce.setSoftReset();
+            if (isReset) {
+                walletResetSetup();
+            } else {
+                var msg = { referrer: walletChoice.referrer }
+                followReferrer(msg);
+            }
+        }
+        copyFunction: Commerce.copyKeyFileFrom;
+        z: 997;
+        visible: (root.activeView === "preexisting") || (root.activeView === "conflicting");
+        activeView: root.activeView;
+        anchors.fill: parent;
+    }
+
     WalletSetup {
         id: walletSetup;
         visible: root.activeView === "walletSetup";
@@ -178,14 +199,7 @@ Rectangle {
         Connections {
             onSendSignalToWallet: {
                 if (msg.method === 'walletSetup_finished') {
-                    if (msg.referrer === '' || msg.referrer === 'marketplace cta') {
-                        root.activeView = "initialize";
-                        Commerce.getWalletStatus();
-                    } else if (msg.referrer === 'purchases') {
-                        sendToScript({method: 'goToPurchases'});
-                    } else {
-                        sendToScript({method: 'goToMarketplaceItemPage', itemId: msg.referrer});
-                    }
+                    followReferrer(msg);
                 } else if (msg.method === 'walletSetup_raiseKeyboard') {
                     root.keyboardRaised = true;
                     root.isPassword = msg.isPasswordField;
@@ -738,6 +752,7 @@ Rectangle {
         switch (message.method) {
             case 'updateWalletReferrer':
                 walletSetup.referrer = message.referrer;
+                walletChoice.referrer = message.referrer;
             break;
             case 'inspectionCertificate_resetCert':
                 // NOP
@@ -768,6 +783,28 @@ Rectangle {
             return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
         });
     }
+
+    function walletResetSetup() {
+        root.activeView = "walletSetup";
+        var timestamp = new Date();
+        walletSetup.startingTimestamp = timestamp;
+        walletSetup.setupAttemptID = generateUUID();
+        UserActivityLogger.commerceWalletSetupStarted(timestamp, walletSetup.setupAttemptID, walletSetup.setupFlowVersion, walletSetup.referrer ? walletSetup.referrer : "wallet app",
+            (AddressManager.placename || AddressManager.hostname || '') + (AddressManager.pathname ? AddressManager.pathname.match(/\/[^\/]+/)[0] : ''));
+    }
+
+    function followReferrer(msg) {
+        if (msg.referrer === '' || msg.referrer === 'marketplace cta') {
+            root.activeView = "initialize";
+            Commerce.getWalletStatus();
+        } else if (msg.referrer === 'purchases') {
+            sendToScript({method: 'goToPurchases'});
+        } else if (msg.referrer === 'marketplace cta' || msg.referrer === 'mainPage') {
+            sendToScript({method: 'goToMarketplaceMainPage', itemId: msg.referrer});
+        } else {
+            sendToScript({method: 'goToMarketplaceItemPage', itemId: msg.referrer});
+        }
+    }
     //
     // FUNCTION DEFINITIONS END
     //

interface/resources/qml/hifi/commerce/wallet/WalletChoice.qml

@@ -0,0 +1,297 @@
+//
+//  WalletChoice.qml
+//  qml/hifi/commerce/wallet
+//
+//  WalletChoice
+//
+//  Created by Howard Stearns
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+import Hifi 1.0 as Hifi
+import QtQuick 2.5
+import "../common" as HifiCommerceCommon
+import "../../../styles-uit"
+import "../../../controls-uit" as HifiControlsUit
+
+
+Item {
+    HifiConstants { id: hifi; }
+
+    id: root;
+    property string activeView: "conflict";
+    property var proceedFunction: nil;
+    property var copyFunction: nil;
+    property string referrer: "";
+
+    Image {
+        anchors.fill: parent;
+        source: "images/wallet-bg.jpg";
+    }
+
+    HifiCommerceCommon.CommerceLightbox {
+        id: lightboxPopup;
+        visible: false;
+        anchors.fill: parent;
+    }
+
+    // This object is always used in a popup.
+    // This MouseArea is used to prevent a user from being
+    //     able to click on a button/mouseArea underneath the popup.
+    MouseArea {
+        anchors.fill: parent;
+        propagateComposedEvents: false;
+        hoverEnabled: true;
+    }
+
+    //
+    // TITLE BAR START
+    //
+    Item {
+        id: titleBarContainer;
+        // Size
+        height: 50;
+        // Anchors
+        anchors.left: parent.left;
+        anchors.top: parent.top;
+        anchors.right: parent.right;
+
+        // Wallet icon
+        HiFiGlyphs {
+            id: walletIcon;
+            text: hifi.glyphs.wallet;
+            // Size
+            size: parent.height * 0.8;
+            // Anchors
+            anchors.left: parent.left;
+            anchors.leftMargin: 8;
+            anchors.verticalCenter: parent.verticalCenter;
+            // Style
+            color: hifi.colors.blueHighlight;
+        }
+
+        // Title Bar text
+        RalewayRegular {
+            id: titleBarText;
+            text: "Wallet Setup";
+            // Text size
+            size: hifi.fontSizes.overlayTitle;
+            // Anchors
+            anchors.top: parent.top;
+            anchors.left: walletIcon.right;
+            anchors.leftMargin: 8;
+            anchors.bottom: parent.bottom;
+            width: paintedWidth;
+            // Style
+            color: hifi.colors.white;
+            // Alignment
+            horizontalAlignment: Text.AlignHLeft;
+            verticalAlignment: Text.AlignVCenter;
+        }
+    }
+    //
+    // TITLE BAR END
+    //
+
+    //
+    // MAIN PAGE START
+    //
+    Item {
+        id: preexistingContainer;
+        // Anchors
+        anchors.top: titleBarContainer.bottom;
+        anchors.bottom: parent.bottom;
+        anchors.left: parent.left;
+        anchors.right: parent.right;
+
+        HiFiGlyphs {
+            id: bigKeyIcon;
+            text: hifi.glyphs.walletKey;
+            // Size
+            size: 180;
+            // Anchors
+            anchors.top: parent.top;
+            anchors.topMargin: 40;
+            anchors.horizontalCenter: parent.horizontalCenter;
+            // Style
+            color: hifi.colors.white;
+        }
+
+        RalewayRegular {
+            id: text01;
+            text: root.activeView === "preexisting" ? 
+                "Where are your private keys?" :
+                "Hmm, your keys are different"
+            // Text size
+            size: 26;
+            // Anchors
+            anchors.top: bigKeyIcon.bottom;
+            anchors.left: parent.left;
+            anchors.leftMargin: 16;
+            anchors.right: parent.right;
+            anchors.rightMargin: 16;
+            height: paintedHeight;
+            width: paintedWidth;
+            // Style
+            color: hifi.colors.white;
+            wrapMode: Text.WordWrap;
+            // Alignment
+            horizontalAlignment: Text.AlignHCenter;
+            verticalAlignment: Text.AlignVCenter;
+        }
+
+        RalewayRegular {
+            id: text02;
+            text: root.activeView === "preexisting" ? 
+                "Our records indicate that you created a wallet, but the private keys are not in the folder where we checked." :
+                "Our records indicate that you created a wallet with different keys than the keys you're providing."
+            // Text size
+            size: 18;
+            // Anchors
+            anchors.top: text01.bottom;
+            anchors.topMargin: 40;
+            anchors.left: parent.left;
+            anchors.leftMargin: 65;
+            anchors.right: parent.right;
+            anchors.rightMargin: 65;
+            height: paintedHeight;
+            width: paintedWidth;
+            // Style
+            color: hifi.colors.white;
+            wrapMode: Text.WordWrap;
+            // Alignment
+            horizontalAlignment: Text.AlignHCenter;
+            verticalAlignment: Text.AlignVCenter;
+        }
+
+        // "Locate" button
+        HifiControlsUit.Button {
+            id: locateButton;
+            color: hifi.buttons.blue;
+            colorScheme: hifi.colorSchemes.dark;
+            anchors.top: text02.bottom;
+            anchors.topMargin: 40;
+            anchors.horizontalCenter: parent.horizontalCenter;
+            width: parent.width/2;
+            height: 50;
+            text: root.activeView === "preexisting" ? 
+                "LOCATE MY KEYS" :
+                "LOCATE OTHER KEYS"
+            onClicked: {
+                walletChooser();
+            }
+        }
+
+        // "Create New" OR "Continue" button
+        HifiControlsUit.Button {
+            id: button02;
+            color: hifi.buttons.none;
+            colorScheme: hifi.colorSchemes.dark;
+            anchors.top: locateButton.bottom;
+            anchors.topMargin: 20;
+            anchors.horizontalCenter: parent.horizontalCenter;
+            width: parent.width/2;
+            height: 50;
+            text: root.activeView === "preexisting" ? 
+                "CREATE NEW WALLET" :
+                "CONTINUE WITH THESE KEYS"
+            onClicked: {
+                    lightboxPopup.titleText = "Are you sure?";
+                    lightboxPopup.bodyText = "Taking this step will abandon your old wallet and you will no " +
+                        "longer be able to access your money and your past purchases.<br><br>" +
+                        "This step should only be used if you cannot find your keys.<br><br>" +
+                        "This step cannot be undone.";
+                    lightboxPopup.button1color = hifi.buttons.red;
+                    lightboxPopup.button1text = "YES, CREATE NEW WALLET";
+                    lightboxPopup.button1method = "root.visible = false;proceed(true);";
+                    lightboxPopup.button2text = "CANCEL";
+                    lightboxPopup.button2method = "root.visible = false;"
+                    lightboxPopup.buttonLayout = "topbottom";
+                    lightboxPopup.visible = true;
+            }
+        }
+
+        // "What's This?" link
+        RalewayRegular {
+            id: whatsThisLink;
+            text: '<font color="#FFFFFF"><a href="#whatsthis">What\'s this?</a></font>';
+            // Anchors
+            anchors.bottom: parent.bottom;
+            anchors.bottomMargin: 48;
+            anchors.horizontalCenter: parent.horizontalCenter;
+            width: paintedWidth;
+            height: paintedHeight;
+            // Text size
+            size: 18;
+            // Style
+            color: hifi.colors.white;
+
+            MouseArea {
+                anchors.fill: parent;
+
+                onClicked: {
+                    if (root.activeView === "preexisting") {
+                        lightboxPopup.titleText = "Your wallet's private keys are not in the folder we expected";
+                        lightboxPopup.bodyText = "We see that you have created a wallet but the private keys " +
+                            "for it seem to have been moved to a different folder.<br><br>" +
+                            "To tell us where the keys are, click 'Locate My Keys'. <br><br>" +
+                            "If you'd prefer to create a new wallet (not recommended - you will lose your money and past " +
+                            "purchases), click 'Create New Wallet'.";
+                        lightboxPopup.button1text = "CLOSE";
+                        lightboxPopup.button1method = "root.visible = false;"
+                        lightboxPopup.visible = true;
+                    } else {
+                        lightboxPopup.titleText = "You may have set up more than one wallet";
+                        lightboxPopup.bodyText = "We see that the private keys stored on your computer are different " +
+                            "from the ones you used last time. This may mean that you set up more than one wallet. " +
+                            "If you would like to use these keys, click 'Continue With These Keys'.<br><br>" +
+                            "If you would prefer to use another wallet, click 'Locate Other Keys' to show us where " +
+                            "you've stored the private keys for that wallet.";
+                        lightboxPopup.button1text = "CLOSE";
+                        lightboxPopup.button1method = "root.visible = false;"
+                        lightboxPopup.visible = true;
+                    }
+                }
+            }
+        }
+    }
+    //
+    // MAIN PAGE END
+    //
+    
+    //
+    // FUNCTION DEFINITIONS START
+    //
+    function onFileOpenChanged(filename) {
+        // disconnect the event, otherwise the requests will stack up
+        try { // Not all calls to onFileOpenChanged() connect an event.
+            Window.browseChanged.disconnect(onFileOpenChanged);
+        } catch (e) {
+            console.log('WalletChoice.qml ignoring', e);
+        }
+        if (filename) {
+            if (copyFunction && copyFunction(filename)) {
+                proceed(false);
+             } else {
+                console.log("WalletChoice.qml copyFunction", copyFunction, "failed.");
+             }
+        } // Else we're still at WalletChoice
+    }
+    function walletChooser() {
+        Window.browseChanged.connect(onFileOpenChanged); 
+        Window.browseAsync("Locate your .hifikey file", "", "*.hifikey");
+    }
+    function proceed(isReset) {
+        if (!proceedFunction) {
+            console.log("Provide a function of no arguments to WalletChoice.qml.");
+        } else {
+            proceedFunction(isReset);
+        }
+    }
+    //
+    // FUNCTION DEFINITIONS END
+    //
+}

interface/resources/qml/hifi/commerce/wallet/WalletHome.qml

@@ -310,7 +310,7 @@ Item {
                 height: parent.height;
 
                 HifiControlsUit.Separator {
-                colorScheme: 1;
+                    colorScheme: 1;
                     anchors.left: parent.left;
                     anchors.right: parent.right;
                     anchors.top: parent.top;
@@ -318,20 +318,42 @@ Item {
 
                 RalewayRegular {
                     id: noActivityText;
-                text: "<b>The Wallet app is in closed Beta.</b><br><br>To request entry and <b>receive free HFC</b>, please contact " +
-                "<b>info@highfidelity.com</b> with your High Fidelity account username and the email address registered to that account.";
-                // Text size
-                size: 24;
-                // Style
-                color: hifi.colors.blueAccent;
-                anchors.left: parent.left;
-                anchors.leftMargin: 12;
-                anchors.right: parent.right;
-                anchors.rightMargin: 12;
-                anchors.verticalCenter: parent.verticalCenter;
-                height: paintedHeight;
-                wrapMode: Text.WordWrap;
-                horizontalAlignment: Text.AlignHCenter;
+                    text: "Congrats! Your wallet is all set!<br><br>" +
+                        "<b>Where's my HFC?</b><br>" +
+                        "High Fidelity commerce is in open beta right now. Want more HFC? Get it by meeting with a banker at " +
+                        "<a href='#goToBank'>BankOfHighFidelity</a>!"
+                    // Text size
+                    size: 22;
+                    // Style
+                    color: hifi.colors.blueAccent;
+                    anchors.top: parent.top;
+                    anchors.topMargin: 36;
+                    anchors.left: parent.left;
+                    anchors.leftMargin: 12;
+                    anchors.right: parent.right;
+                    anchors.rightMargin: 12;
+                    height: paintedHeight;
+                    wrapMode: Text.WordWrap;
+                    horizontalAlignment: Text.AlignHCenter;
+
+                    onLinkActivated: {
+                        sendSignalToWallet({ method: "transactionHistory_goToBank" });
+                    }
+                }
+
+                HifiControlsUit.Button {
+                    id: bankButton;
+                    color: hifi.buttons.blue;
+                    colorScheme: hifi.colorSchemes.dark;
+                    anchors.top: noActivityText.bottom;
+                    anchors.topMargin: 30;
+                    anchors.horizontalCenter: parent.horizontalCenter;
+                    width: parent.width/2;
+                    height: 50;
+                    text: "VISIT BANK OF HIGH FIDELITY";
+                    onClicked: {
+                        sendSignalToWallet({ method: "transactionHistory_goToBank" });
+                    }
                 }
             }
             

interface/resources/qml/hifi/dialogs/GraphicsPreferencesDialog.qml

@@ -0,0 +1,19 @@
+import QtQuick 2.5
+import Qt.labs.settings 1.0
+
+import "../../dialogs"
+
+PreferencesDialog {
+    id: root
+    objectName: "GraphicsPreferencesDialog"
+    title: "Graphics Settings"
+    showCategories: ["Graphics"]
+    property var settings: Settings {
+        category: root.objectName
+        property alias x: root.x
+        property alias y: root.y
+        property alias width: root.width
+        property alias height: root.height
+    }
+}
+

interface/resources/qml/hifi/tablet/TabletGraphicsPreferences.qml

@@ -0,0 +1,37 @@
+//
+//  TabletGraphicsPreferences.qml
+//
+//  Created by Vlad Stelmahovsky on 12 Mar 2017.
+//  Copyright 2017 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+import QtQuick 2.5
+import QtQuick.Controls 1.4
+import "tabletWindows"
+import "../../dialogs"
+
+StackView {
+    id: profileRoot
+    initialItem: root
+    objectName: "stack"
+    property string title: "Graphics Settings"
+
+    signal sendToScript(var message);
+
+    function pushSource(path) {
+        profileRoot.push(Qt.resolvedUrl(path));
+    }
+
+    function popSource() {
+        profileRoot.pop();
+    }
+
+    TabletPreferencesDialog {
+        id: root
+        objectName: "TabletGraphicsPreferences"
+        showCategories: ["Graphics"]
+    }
+}

interface/src/Application.cpp

@@ -167,6 +167,7 @@
 #include "scripting/AccountServicesScriptingInterface.h"
 #include "scripting/HMDScriptingInterface.h"
 #include "scripting/MenuScriptingInterface.h"
+#include "graphics-scripting/GraphicsScriptingInterface.h"
 #include "scripting/SettingsScriptingInterface.h"
 #include "scripting/WindowScriptingInterface.h"
 #include "scripting/ControllerScriptingInterface.h"
@@ -576,10 +577,7 @@ void messageHandler(QtMsgType type, const QMessageLogContext& context, const QSt
     QString logMessage = LogHandler::getInstance().printMessage((LogMsgType) type, context, message);
 
     if (!logMessage.isEmpty()) {
-#ifdef Q_OS_WIN
-        OutputDebugStringA(logMessage.toLocal8Bit().constData());
-        OutputDebugStringA("\n");
-#elif defined Q_OS_ANDROID
+#ifdef Q_OS_ANDROID
         const char * local=logMessage.toStdString().c_str();
         switch (type) {
             case QtDebugMsg:
@@ -600,10 +598,75 @@ void messageHandler(QtMsgType type, const QMessageLogContext& context, const QSt
                 abort();
         }
 #endif
-        qApp->getLogger()->addMessage(qPrintable(logMessage + "\n"));
+        qApp->getLogger()->addMessage(qPrintable(logMessage));
     }
 }
 
+
+class ApplicationMeshProvider : public scriptable::ModelProviderFactory  {
+public:
+    virtual scriptable::ModelProviderPointer lookupModelProvider(const QUuid& uuid) override {
+        bool success;
+        if (auto nestable = DependencyManager::get<SpatialParentFinder>()->find(uuid, success).lock()) {
+            auto type = nestable->getNestableType();
+#ifdef SCRIPTABLE_MESH_DEBUG
+            qCDebug(interfaceapp) << "ApplicationMeshProvider::lookupModelProvider" << uuid << SpatiallyNestable::nestableTypeToString(type);
+#endif
+            switch (type) {
+            case NestableType::Entity:
+                return getEntityModelProvider(static_cast<EntityItemID>(uuid));
+            case NestableType::Overlay:
+                return getOverlayModelProvider(static_cast<OverlayID>(uuid));
+            case NestableType::Avatar:
+                return getAvatarModelProvider(uuid);
+            }
+        }
+        return nullptr;
+    }
+
+private:
+    scriptable::ModelProviderPointer getEntityModelProvider(EntityItemID entityID) {
+        scriptable::ModelProviderPointer provider;
+        auto entityTreeRenderer = qApp->getEntities();
+        auto entityTree = entityTreeRenderer->getTree();
+        if (auto entity = entityTree->findEntityByID(entityID)) {
+            if (auto renderer = entityTreeRenderer->renderableForEntityId(entityID)) {
+                provider = std::dynamic_pointer_cast<scriptable::ModelProvider>(renderer);
+                provider->modelProviderType = NestableType::Entity;
+            } else {
+                qCWarning(interfaceapp) << "no renderer for entity ID" << entityID.toString();
+            }
+        }
+        return provider;
+    }
+
+    scriptable::ModelProviderPointer getOverlayModelProvider(OverlayID overlayID) {
+        scriptable::ModelProviderPointer provider;
+        auto &overlays = qApp->getOverlays();
+        if (auto overlay = overlays.getOverlay(overlayID)) {
+            if (auto base3d = std::dynamic_pointer_cast<Base3DOverlay>(overlay)) {
+                provider = std::dynamic_pointer_cast<scriptable::ModelProvider>(base3d);
+                provider->modelProviderType = NestableType::Overlay;
+            } else {
+                qCWarning(interfaceapp) << "no renderer for overlay ID" << overlayID.toString();
+            }
+        } else {
+            qCWarning(interfaceapp) << "overlay not found" << overlayID.toString();
+        }
+        return provider;
+    }
+
+    scriptable::ModelProviderPointer getAvatarModelProvider(QUuid sessionUUID) {
+        scriptable::ModelProviderPointer provider;
+        auto avatarManager = DependencyManager::get<AvatarManager>();
+        if (auto avatar = avatarManager->getAvatarBySessionID(sessionUUID)) {
+            provider = std::dynamic_pointer_cast<scriptable::ModelProvider>(avatar);
+            provider->modelProviderType = NestableType::Avatar;
+        }
+        return provider;
+    }
+};
+
 static const QString STATE_IN_HMD = "InHMD";
 static const QString STATE_CAMERA_FULL_SCREEN_MIRROR = "CameraFSM";
 static const QString STATE_CAMERA_FIRST_PERSON = "CameraFirstPerson";
@@ -721,6 +784,7 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
     DependencyManager::set<AccountManager>(std::bind(&Application::getUserAgent, qApp));
     DependencyManager::set<StatTracker>();
     DependencyManager::set<ScriptEngines>(ScriptEngine::CLIENT_SCRIPT);
+    DependencyManager::set<ScriptInitializerMixin, NativeScriptInitializers>();
     DependencyManager::set<Preferences>();
     DependencyManager::set<recording::Deck>();
     DependencyManager::set<recording::Recorder>();
@@ -749,6 +813,9 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
     DependencyManager::set<ResourceCacheSharedItems>();
     DependencyManager::set<DesktopScriptingInterface>();
     DependencyManager::set<EntityScriptingInterface>(true);
+    DependencyManager::set<GraphicsScriptingInterface>();
+    DependencyManager::registerInheritance<scriptable::ModelProviderFactory, ApplicationMeshProvider>();
+    DependencyManager::set<ApplicationMeshProvider>();
     DependencyManager::set<RecordingScriptingInterface>();
     DependencyManager::set<WindowScriptingInterface>();
     DependencyManager::set<HMDScriptingInterface>();
@@ -1519,6 +1586,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         settingsTimer->setSingleShot(false);
         settingsTimer->setInterval(SAVE_SETTINGS_INTERVAL); // 10s, Qt::CoarseTimer acceptable
         QObject::connect(settingsTimer, &QTimer::timeout, this, &Application::saveSettings);
+        settingsTimer->start();
     }, QThread::LowestPriority);
 
     if (Menu::getInstance()->isOptionChecked(MenuOption::FirstPerson)) {
@@ -6050,6 +6118,9 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEnginePointe
     scriptEngine->registerGlobalObject("Scene", DependencyManager::get<SceneScriptingInterface>().data());
     scriptEngine->registerGlobalObject("Render", _renderEngine->getConfiguration().get());
 
+    GraphicsScriptingInterface::registerMetaTypes(scriptEngine.data());
+    scriptEngine->registerGlobalObject("Graphics", DependencyManager::get<GraphicsScriptingInterface>().data());
+
     scriptEngine->registerGlobalObject("ScriptDiscoveryService", DependencyManager::get<ScriptEngines>().data());
     scriptEngine->registerGlobalObject("Reticle", getApplicationCompositor().getReticleInterface());
 
@@ -6312,13 +6383,14 @@ bool Application::askToWearAvatarAttachmentUrl(const QString& url) {
 void Application::replaceDomainContent(const QString& url) {
     qCDebug(interfaceapp) << "Attempting to replace domain content: " << url;
     QByteArray urlData(url.toUtf8());
-    auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
+    auto limitedNodeList = DependencyManager::get<NodeList>();
+    const auto& domainHandler = limitedNodeList->getDomainHandler();
     limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
         return node->getType() == NodeType::EntityServer && node->getActiveSocket();
-    }, [&urlData, limitedNodeList](const SharedNodePointer& octreeNode) {
+    }, [&urlData, limitedNodeList, &domainHandler](const SharedNodePointer& octreeNode) {
         auto octreeFilePacket = NLPacket::create(PacketType::OctreeFileReplacementFromUrl, urlData.size(), true);
         octreeFilePacket->write(urlData);
-        limitedNodeList->sendPacket(std::move(octreeFilePacket), *octreeNode);
+        limitedNodeList->sendPacket(std::move(octreeFilePacket), domainHandler.getSockAddr());
     });
     auto addressManager = DependencyManager::get<AddressManager>();
     addressManager->handleLookupString(DOMAIN_SPAWNING_POINT);

interface/src/Menu.cpp

@@ -43,7 +43,6 @@
 #include "ui/StandAloneJSConsole.h"
 #include "InterfaceLogging.h"
 #include "LocationBookmarks.h"
-#include "DeferredLightingEffect.h"
 
 #if defined(Q_OS_MAC) || defined(Q_OS_WIN)
 #include "SpeechRecognizer.h"
@@ -362,15 +361,10 @@ Menu::Menu() {
     MenuWrapper* developerMenu = addMenu("Developer", "Developer");
 
     // Developer > Graphics...
-    MenuWrapper* graphicsOptionsMenu = developerMenu->addMenu("Render");
-    action = addCheckableActionToQMenuAndActionHash(graphicsOptionsMenu, MenuOption::Shadows, 0, true);
-    connect(action, &QAction::triggered, [action] {
-        DependencyManager::get<DeferredLightingEffect>()->setShadowMapEnabled(action->isChecked());
-    });
-
-    action = addCheckableActionToQMenuAndActionHash(graphicsOptionsMenu, MenuOption::AmbientOcclusion, 0, false);
-    connect(action, &QAction::triggered, [action] {
-        DependencyManager::get<DeferredLightingEffect>()->setAmbientOcclusionEnabled(action->isChecked());
+    action = addActionToQMenuAndActionHash(developerMenu, "Graphics...");
+    connect(action, &QAction::triggered, [] {
+        qApp->showDialog(QString("hifi/dialogs/GraphicsPreferencesDialog.qml"),
+            QString("hifi/tablet/TabletGraphicsPreferences.qml"), "GraphicsPreferencesDialog");
     });
 
     // Developer > UI >>>

interface/src/Menu.h

@@ -204,8 +204,6 @@ namespace MenuOption {
     const QString WorldAxes = "World Axes";
     const QString DesktopTabletToToolbar = "Desktop Tablet Becomes Toolbar";
     const QString HMDTabletToToolbar = "HMD Tablet Becomes Toolbar";
-    const QString Shadows = "Shadows";
-    const QString AmbientOcclusion = "AmbientOcclusion";
 }
 
 #endif // hifi_Menu_h

interface/src/avatar/MyAvatar.cpp

@@ -1115,7 +1115,6 @@ void MyAvatar::setEnableDebugDrawIKChains(bool isEnabled) {
 
 void MyAvatar::setEnableMeshVisible(bool isEnabled) {
     _skeletonModel->setVisibleInScene(isEnabled, qApp->getMain3DScene(), render::ItemKey::TAG_BITS_NONE, true);
-    _skeletonModel->setCanCastShadow(isEnabled, qApp->getMain3DScene(), render::ItemKey::TAG_BITS_NONE, true);
 }
 
 void MyAvatar::setEnableInverseKinematics(bool isEnabled) {
@@ -1468,7 +1467,6 @@ void MyAvatar::setSkeletonModelURL(const QUrl& skeletonModelURL) {
     int skeletonModelChangeCount = _skeletonModelChangeCount;
     Avatar::setSkeletonModelURL(skeletonModelURL);
     _skeletonModel->setVisibleInScene(true, qApp->getMain3DScene(), render::ItemKey::TAG_BITS_NONE, true);
-    _skeletonModel->setCanCastShadow(true, qApp->getMain3DScene(), render::ItemKey::TAG_BITS_NONE, true);
     _headBoneSet.clear();
     _cauterizationNeedsUpdate = true;
 
@@ -1850,6 +1848,12 @@ void MyAvatar::attach(const QString& modelURL, const QString& jointName,
     Avatar::attach(modelURL, jointName, translation, rotation, scale, isSoft, allowDuplicates, useSaved);
 }
 
+void MyAvatar::setVisibleInSceneIfReady(Model* model, const render::ScenePointer& scene, bool visible) {
+    if (model->isActive() && model->isRenderable()) {
+        model->setVisibleInScene(visible, scene, render::ItemKey::TAG_BITS_NONE, true);
+    }
+}
+
 void MyAvatar::initHeadBones() {
     int neckJointIndex = -1;
     if (_skeletonModel->isLoaded()) {
@@ -2039,11 +2043,8 @@ void MyAvatar::preDisplaySide(RenderArgs* renderArgs) {
                 _attachmentData[i].jointName.compare("RightEye", Qt::CaseInsensitive) == 0 ||
                 _attachmentData[i].jointName.compare("HeadTop_End", Qt::CaseInsensitive) == 0 ||
                 _attachmentData[i].jointName.compare("Face", Qt::CaseInsensitive) == 0) {
-
                 _attachmentModels[i]->setVisibleInScene(shouldDrawHead, qApp->getMain3DScene(),
                                                         render::ItemKey::TAG_BITS_NONE, true);
-
-                _attachmentModels[i]->setCanCastShadow(shouldDrawHead, qApp->getMain3DScene(), render::ItemKey::TAG_BITS_NONE, true);
             }
         }
     }

interface/src/avatar/MyAvatar.h

@@ -681,6 +681,8 @@ private:
     // These are made private for MyAvatar so that you will use the "use" methods instead
     virtual void setSkeletonModelURL(const QUrl& skeletonModelURL) override;
 
+    void setVisibleInSceneIfReady(Model* model, const render::ScenePointer& scene, bool visiblity);
+
     virtual void updatePalms() override {}
     void lateUpdatePalms();
 

interface/src/commerce/Ledger.cpp

@@ -22,6 +22,8 @@
 // inventory answers {status: 'success', data: {assets: [{id: "guid", title: "name", preview: "url"}....]}}
 // balance answers {status: 'success', data: {balance: integer}}
 // buy and receive_at answer {status: 'success'}
+// account synthesizes a result {status: 'success', data: {keyStatus: "preexisting"|"conflicting"|"ok"}}
+
 
 QJsonObject Ledger::apiResponse(const QString& label, QNetworkReply& reply) {
     QByteArray response = reply.readAll();
@@ -99,7 +101,7 @@ void Ledger::buy(const QString& hfc_key, int cost, const QString& asset_id, cons
     signedSend("transaction", transactionString, hfc_key, "buy", "buySuccess", "buyFailure", controlled_failure);
 }
 
-bool Ledger::receiveAt(const QString& hfc_key, const QString& old_key) {
+bool Ledger::receiveAt(const QString& hfc_key, const QString& signing_key) {
     auto accountManager = DependencyManager::get<AccountManager>();
     if (!accountManager->isLoggedIn()) {
         qCWarning(commerce) << "Cannot set receiveAt when not logged in.";
@@ -108,7 +110,7 @@ bool Ledger::receiveAt(const QString& hfc_key, const QString& old_key) {
         return false; // We know right away that we will fail, so tell the caller.
     }
 
-    signedSend("public_key", hfc_key.toUtf8(), old_key, "receive_at", "receiveAtSuccess", "receiveAtFailure");
+    signedSend("public_key", hfc_key.toUtf8(), signing_key, "receive_at", "receiveAtSuccess", "receiveAtFailure");
     return true; // Note that there may still be an asynchronous signal of failure that callers might be interested in.
 }
 
@@ -179,7 +181,7 @@ QString transactionString(const QJsonObject& valueObject) {
     } else {
         result += valueObject["message"].toString();
     }
-    
+
     // no matter what we append a smaller date to the bottom of this...
     result += QString("<br><font size='-2' color='#1080B8'>%1").arg(createdAt.toLocalTime().toString(Qt::DefaultLocaleShortDate));
     return result;
@@ -246,18 +248,33 @@ void Ledger::accountSuccess(QNetworkReply& reply) {
     auto iv = QByteArray::fromBase64(data["iv"].toString().toUtf8());
     auto ckey = QByteArray::fromBase64(data["ckey"].toString().toUtf8());
     QString remotePublicKey = data["public_key"].toString();
+    bool isOverride = wallet->wasSoftReset();
 
     wallet->setSalt(salt);
     wallet->setIv(iv);
     wallet->setCKey(ckey);
 
+    QString keyStatus = "ok";
     QStringList localPublicKeys = wallet->listPublicKeys();
-    if (remotePublicKey.isEmpty() && !localPublicKeys.isEmpty()) {
-        receiveAt(localPublicKeys.first(), "");
+    if (remotePublicKey.isEmpty() || isOverride) {
+        if (!localPublicKeys.isEmpty()) {
+            QString key = localPublicKeys.first();
+            receiveAt(key, key);
+        }
+    } else {
+        if (localPublicKeys.isEmpty()) {
+            keyStatus = "preexisting";
+        } else if (localPublicKeys.first() != remotePublicKey) {
+            keyStatus = "conflicting";
+        }
     }
 
     // none of the hfc account info should be emitted
-    emit accountResult(QJsonObject{ {"status", "success"} });
+    QJsonObject json;
+    QJsonObject responseData{ { "status", "success"} };
+    json["keyStatus"] = keyStatus;
+    responseData["data"] = json;
+    emit accountResult(responseData);
 }
 
 void Ledger::accountFailure(QNetworkReply& reply) {

interface/src/commerce/Ledger.h

@@ -26,7 +26,7 @@ class Ledger : public QObject, public Dependency {
 
 public:
     void buy(const QString& hfc_key, int cost, const QString& asset_id, const QString& inventory_key, const bool controlled_failure = false);
-    bool receiveAt(const QString& hfc_key, const QString& old_key);
+    bool receiveAt(const QString& hfc_key, const QString& signing_key);
     void balance(const QStringList& keys);
     void inventory(const QStringList& keys);
     void history(const QStringList& keys, const int& pageNumber);

interface/src/commerce/QmlCommerce.cpp

@@ -62,6 +62,11 @@ void QmlCommerce::getKeyFilePathIfExists() {
     emit keyFilePathIfExistsResult(wallet->getKeyFilePath());
 }
 
+bool QmlCommerce::copyKeyFileFrom(const QString& pathname) {
+    auto wallet = DependencyManager::get<Wallet>();
+    return wallet->copyKeyFileFrom(pathname);
+}
+
 void QmlCommerce::getWalletAuthenticatedStatus() {
     auto wallet = DependencyManager::get<Wallet>();
     emit walletAuthenticatedStatusResult(wallet->walletIsAuthenticatedWithPassphrase());
@@ -128,6 +133,11 @@ void QmlCommerce::changePassphrase(const QString& oldPassphrase, const QString&
     }
 }
 
+void QmlCommerce::setSoftReset() {
+    auto wallet = DependencyManager::get<Wallet>();
+    wallet->setSoftReset();
+}
+
 void QmlCommerce::setPassphrase(const QString& passphrase) {
     auto wallet = DependencyManager::get<Wallet>();
     wallet->setPassphrase(passphrase);

interface/src/commerce/QmlCommerce.h

@@ -61,10 +61,12 @@ protected:
     Q_INVOKABLE void getKeyFilePathIfExists();
     Q_INVOKABLE void getSecurityImage();
     Q_INVOKABLE void getWalletAuthenticatedStatus();
+    Q_INVOKABLE bool copyKeyFileFrom(const QString& pathname);
 
     Q_INVOKABLE void chooseSecurityImage(const QString& imageFile);
     Q_INVOKABLE void setPassphrase(const QString& passphrase);
     Q_INVOKABLE void changePassphrase(const QString& oldPassphrase, const QString& newPassphrase);
+    Q_INVOKABLE void setSoftReset();
 
     Q_INVOKABLE void buy(const QString& assetId, int cost, const bool controlledFailure = false);
     Q_INVOKABLE void balance();

interface/src/commerce/Wallet.cpp

@@ -59,6 +59,23 @@ QString keyFilePath() {
     auto accountManager = DependencyManager::get<AccountManager>();
     return PathUtils::getAppDataFilePath(QString("%1.%2").arg(accountManager->getAccountInfo().getUsername(), KEY_FILE));
 }
+bool Wallet::copyKeyFileFrom(const QString& pathname) {
+    QString existing = getKeyFilePath();
+    qCDebug(commerce) << "Old keyfile" << existing;
+    if (!existing.isEmpty()) {
+        QString backup = QString(existing).insert(existing.indexOf(KEY_FILE) - 1,
+            QDateTime::currentDateTime().toString(Qt::ISODate).replace(":", ""));
+        qCDebug(commerce) << "Renaming old keyfile to" << backup;
+        if (!QFile::rename(existing, backup)) {
+            qCCritical(commerce) << "Unable to backup" << existing << "to" << backup;
+            return false;
+        }
+    }
+    QString destination = keyFilePath();
+    bool result = QFile::copy(pathname, destination);
+    qCDebug(commerce) << "copy" << pathname << "to" << destination << "=>" << result;
+    return result;
+}
 
 // use the cached _passphrase if it exists, otherwise we need to prompt
 int passwordCallback(char* password, int maxPasswordSize, int rwFlag, void* u) {
@@ -300,17 +317,24 @@ Wallet::Wallet() {
     packetReceiver.registerListener(PacketType::ChallengeOwnership, this, "handleChallengeOwnershipPacket");
     packetReceiver.registerListener(PacketType::ChallengeOwnershipRequest, this, "handleChallengeOwnershipPacket");
 
-    connect(ledger.data(), &Ledger::accountResult, this, [&]() {
+    connect(ledger.data(), &Ledger::accountResult, this, [&](QJsonObject result) {
         auto wallet = DependencyManager::get<Wallet>();
         auto walletScriptingInterface = DependencyManager::get<WalletScriptingInterface>();
         uint status;
+        QString keyStatus = result.contains("data") ? result["data"].toObject()["keyStatus"].toString() : "";
 
         if (wallet->getKeyFilePath() == "" || !wallet->getSecurityImage()) {
-            status = (uint)WalletStatus::WALLET_STATUS_NOT_SET_UP;
+            if (keyStatus == "preexisting") {
+                status = (uint) WalletStatus::WALLET_STATUS_PREEXISTING;
+            } else{
+                status = (uint) WalletStatus::WALLET_STATUS_NOT_SET_UP;
+            }
         } else if (!wallet->walletIsAuthenticatedWithPassphrase()) {
-            status = (uint)WalletStatus::WALLET_STATUS_NOT_AUTHENTICATED;
+            status = (uint) WalletStatus::WALLET_STATUS_NOT_AUTHENTICATED;
+        } else if (keyStatus == "conflicting") {
+            status = (uint) WalletStatus::WALLET_STATUS_CONFLICTING;
         } else {
-            status = (uint)WalletStatus::WALLET_STATUS_READY;
+            status = (uint) WalletStatus::WALLET_STATUS_READY;
         }
 
         walletScriptingInterface->setWalletStatus(status);
@@ -524,17 +548,17 @@ bool Wallet::generateKeyPair() {
 
     // TODO: redo this soon -- need error checking and so on
     writeSecurityImage(_securityImage, keyFilePath());
-    QString oldKey = _publicKeys.count() == 0 ? "" : _publicKeys.last();
     QString key = keyPair.first->toBase64();
     _publicKeys.push_back(key);
     qCDebug(commerce) << "public key:" << key;
+    _isOverridingServer = false;
 
     // It's arguable whether we want to change the receiveAt every time, but:
     // 1. It's certainly needed the first time, when createIfNeeded answers true.
     // 2. It is maximally private, and we can step back from that later if desired.
     // 3. It maximally exercises all the machinery, so we are most likely to surface issues now.
     auto ledger = DependencyManager::get<Ledger>();
-    return ledger->receiveAt(key, oldKey);
+    return ledger->receiveAt(key, key);
 }
 
 QStringList Wallet::listPublicKeys() {

interface/src/commerce/Wallet.h

@@ -35,6 +35,7 @@ public:
     void chooseSecurityImage(const QString& imageFile);
     bool getSecurityImage();
     QString getKeyFilePath();
+    bool copyKeyFileFrom(const QString& pathname);
 
     void setSalt(const QByteArray& salt) { _salt = salt; }
     QByteArray getSalt() { return _salt; }
@@ -48,11 +49,15 @@ public:
     bool getPassphraseIsCached() { return !(_passphrase->isEmpty()); }
     bool walletIsAuthenticatedWithPassphrase();
     bool changePassphrase(const QString& newPassphrase);
+    void setSoftReset() { _isOverridingServer = true;  }
+    bool wasSoftReset() { bool was = _isOverridingServer; _isOverridingServer = false; return was; }
 
     void getWalletStatus();
     enum WalletStatus {
         WALLET_STATUS_NOT_LOGGED_IN = 0,
         WALLET_STATUS_NOT_SET_UP,
+        WALLET_STATUS_PREEXISTING,
+        WALLET_STATUS_CONFLICTING,
         WALLET_STATUS_NOT_AUTHENTICATED,
         WALLET_STATUS_READY
     };
@@ -73,6 +78,7 @@ private:
     QByteArray _iv;
     QByteArray _ckey;
     QString* _passphrase { new QString("") };
+    bool _isOverridingServer { false };
 
     bool writeWallet(const QString& newPassphrase = QString(""));
     void updateImageProvider();

interface/src/main.cpp

@@ -38,6 +38,7 @@ extern "C" {
 #endif
 
 int main(int argc, const char* argv[]) {
+    setupHifiApplication(BuildInfo::INTERFACE_NAME);
 
 #ifdef Q_OS_LINUX
     QApplication::setAttribute(Qt::AA_DontUseNativeMenuBar);
@@ -51,17 +52,9 @@ int main(int argc, const char* argv[]) {
     QCoreApplication::setAttribute(Qt::AA_UseOpenGLES);
 #endif
 
-    disableQtBearerPoll(); // Fixes wifi ping spikes
-
     QElapsedTimer startupTime;
     startupTime.start();
 
-    // Set application infos
-    QCoreApplication::setApplicationName(BuildInfo::INTERFACE_NAME);
-    QCoreApplication::setOrganizationName(BuildInfo::MODIFIED_ORGANIZATION);
-    QCoreApplication::setOrganizationDomain(BuildInfo::ORGANIZATION_DOMAIN);
-    QCoreApplication::setApplicationVersion(BuildInfo::VERSION);
-
     Setting::init();
 
     // Instance UserActivityLogger now that the settings are loaded

interface/src/ui/DomainConnectionModel.cpp

@@ -98,4 +98,4 @@ void DomainConnectionModel::refresh() {
     //inform view that we want refresh data
     beginResetModel();
     endResetModel();
-}
+}

interface/src/ui/overlays/Base3DOverlay.h

@@ -13,10 +13,10 @@
 
 #include <Transform.h>
 #include <SpatiallyNestable.h>
-
+#include <graphics-scripting/Forward.h>
 #include "Overlay.h"
 
-class Base3DOverlay : public Overlay, public SpatiallyNestable {
+class Base3DOverlay : public Overlay, public SpatiallyNestable, public scriptable::ModelProvider {
     Q_OBJECT
     using Parent = Overlay;
 
@@ -36,6 +36,7 @@ public:
     virtual bool is3D() const override { return true; }
 
     virtual uint32_t fetchMetaSubItems(render::ItemIDs& subItems) const override { subItems.push_back(getRenderItemID()); return (uint32_t) subItems.size(); }
+    virtual scriptable::ScriptableModelBase getScriptableModel() override { return scriptable::ScriptableModelBase(); }
 
     // TODO: consider implementing registration points in this class
     glm::vec3 getCenter() const { return getWorldPosition(); }

interface/src/ui/overlays/Cube3DOverlay.cpp

@@ -187,3 +187,14 @@ Transform Cube3DOverlay::evalRenderTransform() {
     transform.setRotation(rotation);
     return transform;
 }
+
+scriptable::ScriptableModelBase Cube3DOverlay::getScriptableModel() {
+    auto geometryCache = DependencyManager::get<GeometryCache>();
+    auto vertexColor = ColorUtils::toVec3(_color);
+    scriptable::ScriptableModelBase result;
+    if (auto mesh = geometryCache->meshFromShape(GeometryCache::Cube, vertexColor)) {
+        result.objectID = getID();
+        result.append(mesh);
+    }
+    return result;
+}

interface/src/ui/overlays/Cube3DOverlay.h

@@ -32,6 +32,7 @@ public:
     void setProperties(const QVariantMap& properties) override;
     QVariant getProperty(const QString& property) override;
 
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
 protected:
     Transform evalRenderTransform() override;
 

interface/src/ui/overlays/ModelOverlay.cpp

@@ -75,6 +75,7 @@ void ModelOverlay::update(float deltatime) {
     render::ScenePointer scene = qApp->getMain3DScene();
     render::Transaction transaction;
     if (_model->needsFixupInScene()) {
+        emit DependencyManager::get<scriptable::ModelProviderFactory>()->modelRemovedFromScene(getID(), NestableType::Overlay, _model);
         _model->removeFromScene(scene, transaction);
         _model->addToScene(scene, transaction);
 
@@ -84,6 +85,7 @@ void ModelOverlay::update(float deltatime) {
             modelOverlay->setSubRenderItemIDs(newRenderItemIDs);
         });
         processMaterials();
+        emit DependencyManager::get<scriptable::ModelProviderFactory>()->modelAddedToScene(getID(), NestableType::Overlay, _model);
     }
     if (_visibleDirty) {
         _visibleDirty = false;
@@ -110,12 +112,14 @@ bool ModelOverlay::addToScene(Overlay::Pointer overlay, const render::ScenePoint
     Volume3DOverlay::addToScene(overlay, scene, transaction);
     _model->addToScene(scene, transaction);
     processMaterials();
+    emit DependencyManager::get<scriptable::ModelProviderFactory>()->modelAddedToScene(getID(), NestableType::Overlay, _model);
     return true;
 }
 
 void ModelOverlay::removeFromScene(Overlay::Pointer overlay, const render::ScenePointer& scene, render::Transaction& transaction) {
     Volume3DOverlay::removeFromScene(overlay, scene, transaction);
     _model->removeFromScene(scene, transaction);
+    emit DependencyManager::get<scriptable::ModelProviderFactory>()->modelRemovedFromScene(getID(), NestableType::Overlay, _model);
     transaction.updateItem<Overlay>(getRenderItemID(), [](Overlay& data) {
         auto modelOverlay = static_cast<ModelOverlay*>(&data);
         modelOverlay->clearSubRenderItemIDs();
@@ -659,4 +663,23 @@ void ModelOverlay::processMaterials() {
             material.pop();
         }
     }
-}
+}
+
+bool ModelOverlay::canReplaceModelMeshPart(int meshIndex, int partIndex) {
+    // TODO: bounds checking; for now just used to indicate provider generally supports mesh updates
+    return _model && _model->isLoaded();
+}
+
+bool ModelOverlay::replaceScriptableModelMeshPart(scriptable::ScriptableModelBasePointer newModel, int meshIndex, int partIndex) {
+    return canReplaceModelMeshPart(meshIndex, partIndex) &&
+        _model->replaceScriptableModelMeshPart(newModel, meshIndex, partIndex);
+}
+
+scriptable::ScriptableModelBase ModelOverlay::getScriptableModel() {
+    if (!_model || !_model->isLoaded()) {
+        return Base3DOverlay::getScriptableModel();
+    }
+    auto result = _model->getScriptableModel();
+    result.objectID = getID();
+    return result;
+}

interface/src/ui/overlays/ModelOverlay.h

@@ -62,6 +62,10 @@ public:
     void addMaterial(graphics::MaterialLayer material, const std::string& parentMaterialName) override;
     void removeMaterial(graphics::MaterialPointer material, const std::string& parentMaterialName) override;
 
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
+    virtual bool canReplaceModelMeshPart(int meshIndex, int partIndex) override;
+    virtual bool replaceScriptableModelMeshPart(scriptable::ScriptableModelBasePointer model, int meshIndex, int partIndex) override;
+
 protected:
     Transform evalRenderTransform() override;
 

interface/src/ui/overlays/Shape3DOverlay.cpp

@@ -167,3 +167,14 @@ Transform Shape3DOverlay::evalRenderTransform() {
     transform.setRotation(rotation);
     return transform;
 }
+
+scriptable::ScriptableModelBase Shape3DOverlay::getScriptableModel() {
+    auto geometryCache = DependencyManager::get<GeometryCache>();
+    auto vertexColor = ColorUtils::toVec3(_color);
+    scriptable::ScriptableModelBase result;
+    result.objectID = getID();
+    if (auto mesh = geometryCache->meshFromShape(_shape, vertexColor)) {
+        result.append(mesh);
+    }
+    return result;
+}

interface/src/ui/overlays/Shape3DOverlay.h

@@ -33,6 +33,7 @@ public:
     void setProperties(const QVariantMap& properties) override;
     QVariant getProperty(const QString& property) override;
 
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
 protected:
     Transform evalRenderTransform() override;
 

interface/src/ui/overlays/Sphere3DOverlay.cpp

@@ -123,3 +123,15 @@ Transform Sphere3DOverlay::evalRenderTransform() {
 
     return transform;
 }
+
+
+scriptable::ScriptableModelBase Sphere3DOverlay::getScriptableModel() {
+    auto geometryCache = DependencyManager::get<GeometryCache>();
+    auto vertexColor = ColorUtils::toVec3(_color);
+    scriptable::ScriptableModelBase result;
+    if (auto mesh = geometryCache->meshFromShape(GeometryCache::Sphere, vertexColor)) {
+        result.objectID = getID();
+        result.append(mesh);
+    }
+    return result;
+}

interface/src/ui/overlays/Sphere3DOverlay.h

@@ -28,6 +28,7 @@ public:
 
     virtual Sphere3DOverlay* createClone() const override;
 
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
 protected:
     Transform evalRenderTransform() override;
 };

interface/src/ui/overlays/Web3DOverlay.cpp

@@ -26,6 +26,7 @@
 #include <GeometryUtil.h>
 #include <gl/GLHelpers.h>
 #include <scripting/HMDScriptingInterface.h>
+#include <scripting/WindowScriptingInterface.h>
 #include <ui/OffscreenQmlSurface.h>
 #include <ui/OffscreenQmlSurfaceCache.h>
 #include <ui/TabletScriptingInterface.h>
@@ -233,6 +234,7 @@ void Web3DOverlay::setupQmlSurface() {
         _webSurface->getSurfaceContext()->setContextProperty("Controller", DependencyManager::get<controller::ScriptingInterface>().data());
         _webSurface->getSurfaceContext()->setContextProperty("Pointers", DependencyManager::get<PointerScriptingInterface>().data());
         _webSurface->getSurfaceContext()->setContextProperty("Web3DOverlay", this);
+        _webSurface->getSurfaceContext()->setContextProperty("Window", DependencyManager::get<WindowScriptingInterface>().data());
 
         _webSurface->getSurfaceContext()->setContextProperty("pathToFonts", "../../");
 

libraries/avatars-renderer/CMakeLists.txt

@@ -14,5 +14,6 @@ include_hifi_library_headers(audio)
 include_hifi_library_headers(entities)
 include_hifi_library_headers(octree)
 include_hifi_library_headers(task)
+include_hifi_library_headers(graphics-scripting) # for ScriptableModel.h
 
 target_bullet()

libraries/avatars-renderer/src/avatars-renderer/Avatar.cpp

@@ -35,6 +35,8 @@
 #include "ModelEntityItem.h"
 #include "RenderableModelEntityItem.h"
 
+#include <graphics-scripting/Forward.h>
+
 #include "Logging.h"
 
 using namespace std;
@@ -576,6 +578,7 @@ void Avatar::addToScene(AvatarSharedPointer self, const render::ScenePointer& sc
     }
 
     _mustFadeIn = true;
+    emit DependencyManager::get<scriptable::ModelProviderFactory>()->modelAddedToScene(getSessionUUID(), NestableType::Avatar, _skeletonModel);
 }
 
 void Avatar::fadeIn(render::ScenePointer scene) {
@@ -625,6 +628,7 @@ void Avatar::removeFromScene(AvatarSharedPointer self, const render::ScenePointe
     for (auto& attachmentModel : _attachmentModels) {
         attachmentModel->removeFromScene(scene, transaction);
     }
+    emit DependencyManager::get<scriptable::ModelProviderFactory>()->modelRemovedFromScene(getSessionUUID(), NestableType::Avatar, _skeletonModel);
 }
 
 void Avatar::updateRenderItem(render::Transaction& transaction) {
@@ -1789,4 +1793,13 @@ void Avatar::processMaterials() {
             material.pop();
         }
     }
+}
+
+scriptable::ScriptableModelBase Avatar::getScriptableModel() {
+    if (!_skeletonModel || !_skeletonModel->isLoaded()) {
+        return scriptable::ScriptableModelBase();
+    }
+    auto result = _skeletonModel->getScriptableModel();
+    result.objectID = getSessionUUID().isNull() ? AVATAR_SELF_ID : getSessionUUID();
+    return result;
 }

libraries/avatars-renderer/src/avatars-renderer/Avatar.h

@@ -20,6 +20,7 @@
 #include <AvatarData.h>
 #include <ShapeInfo.h>
 #include <render/Scene.h>
+#include <graphics-scripting/Forward.h>
 #include <GLMHelpers.h>
 
 
@@ -53,7 +54,7 @@ class Texture;
 
 using AvatarPhysicsCallback = std::function<void(uint32_t)>;
 
-class Avatar : public AvatarData {
+class Avatar : public AvatarData, public scriptable::ModelProvider {
     Q_OBJECT
 
     /**jsdoc
@@ -275,6 +276,8 @@ public:
     void addMaterial(graphics::MaterialLayer material, const std::string& parentMaterialName) override;
     void removeMaterial(graphics::MaterialPointer material, const std::string& parentMaterialName) override;
 
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
+
 public slots:
 
     // FIXME - these should be migrated to use Pose data instead

libraries/embedded-webserver/src/HTTPConnection.cpp

@@ -133,12 +133,56 @@ QList<FormData> HTTPConnection::parseFormData() const {
 }
 
 void HTTPConnection::respond(const char* code, const QByteArray& content, const char* contentType, const Headers& headers) {
+    respondWithStatusAndHeaders(code, contentType, headers, content.size());
+
+    _socket->write(content);
+
+    _socket->disconnectFromHost();
+
+    // make sure we receive no further read notifications
+    disconnect(_socket, &QTcpSocket::readyRead, this, nullptr);
+}
+
+void HTTPConnection::respond(const char* code, std::unique_ptr<QIODevice> device, const char* contentType, const Headers& headers) {
+    _responseDevice = std::move(device);
+
+    if (_responseDevice->isSequential()) {
+        qWarning() << "Error responding to HTTPConnection: sequential IO devices not supported";
+        respondWithStatusAndHeaders(StatusCode500, contentType, headers, 0);
+        _socket->disconnect(SIGNAL(readyRead()), this);
+        _socket->disconnectFromHost();
+        return;
+    }
+
+    int totalToBeWritten = _responseDevice->size();
+    respondWithStatusAndHeaders(code, contentType, headers, totalToBeWritten);
+
+    if (_responseDevice->atEnd()) {
+        _socket->disconnectFromHost();
+    } else {
+        connect(_socket, &QTcpSocket::bytesWritten, this, [this, totalToBeWritten](size_t bytes) mutable {
+            constexpr size_t HTTP_RESPONSE_CHUNK_SIZE = 1024 * 10;
+            if (!_responseDevice->atEnd()) {
+                totalToBeWritten -= _socket->write(_responseDevice->read(HTTP_RESPONSE_CHUNK_SIZE));
+                if (_responseDevice->atEnd()) {
+                    _socket->disconnectFromHost();
+                    disconnect(_socket, &QTcpSocket::bytesWritten, this, nullptr);
+                }
+            }
+        });
+
+    }
+
+    // make sure we receive no further read notifications
+    disconnect(_socket, &QTcpSocket::readyRead, this, nullptr);
+}
+
+void HTTPConnection::respondWithStatusAndHeaders(const char* code, const char* contentType, const Headers& headers, qint64 contentLength) {
     _socket->write("HTTP/1.1 ");
+
     _socket->write(code);
     _socket->write("\r\n");
 
-    int csize = content.size();
-
     for (Headers::const_iterator it = headers.constBegin(), end = headers.constEnd();
             it != end; it++) {
         _socket->write(it.key());
@@ -146,9 +190,10 @@ void HTTPConnection::respond(const char* code, const QByteArray& content, const
         _socket->write(it.value());
         _socket->write("\r\n");
     }
-    if (csize > 0) {
+
+    if (contentLength > 0) {
         _socket->write("Content-Length: ");
-        _socket->write(QByteArray::number(csize));
+        _socket->write(QByteArray::number(contentLength));
         _socket->write("\r\n");
 
         _socket->write("Content-Type: ");
@@ -156,21 +201,16 @@ void HTTPConnection::respond(const char* code, const QByteArray& content, const
         _socket->write("\r\n");
     }
     _socket->write("Connection: close\r\n\r\n");
-
-    if (csize > 0) {
-        _socket->write(content);
-    }
-
-    // make sure we receive no further read notifications
-    _socket->disconnect(SIGNAL(readyRead()), this);
-
-    _socket->disconnectFromHost();
 }
 
 void HTTPConnection::readRequest() {
     if (!_socket->canReadLine()) {
         return;
     }
+    if (!_requestUrl.isEmpty()) {
+        qDebug() << "Request URL was already set";
+        return;
+    }
     // parse out the method and resource
     QByteArray line = _socket->readLine().trimmed();
     if (line.startsWith("HEAD")) {

libraries/embedded-webserver/src/HTTPConnection.h

@@ -26,6 +26,8 @@
 #include <QPair>
 #include <QUrl>
 
+#include <memory>
+
 class QTcpSocket;
 class HTTPManager;
 class MaskFilter;
@@ -87,6 +89,9 @@ public:
     void respond (const char* code, const QByteArray& content = QByteArray(),
         const char* contentType = DefaultContentType,
         const Headers& headers = Headers());
+    void respond (const char* code, std::unique_ptr<QIODevice> device,
+        const char* contentType = DefaultContentType,
+        const Headers& headers = Headers());
 
 protected slots:
 
@@ -100,6 +105,7 @@ protected slots:
     void readContent ();
 
 protected:
+    void respondWithStatusAndHeaders(const char* code, const char* contentType, const Headers& headers, qint64 size);
 
     /// The parent HTTP manager
     HTTPManager* _parentManager;
@@ -127,6 +133,9 @@ protected:
 
     /// The content of the request.
     QByteArray _requestContent;
+
+    /// Response content
+    std::unique_ptr<QIODevice> _responseDevice;
 };
 
 #endif // hifi_HTTPConnection_h

libraries/embedded-webserver/src/HTTPManager.cpp

@@ -98,13 +98,14 @@ bool HTTPManager::handleHTTPRequest(HTTPConnection* connection, const QUrl& url,
             // file exists, serve it
             static QMimeDatabase mimeDatabase;
             
-            QFile localFile(filePath);
-            localFile.open(QIODevice::ReadOnly);
-            QByteArray localFileData = localFile.readAll();
+            auto localFile = std::unique_ptr<QFile>(new QFile(filePath));
+            localFile->open(QIODevice::ReadOnly);
+            QByteArray localFileData;
             
             QFileInfo localFileInfo(filePath);
             
             if (localFileInfo.completeSuffix() == "shtml") {
+                localFileData = localFile->readAll();
                 // this is a file that may have some SSI statements
                 // the only thing we support is the include directive, but check the contents for that
                 
@@ -153,8 +154,12 @@ bool HTTPManager::handleHTTPRequest(HTTPConnection* connection, const QUrl& url,
                 ? QString { "text/html" }
                 : mimeDatabase.mimeTypeForFile(filePath).name();
 
-            connection->respond(HTTPConnection::StatusCode200, localFileData, qPrintable(mimeType));
-            
+            if (localFileData.isNull()) {
+                connection->respond(HTTPConnection::StatusCode200, std::move(localFile), qPrintable(mimeType));
+            } else {
+                connection->respond(HTTPConnection::StatusCode200, localFileData, qPrintable(mimeType));
+            }
+
             return true;
         }
     }

libraries/entities-renderer/CMakeLists.txt

@@ -14,6 +14,7 @@ include_hifi_library_headers(entities)
 include_hifi_library_headers(avatars)
 include_hifi_library_headers(controllers)
 include_hifi_library_headers(task)
+include_hifi_library_headers(graphics-scripting) # for Forward.h
 
 target_bullet()
 target_polyvox()

libraries/entities-renderer/src/RenderableEntityItem.cpp

@@ -164,12 +164,7 @@ ItemKey EntityRenderer::getKey() {
         return ItemKey::Builder::transparentShape().withTypeMeta().withTagBits(render::ItemKey::TAG_BITS_0 | render::ItemKey::TAG_BITS_1);
     }
 
-    // This allows shapes to cast shadows
-    if (_canCastShadow) {
-        return ItemKey::Builder::opaqueShape().withTypeMeta().withTagBits(render::ItemKey::TAG_BITS_0 | render::ItemKey::TAG_BITS_1).withShadowCaster();
-    } else {
-        return ItemKey::Builder::opaqueShape().withTypeMeta().withTagBits(render::ItemKey::TAG_BITS_0 | render::ItemKey::TAG_BITS_1);
-    }
+    return ItemKey::Builder::opaqueShape().withTypeMeta().withTagBits(render::ItemKey::TAG_BITS_0 | render::ItemKey::TAG_BITS_1);
 }
 
 uint32_t EntityRenderer::metaFetchMetaSubItems(ItemIDs& subItems) {
@@ -382,7 +377,6 @@ void EntityRenderer::doRenderUpdateSynchronous(const ScenePointer& scene, Transa
 
         _moving = entity->isMovingRelativeToParent();
         _visible = entity->getVisible();
-        _canCastShadow = entity->getCanCastShadow();
         _cauterized = entity->getCauterized();
         _needsRenderUpdate = false;
     });

libraries/entities-renderer/src/RenderableEntityItem.h

@@ -17,13 +17,14 @@
 #include <Sound.h>
 #include "AbstractViewStateInterface.h"
 #include "EntitiesRendererLogging.h"
+#include <graphics-scripting/Forward.h>
 
 class EntityTreeRenderer;
 
 namespace render { namespace entities {
 
 // Base class for all renderable entities
-class EntityRenderer : public QObject, public std::enable_shared_from_this<EntityRenderer>, public PayloadProxyInterface, protected ReadWriteLockable {
+class EntityRenderer : public QObject, public std::enable_shared_from_this<EntityRenderer>, public PayloadProxyInterface, protected ReadWriteLockable, public scriptable::ModelProvider {
     Q_OBJECT
 
     using Pointer = std::shared_ptr<EntityRenderer>;
@@ -37,7 +38,7 @@ public:
     virtual bool wantsKeyboardFocus() const { return false; }
     virtual void setProxyWindow(QWindow* proxyWindow) {}
     virtual QObject* getEventHandler() { return nullptr; }
-    const EntityItemPointer& getEntity() { return _entity; }
+    const EntityItemPointer& getEntity() const { return _entity; }
     const ItemID& getRenderItemID() const { return _renderItemID; }
 
     const SharedSoundPointer& getCollisionSound() { return _collisionSound; }
@@ -57,6 +58,8 @@ public:
     virtual void addMaterial(graphics::MaterialLayer material, const std::string& parentMaterialName);
     virtual void removeMaterial(graphics::MaterialPointer material, const std::string& parentMaterialName);
 
+    virtual scriptable::ScriptableModelBase getScriptableModel() override { return scriptable::ScriptableModelBase(); }
+
 protected:
     virtual bool needsRenderUpdateFromEntity() const final { return needsRenderUpdateFromEntity(_entity); }
     virtual void onAddToScene(const EntityItemPointer& entity);
@@ -126,7 +129,6 @@ protected:
     bool _isFading{ _entitiesShouldFadeFunction() };
     bool _prevIsTransparent { false };
     bool _visible { false };
-    bool _canCastShadow { false };
     bool _cauterized { false };
     bool _moving { false };
     bool _needsRenderUpdate { false };

libraries/entities-renderer/src/RenderableModelEntityItem.cpp

@@ -955,6 +955,7 @@ QStringList RenderableModelEntityItem::getJointNames() const {
     return result;
 }
 
+// FIXME: deprecated; remove >= RC67
 bool RenderableModelEntityItem::getMeshes(MeshProxyList& result) {
     auto model = getModel();
     if (!model || !model->isLoaded()) {
@@ -964,6 +965,34 @@ bool RenderableModelEntityItem::getMeshes(MeshProxyList& result) {
     return !result.isEmpty();
 }
 
+scriptable::ScriptableModelBase render::entities::ModelEntityRenderer::getScriptableModel() {
+    auto model = resultWithReadLock<ModelPointer>([this]{ return _model; });
+
+    if (!model || !model->isLoaded()) {
+        return scriptable::ScriptableModelBase();
+    }
+
+    auto result = _model->getScriptableModel();
+    result.objectID = getEntity()->getID();
+    return result;
+}
+
+bool render::entities::ModelEntityRenderer::canReplaceModelMeshPart(int meshIndex, int partIndex) {
+    // TODO: for now this method is just used to indicate that this provider generally supports mesh updates
+    auto model = resultWithReadLock<ModelPointer>([this]{ return _model; });
+    return model && model->isLoaded();
+}
+
+bool render::entities::ModelEntityRenderer::replaceScriptableModelMeshPart(scriptable::ScriptableModelBasePointer newModel, int meshIndex, int partIndex) {
+    auto model = resultWithReadLock<ModelPointer>([this]{ return _model; });
+
+    if (!model || !model->isLoaded()) {
+        return false;
+    }
+
+    return model->replaceScriptableModelMeshPart(newModel, meshIndex, partIndex);
+}
+
 void RenderableModelEntityItem::simulateRelayedJoints() {
     ModelPointer model = getModel();
     if (model && model->isLoaded()) {
@@ -1057,7 +1086,6 @@ void ModelEntityRenderer::onRemoveFromSceneTyped(const TypedEntityPointer& entit
     entity->setModel({});
 }
 
-
 void ModelEntityRenderer::animate(const TypedEntityPointer& entity) {
     if (!_animation || !_animation->isLoaded()) {
         return;
@@ -1281,6 +1309,8 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce
                 auto entityRenderer = static_cast<EntityRenderer*>(&data);
                 entityRenderer->clearSubRenderItemIDs();
             });
+            emit DependencyManager::get<scriptable::ModelProviderFactory>()->
+                modelRemovedFromScene(entity->getEntityItemID(), NestableType::Entity, _model);
         }
         return;
     }
@@ -1291,6 +1321,10 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce
         connect(model.get(), &Model::setURLFinished, this, [&](bool didVisualGeometryRequestSucceed) {
             setKey(didVisualGeometryRequestSucceed);
             emit requestRenderUpdate();
+            if(didVisualGeometryRequestSucceed) {
+                emit DependencyManager::get<scriptable::ModelProviderFactory>()->
+                    modelAddedToScene(entity->getEntityItemID(), NestableType::Entity, _model);
+            }
         });
         connect(model.get(), &Model::requestRenderUpdate, this, &ModelEntityRenderer::requestRenderUpdate);
         connect(entity.get(), &RenderableModelEntityItem::requestCollisionGeometryUpdate, this, &ModelEntityRenderer::flagForCollisionGeometryUpdate);
@@ -1361,10 +1395,6 @@ void ModelEntityRenderer::doRenderUpdateSynchronousTyped(const ScenePointer& sce
     }
     // TODO? early exit here when not visible?
 
-    if (model->canCastShadow() != _canCastShadow) {
-        model->setCanCastShadow(_canCastShadow, scene, viewTaskBits, false);
-    }
-
     if (_needsCollisionGeometryUpdate) {
         setCollisionMeshKey(entity->getCollisionMeshKey());
         _needsCollisionGeometryUpdate = false;

libraries/entities-renderer/src/RenderableModelEntityItem.h

@@ -111,7 +111,7 @@ public:
     virtual int getJointIndex(const QString& name) const override;
     virtual QStringList getJointNames() const override;
 
-    bool getMeshes(MeshProxyList& result) override;
+    bool getMeshes(MeshProxyList& result) override; // deprecated
     const void* getCollisionMeshKey() const { return _collisionMeshKey; }
 
 signals:
@@ -142,6 +142,9 @@ class ModelEntityRenderer : public TypedEntityRenderer<RenderableModelEntityItem
 
 public:
     ModelEntityRenderer(const EntityItemPointer& entity);
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
+    virtual bool canReplaceModelMeshPart(int meshIndex, int partIndex) override;
+    virtual bool replaceScriptableModelMeshPart(scriptable::ScriptableModelBasePointer model, int meshIndex, int partIndex) override;
 
     void addMaterial(graphics::MaterialLayer material, const std::string& parentMaterialName) override;
     void removeMaterial(graphics::MaterialPointer material, const std::string& parentMaterialName) override;

libraries/entities-renderer/src/RenderablePolyLineEntityItem.cpp

@@ -281,6 +281,10 @@ std::vector<PolyLineEntityRenderer::Vertex> PolyLineEntityRenderer::updateVertic
     return vertices;
 }
 
+scriptable::ScriptableModelBase PolyLineEntityRenderer::getScriptableModel() {
+    // TODO: adapt polyline into a triangles mesh...
+    return EntityRenderer::getScriptableModel();
+}
 
 void PolyLineEntityRenderer::doRender(RenderArgs* args) {
     if (_empty) {
@@ -319,4 +323,4 @@ void PolyLineEntityRenderer::doRender(RenderArgs* args) {
 #endif
 
     batch.draw(gpu::TRIANGLE_STRIP, _numVertices, 0);
-}
+}

libraries/entities-renderer/src/RenderablePolyLineEntityItem.h

@@ -25,6 +25,7 @@ class PolyLineEntityRenderer : public TypedEntityRenderer<PolyLineEntityItem> {
 public:
     PolyLineEntityRenderer(const EntityItemPointer& entity);
 
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
 protected:
     virtual bool needsRenderUpdateFromTypedEntity(const TypedEntityPointer& entity) const override;
     virtual void doRenderUpdateSynchronousTyped(const ScenePointer& scene, 

libraries/entities-renderer/src/RenderablePolyVoxEntityItem.cpp

@@ -1418,6 +1418,7 @@ void RenderablePolyVoxEntityItem::bonkNeighbors() {
     }
 }
 
+// deprecated
 bool RenderablePolyVoxEntityItem::getMeshes(MeshProxyList& result) {
     if (!updateDependents()) {
         return false;
@@ -1450,6 +1451,37 @@ bool RenderablePolyVoxEntityItem::getMeshes(MeshProxyList& result) {
     return success;
 }
 
+scriptable::ScriptableModelBase RenderablePolyVoxEntityItem::getScriptableModel() {
+    if (!updateDependents() || !_mesh) {
+        return scriptable::ScriptableModelBase();
+    }
+
+    bool success = false;
+    glm::mat4 transform = voxelToLocalMatrix();
+    scriptable::ScriptableModelBase result;
+    result.objectID = getThisPointer()->getID();
+    withReadLock([&] {
+        gpu::BufferView::Index numVertices = (gpu::BufferView::Index)_mesh->getNumVertices();
+        if (!_meshReady) {
+            // we aren't ready to return a mesh.  the caller will have to try again later.
+            success = false;
+        } else if (numVertices == 0) {
+            // we are ready, but there are no triangles in the mesh.
+            success = true;
+        } else {
+            success = true;
+            // the mesh will be in voxel-space.  transform it into object-space
+            result.append(_mesh->map(
+                [=](glm::vec3 position){ return glm::vec3(transform * glm::vec4(position, 1.0f)); },
+                [=](glm::vec3 color){ return color; },
+                [=](glm::vec3 normal){ return glm::normalize(glm::vec3(transform * glm::vec4(normal, 0.0f))); },
+                [&](uint32_t index){ return index; }
+            ));
+        }
+    });
+    return result;
+}
+
 using namespace render;
 using namespace render::entities;
 

libraries/entities-renderer/src/RenderablePolyVoxEntityItem.h

@@ -32,7 +32,7 @@ namespace render { namespace entities {
 class PolyVoxEntityRenderer;
 } } 
 
-class RenderablePolyVoxEntityItem : public PolyVoxEntityItem {
+class RenderablePolyVoxEntityItem : public PolyVoxEntityItem, public scriptable::ModelProvider {
     friend class render::entities::PolyVoxEntityRenderer;
 
 public:
@@ -113,7 +113,8 @@ public:
 
     void setVolDataDirty() { withWriteLock([&] { _volDataDirty = true; _meshReady = false; }); }
 
-    bool getMeshes(MeshProxyList& result) override;
+    bool getMeshes(MeshProxyList& result) override; // deprecated
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
 
 private:
     bool updateOnCount(const ivec3& v, uint8_t toValue);
@@ -163,6 +164,9 @@ class PolyVoxEntityRenderer : public TypedEntityRenderer<RenderablePolyVoxEntity
 
 public:
     PolyVoxEntityRenderer(const EntityItemPointer& entity);
+    virtual scriptable::ScriptableModelBase getScriptableModel() override {
+        return asTypedEntity<RenderablePolyVoxEntityItem>()->getScriptableModel();
+    }
     
 protected:
     virtual ItemKey getKey() override { return ItemKey::Builder::opaqueShape().withTagBits(render::ItemKey::TAG_BITS_0 | render::ItemKey::TAG_BITS_1); }

libraries/entities-renderer/src/RenderableShapeEntityItem.cpp

@@ -163,3 +163,18 @@ void ShapeEntityRenderer::doRender(RenderArgs* args) {
     const auto triCount = geometryCache->getShapeTriangleCount(geometryShape);
     args->_details._trianglesRendered += (int)triCount;
 }
+
+scriptable::ScriptableModelBase ShapeEntityRenderer::getScriptableModel()  {
+    scriptable::ScriptableModelBase result;
+    auto geometryCache = DependencyManager::get<GeometryCache>();
+    auto geometryShape = geometryCache->getShapeForEntityShape(_shape);
+    glm::vec3 vertexColor;
+    if (_materials["0"].top().material) {
+        vertexColor = _materials["0"].top().material->getAlbedo();
+    }
+    if (auto mesh = geometryCache->meshFromShape(geometryShape, vertexColor)) {
+        result.objectID = getEntity()->getID();
+        result.append(mesh);
+    }
+    return result;
+}

libraries/entities-renderer/src/RenderableShapeEntityItem.h

@@ -22,6 +22,8 @@ class ShapeEntityRenderer : public TypedEntityRenderer<ShapeEntityItem> {
 public:
     ShapeEntityRenderer(const EntityItemPointer& entity);
 
+    virtual scriptable::ScriptableModelBase getScriptableModel() override;
+
 private:
     virtual bool needsRenderUpdate() const override;
     virtual bool needsRenderUpdateFromTypedEntity(const TypedEntityPointer& entity) const override;

libraries/entities-renderer/src/RenderableZoneEntityItem.cpp

@@ -330,7 +330,6 @@ void ZoneEntityRenderer::updateKeySunFromEntity(const TypedEntityPointer& entity
     sunLight->setColor(ColorUtils::toVec3(_keyLightProperties.getColor()));
     sunLight->setIntensity(_keyLightProperties.getIntensity());
     sunLight->setDirection(entity->getTransform().getRotation() * _keyLightProperties.getDirection());
-    sunLight->setCastShadows(_keyLightProperties.getCastShadows());
 }
 
 void ZoneEntityRenderer::updateAmbientLightFromEntity(const TypedEntityPointer& entity) {