Merge hifi/master of into hifi/workload

.clang-format

@@ -1,12 +1,12 @@
 Language: Cpp
 Standard: Cpp11
-BasedOnStyle: "Chromium" 
+BasedOnStyle: "Chromium"
 ColumnLimit: 128
 IndentWidth: 4
 UseTab: Never
 
 BreakBeforeBraces: Custom
-BraceWrapping:   
+BraceWrapping:
   AfterEnum: true
   AfterClass: false
   AfterControlStatement: false
@@ -21,11 +21,11 @@ BraceWrapping:
 
 
 AccessModifierOffset: -4
-AllowShortFunctionsOnASingleLine: InlineOnly  
-BreakConstructorInitializers: BeforeColon 
-BreakConstructorInitializersBeforeComma: true
+AllowShortFunctionsOnASingleLine: InlineOnly
+BreakConstructorInitializers: AfterColon
+BreakConstructorInitializersBeforeComma: false
 IndentCaseLabels: true
-ReflowComments: false 
+ReflowComments: false
 Cpp11BracedListStyle: false
 ContinuationIndentWidth: 4
 ConstructorInitializerAllOnOneLineOrOnePerLine: false

assignment-client/src/Agent.cpp

@@ -340,7 +340,6 @@ void Agent::scriptRequestFinished() {
     request->deleteLater();
 }
 
-
 void Agent::executeScript() {
     _scriptEngine = scriptEngineFactory(ScriptEngine::AGENT_SCRIPT, _scriptContents, _payload);
 

assignment-client/src/assets/AssetServer.cpp

@@ -257,12 +257,10 @@ AssetServer::AssetServer(ReceivedMessage& message) :
     _transferTaskPool.setMaxThreadCount(TASK_POOL_THREAD_COUNT);
     _bakingTaskPool.setMaxThreadCount(1);
 
+    // Queue all requests until the Asset Server is fully setup
     auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
-    packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo");
-    packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload");
-    packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOperation");
-    
+    packetReceiver.registerListenerForTypes({ PacketType::AssetGet, PacketType::AssetGetInfo, PacketType::AssetUpload, PacketType::AssetMappingOperation }, this, "queueRequests");
+
 #ifdef Q_OS_WIN
     updateConsumedCores();
     QTimer* timer = new QTimer(this);
@@ -291,6 +289,7 @@ void AssetServer::aboutToFinish() {
         if (pendingRunnable) {
             it = _pendingBakes.erase(it);
         } else {
+            qDebug() << "Aborting bake for" << it.key();
             it.value()->abort();
             ++it;
         }
@@ -396,6 +395,7 @@ void AssetServer::completeSetup() {
 
         if (_fileMappings.size() > 0) {
             cleanupUnmappedFiles();
+            cleanupBakedFilesForDeletedAssets();
         }
 
         nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
@@ -417,10 +417,65 @@ void AssetServer::completeSetup() {
 
     PathUtils::removeTemporaryApplicationDirs();
     PathUtils::removeTemporaryApplicationDirs("Oven");
+
+    qCDebug(asset_server) << "Overriding temporary queuing packet handler.";
+    // We're fully setup, override the request queueing handler and replay all requests
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(PacketType::AssetGet, this, "handleAssetGet");
+    packetReceiver.registerListener(PacketType::AssetGetInfo, this, "handleAssetGetInfo");
+    packetReceiver.registerListener(PacketType::AssetUpload, this, "handleAssetUpload");
+    packetReceiver.registerListener(PacketType::AssetMappingOperation, this, "handleAssetMappingOperation");
+
+    replayRequests();
+}
+
+void AssetServer::queueRequests(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode) {
+    qCDebug(asset_server) << "Queuing requests until fully setup";
+
+    QMutexLocker lock { &_queuedRequestsMutex };
+    _queuedRequests.push_back({ packet, senderNode });
+
+    // If we've stopped queueing but the callback was already in flight,
+    // then replay it immediately.
+    if (!_isQueueingRequests) {
+        lock.unlock();
+        replayRequests();
+    }
+}
+
+void AssetServer::replayRequests() {
+    RequestQueue queue;
+    {
+        QMutexLocker lock { &_queuedRequestsMutex };
+        std::swap(queue, _queuedRequests);
+        _isQueueingRequests = false;
+    }
+
+    qCDebug(asset_server) << "Replaying" << queue.size() << "requests.";
+
+    for (const auto& request : queue) {
+        switch (request.first->getType()) {
+            case PacketType::AssetGet:
+                handleAssetGet(request.first, request.second);
+                break;
+            case PacketType::AssetGetInfo:
+                handleAssetGetInfo(request.first, request.second);
+                break;
+            case PacketType::AssetUpload:
+                handleAssetUpload(request.first, request.second);
+                break;
+            case PacketType::AssetMappingOperation:
+                handleAssetMappingOperation(request.first, request.second);
+                break;
+            default:
+                qCWarning(asset_server) << "Unknown queued request type:" << request.first->getType();
+                break;
+        }
+    }
 }
 
 void AssetServer::cleanupUnmappedFiles() {
-    QRegExp hashFileRegex { "^[a-f0-9]{" + QString::number(AssetUtils::SHA256_HASH_HEX_LENGTH) + "}" };
+    QRegExp hashFileRegex { AssetUtils::ASSET_HASH_REGEX_STRING };
 
     auto files = _filesDirectory.entryInfoList(QDir::Files);
 
@@ -452,6 +507,38 @@ void AssetServer::cleanupUnmappedFiles() {
     }
 }
 
+void AssetServer::cleanupBakedFilesForDeletedAssets() {
+    qCInfo(asset_server) << "Performing baked asset cleanup for deleted assets";
+
+    std::set<AssetUtils::AssetHash> bakedHashes;
+
+    for (const auto& it : _fileMappings) {
+        // check if this is a mapping to baked content
+        if (it.first.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
+            // extract the hash from the baked mapping
+            AssetUtils::AssetHash hash = it.first.mid(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER.length(),
+                                                      AssetUtils::SHA256_HASH_HEX_LENGTH);
+
+            // add the hash to our set of hashes for which we have baked content
+            bakedHashes.insert(hash);
+        }
+    }
+
+    // enumerate the hashes for which we have baked content
+    for (const auto& hash : bakedHashes) {
+        // check if we have a mapping that points to this hash
+        auto matchingMapping = std::find_if(std::begin(_fileMappings), std::end(_fileMappings),
+                                            [&hash](const std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash> mappingPair) {
+            return mappingPair.second == hash;
+        });
+
+        if (matchingMapping == std::end(_fileMappings)) {
+            // we didn't find a mapping for this hash, remove any baked content we still have for it
+            removeBakedPathsForDeletedAsset(hash);
+        }
+    }
+}
+
 void AssetServer::handleAssetMappingOperation(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode) {
     using AssetMappingOperationType = AssetUtils::AssetMappingOperationType;
 
@@ -1301,6 +1388,8 @@ void AssetServer::handleCompletedBake(QString originalAssetHash, QString origina
 }
 
 void AssetServer::handleAbortedBake(QString originalAssetHash, QString assetPath) {
+    qDebug() << "Aborted bake:" << originalAssetHash;
+
     // for an aborted bake we don't do anything but remove the BakeAssetTask from our pending bakes
     _pendingBakes.remove(originalAssetHash);
 }

assignment-client/src/assets/AssetServer.h

@@ -44,6 +44,7 @@ public slots:
 private slots:
     void completeSetup();
 
+    void queueRequests(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetGetInfo(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetGet(QSharedPointer<ReceivedMessage> packet, SharedNodePointer senderNode);
     void handleAssetUpload(QSharedPointer<ReceivedMessage> packetList, SharedNodePointer senderNode);
@@ -52,6 +53,8 @@ private slots:
     void sendStatsPacket() override;
 
 private:
+    void replayRequests();
+
     void handleGetMappingOperation(ReceivedMessage& message, NLPacketList& replyPacket);
     void handleGetAllMappingOperation(NLPacketList& replyPacket);
     void handleSetMappingOperation(ReceivedMessage& message, bool hasWriteAccess, NLPacketList& replyPacket);
@@ -80,6 +83,9 @@ private:
     /// Delete any unmapped files from the local asset directory
     void cleanupUnmappedFiles();
 
+    /// Delete any baked files for assets removed from the local asset directory
+    void cleanupBakedFilesForDeletedAssets();
+
     QString getPathToAssetHash(const AssetUtils::AssetHash& assetHash);
 
     std::pair<AssetUtils::BakingStatus, QString> getAssetStatus(const AssetUtils::AssetPath& path, const AssetUtils::AssetHash& hash);
@@ -115,6 +121,11 @@ private:
     QHash<AssetUtils::AssetHash, std::shared_ptr<BakeAssetTask>> _pendingBakes;
     QThreadPool _bakingTaskPool;
 
+    QMutex _queuedRequestsMutex;
+    bool _isQueueingRequests { true };
+    using RequestQueue = QVector<QPair<QSharedPointer<ReceivedMessage>, SharedNodePointer>>;
+    RequestQueue _queuedRequests;
+
     bool _wasColorTextureCompressionEnabled { false };
     bool _wasGrayscaleTextureCompressionEnabled { false  };
     bool _wasNormalTextureCompressionEnabled { false };

assignment-client/src/assets/BakeAssetTask.cpp

@@ -69,8 +69,10 @@ void BakeAssetTask::run() {
 
     _ovenProcess.reset(new QProcess());
 
+    QEventLoop loop;
+
     connect(_ovenProcess.get(), static_cast<void(QProcess::*)(int, QProcess::ExitStatus)>(&QProcess::finished),
-            this, [this, tempOutputDir](int exitCode, QProcess::ExitStatus exitStatus) {
+            this, [&loop, this, tempOutputDir](int exitCode, QProcess::ExitStatus exitStatus) {
         qDebug() << "Baking process finished: " << exitCode << exitStatus;
 
         if (exitStatus == QProcess::CrashExit) {
@@ -108,6 +110,7 @@ void BakeAssetTask::run() {
             emit bakeFailed(_assetHash, _assetPath, errors);
         }
 
+        loop.quit();
     });
 
     qDebug() << "Starting oven for " << _assetPath;
@@ -117,11 +120,21 @@ void BakeAssetTask::run() {
         emit bakeFailed(_assetHash, _assetPath, errors);
         return;
     }
-    _ovenProcess->waitForFinished();
+
+    _isBaking = true;
+
+    loop.exec();
 }
 
 void BakeAssetTask::abort() {
-    if (!_wasAborted.exchange(true)) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "abort");
+        return;
+    }
+    qDebug() << "Aborting BakeAssetTask for" << _assetHash;
+    if (_ovenProcess->state() != QProcess::NotRunning) {
+        qDebug() << "Teminating oven process for" << _assetHash;
+        _wasAborted = true;
         _ovenProcess->terminate();
     }
 }

assignment-client/src/assets/BakeAssetTask.h

@@ -27,12 +27,14 @@ class BakeAssetTask : public QObject, public QRunnable {
 public:
     BakeAssetTask(const AssetUtils::AssetHash& assetHash, const AssetUtils::AssetPath& assetPath, const QString& filePath);
 
+    // Thread-safe inspection methods
     bool isBaking() { return _isBaking.load(); }
+    bool wasAborted() const { return _wasAborted.load(); }
 
     void run() override;
 
+public slots:
     void abort();
-    bool wasAborted() const { return _wasAborted.load(); }
 
 signals:
     void bakeComplete(QString assetHash, QString assetPath, QString tempOutputDir, QVector<QString> outputFiles);

assignment-client/src/audio/AudioMixer.cpp

@@ -9,6 +9,8 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include "AudioMixer.h"
+
 #include <thread>
 
 #include <QtCore/QJsonArray>
@@ -36,8 +38,6 @@
 #include "AvatarAudioStream.h"
 #include "InjectedAudioStream.h"
 
-#include "AudioMixer.h"
-
 static const float DEFAULT_ATTENUATION_PER_DOUBLING_IN_DISTANCE = 0.5f;    // attenuation = -6dB * log2(distance)
 static const int DISABLE_STATIC_JITTER_FRAMES = -1;
 static const float DEFAULT_NOISE_MUTING_THRESHOLD = 1.0f;

assignment-client/src/audio/AudioMixer.h

@@ -18,6 +18,8 @@
 #include <ThreadedAssignment.h>
 #include <UUIDHasher.h>
 
+#include <plugins/Forward.h>
+
 #include "AudioMixerStats.h"
 #include "AudioMixerSlavePool.h"
 

assignment-client/src/audio/AudioMixerClientData.cpp

@@ -9,6 +9,8 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include "AudioMixerClientData.h"
+
 #include <random>
 
 #include <QtCore/QDebug>
@@ -22,8 +24,6 @@
 #include "AudioLogging.h"
 #include "AudioHelpers.h"
 #include "AudioMixer.h"
-#include "AudioMixerClientData.h"
-
 
 AudioMixerClientData::AudioMixerClientData(const QUuid& nodeID) :
     NodeData(nodeID),

assignment-client/src/audio/AudioMixerClientData.h

@@ -21,6 +21,7 @@
 #include <AudioLimiter.h>
 #include <UUIDHasher.h>
 
+#include <plugins/Forward.h>
 #include <plugins/CodecPlugin.h>
 
 #include "PositionalAudioStream.h"

assignment-client/src/audio/AudioMixerSlave.cpp

@@ -9,6 +9,8 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
+#include "AudioMixerSlave.h"
+
 #include <algorithm>
 
 #include <glm/glm.hpp>
@@ -34,8 +36,6 @@
 #include "InjectedAudioStream.h"
 #include "AudioHelpers.h"
 
-#include "AudioMixerSlave.h"
-
 using AudioStreamMap = AudioMixerClientData::AudioStreamMap;
 
 // packet helpers
@@ -49,7 +49,7 @@ void sendEnvironmentPacket(const SharedNodePointer& node, AudioMixerClientData&
 inline float approximateGain(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
         const glm::vec3& relativePosition);
 inline float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudioStream& listeningNodeStream,
-        const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, bool isEcho);
+        const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, float distance, bool isEcho);
 inline float computeAzimuth(const AvatarAudioStream& listeningNodeStream, const PositionalAudioStream& streamToAdd,
         const glm::vec3& relativePosition);
 
@@ -276,7 +276,7 @@ void AudioMixerSlave::addStream(AudioMixerClientData& listenerNodeData, const QU
     glm::vec3 relativePosition = streamToAdd.getPosition() - listeningNodeStream.getPosition();
 
     float distance = glm::max(glm::length(relativePosition), EPSILON);
-    float gain = computeGain(listenerNodeData, listeningNodeStream, streamToAdd, relativePosition, isEcho);
+    float gain = computeGain(listenerNodeData, listeningNodeStream, streamToAdd, relativePosition, distance, isEcho);
     float azimuth = isEcho ? 0.0f : computeAzimuth(listeningNodeStream, listeningNodeStream, relativePosition);
     const int HRTF_DATASET_INDEX = 1;
 
@@ -489,9 +489,6 @@ float approximateGain(const AvatarAudioStream& listeningNodeStream, const Positi
     // avatar: skip attenuation - it is too costly to approximate
 
     // distance attenuation: approximate, ignore zone-specific attenuations
-    // this is a good approximation for streams further than ATTENUATION_START_DISTANCE
-    // those streams closer will be amplified; amplifying close streams is acceptable
-    // when throttling, as close streams are expected to be heard by a user
     float distance = glm::length(relativePosition);
     return gain / distance;
 
@@ -499,7 +496,7 @@ float approximateGain(const AvatarAudioStream& listeningNodeStream, const Positi
 }
 
 float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudioStream& listeningNodeStream,
-        const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, bool isEcho) {
+        const PositionalAudioStream& streamToAdd, const glm::vec3& relativePosition, float distance, bool isEcho) {
     float gain = 1.0f;
 
     // injector: apply attenuation
@@ -536,23 +533,13 @@ float computeGain(const AudioMixerClientData& listenerNodeData, const AvatarAudi
             break;
         }
     }
+    // translate the zone setting to gain per log2(distance)
+    float g = glm::clamp(1.0f - attenuationPerDoublingInDistance, EPSILON, 1.0f);
 
-    // distance attenuation
-    const float ATTENUATION_START_DISTANCE = 1.0f;
-    float distance = glm::length(relativePosition);
-    assert(ATTENUATION_START_DISTANCE > EPSILON);
-    if (distance >= ATTENUATION_START_DISTANCE) {
-
-        // translate the zone setting to gain per log2(distance)
-        float g = 1.0f - attenuationPerDoublingInDistance;
-        g = glm::clamp(g, EPSILON, 1.0f);
-
-        // calculate the distance coefficient using the distance to this node
-        float distanceCoefficient = fastExp2f(fastLog2f(g) * fastLog2f(distance/ATTENUATION_START_DISTANCE));
-
-        // multiply the current attenuation coefficient by the distance coefficient
-        gain *= distanceCoefficient;
-    }
+    // calculate the attenuation using the distance to this node
+    // reference attenuation of 0dB at distance = 1.0m
+    gain *= fastExp2f(fastLog2f(g) * fastLog2f(std::max(distance, HRTF_NEARFIELD_MIN)));
+    gain = std::min(gain, 1.0f / HRTF_NEARFIELD_MIN);
 
     return gain;
 }

assignment-client/src/entities/EntityServer.cpp

@@ -116,7 +116,6 @@ void EntityServer::beforeRun() {
 void EntityServer::entityCreated(const EntityItem& newEntity, const SharedNodePointer& senderNode) {
 }
 
-
 // EntityServer will use the "special packets" to send list of recently deleted entities
 bool EntityServer::hasSpecialPacketsToSend(const SharedNodePointer& node) {
     bool shouldSendDeletedEntities = false;
@@ -277,7 +276,6 @@ int EntityServer::sendSpecialPackets(const SharedNodePointer& node, OctreeQueryN
     return totalBytes;
 }
 
-
 void EntityServer::pruneDeletedEntities() {
     EntityTreePointer tree = std::static_pointer_cast<EntityTree>(_tree);
     if (tree->hasAnyDeletedEntities()) {

assignment-client/src/entities/EntityServer.h

@@ -30,7 +30,6 @@ struct ViewerSendingStats {
 class SimpleEntitySimulation;
 using SimpleEntitySimulationPointer = std::shared_ptr<SimpleEntitySimulation>;
 
-
 class EntityServer : public OctreeServer, public NewlyCreatedEntityHook {
     Q_OBJECT
 public:
@@ -38,7 +37,7 @@ public:
     ~EntityServer();
 
     // Subclasses must implement these methods
-    virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override ;
+    virtual std::unique_ptr<OctreeQueryNode> createOctreeQueryNode() override;
     virtual char getMyNodeType() const override { return NodeType::EntityServer; }
     virtual PacketType getMyQueryMessageType() const override { return PacketType::EntityQuery; }
     virtual const char* getMyServerName() const override { return MODEL_SERVER_NAME; }
@@ -82,12 +81,12 @@ private:
     QReadWriteLock _viewerSendingStatsLock;
     QMap<QUuid, QMap<QUuid, ViewerSendingStats>> _viewerSendingStats;
 
-    static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000; // 45m
-    static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000; // 1h
-    int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 45m
-    int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS; // 1h
+    static const int DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 45 * 60 * 1000;                    // 45m
+    static const int DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = 60 * 60 * 1000;                    // 1h
+    int _MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MINIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS;  // 45m
+    int _MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS = DEFAULT_MAXIMUM_DYNAMIC_DOMAIN_VERIFICATION_TIMER_MS;  // 1h
     QTimer _dynamicDomainVerificationTimer;
     void startDynamicDomainVerification();
 };
 
-#endif // hifi_EntityServer_h
+#endif  // hifi_EntityServer_h

assignment-client/src/main.cpp

@@ -9,22 +9,13 @@
 //  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
 //
 
-#include <LogHandler.h>
+#include <BuildInfo.h>
 #include <SharedUtil.h>
 
 #include "AssignmentClientApp.h"
-#include <BuildInfo.h>
 
 int main(int argc, char* argv[]) {
-    disableQtBearerPoll(); // Fixes wifi ping spikes
-
-    QCoreApplication::setApplicationName(BuildInfo::ASSIGNMENT_CLIENT_NAME);
-    QCoreApplication::setOrganizationName(BuildInfo::MODIFIED_ORGANIZATION);
-    QCoreApplication::setOrganizationDomain(BuildInfo::ORGANIZATION_DOMAIN);
-    QCoreApplication::setApplicationVersion(BuildInfo::VERSION);
-
-    qInstallMessageHandler(LogHandler::verboseMessageHandler);
-    qInfo() << "Starting.";
+    setupHifiApplication(BuildInfo::ASSIGNMENT_CLIENT_NAME);
 
     AssignmentClientApp app(argc, argv);
     

assignment-client/src/octree/OctreeServer.cpp

@@ -33,6 +33,10 @@
 #include <PathUtils.h>
 #include <QtCore/QDir>
 
+#include <OctreeDataUtils.h>
+
+Q_LOGGING_CATEGORY(octree_server, "hifi.octree-server")
+
 int OctreeServer::_clientCount = 0;
 const int MOVING_AVERAGE_SAMPLE_COUNTS = 1000;
 
@@ -84,6 +88,8 @@ int OctreeServer::_longProcessWait = 0;
 int OctreeServer::_shortProcessWait = 0;
 int OctreeServer::_noProcessWait = 0;
 
+static const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
+
 
 void OctreeServer::resetSendingStats() {
     _averageLoopTime.reset();
@@ -202,7 +208,6 @@ void OctreeServer::trackPacketSendingTime(float time) {
     }
 }
 
-
 void OctreeServer::trackProcessWaitTime(float time) {
     const float MAX_SHORT_TIME = 10.0f;
     const float MAX_LONG_TIME = 100.0f;
@@ -283,8 +288,6 @@ void OctreeServer::initHTTPManager(int port) {
     _httpManager = new HTTPManager(QHostAddress::AnyIPv4, port, documentRoot, this, this);
 }
 
-const QString PERSIST_FILE_DOWNLOAD_PATH = "/models.json.gz";
-
 bool OctreeServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler) {
 
 #ifdef FORCE_CRASH
@@ -922,87 +925,6 @@ void OctreeServer::handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> me
     }
 }
 
-void OctreeServer::handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message) {
-    if (!_isFinished && !_isShuttingDown) {
-        // these messages are only allowed to come from the domain server, so make sure that is the case
-        auto nodeList = DependencyManager::get<NodeList>();
-        if (message->getSenderSockAddr() == nodeList->getDomainHandler().getSockAddr()) {
-            // it's far cleaner to load up the new content upon server startup
-            // so here we just store a special file at our persist path
-            // and then force a stop of the server so that it can pick it up when it relaunches
-            if (!_persistAbsoluteFilePath.isEmpty()) {
-                replaceContentFromMessageData(message->getMessage());
-            } else {
-                qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
-            }
-        } else {
-            qDebug() << "Received an octree file replacement that was not from our domain server - refusing to process";
-        }
-    }
-}
-
-// Message->getMessage() contains a QByteArray representation of the URL to download from
-void OctreeServer::handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message) {
-    qInfo() << "Received request to replace content from a url";
-    if (!_isFinished && !_isShuttingDown) {
-        // This call comes from Interface, so we skip our domain server check
-        // but confirm that we have permissions to replace content sets
-        if (DependencyManager::get<NodeList>()->getThisNodeCanReplaceContent()) {
-            if (!_persistAbsoluteFilePath.isEmpty()) {
-                // Convert message data into our URL
-                QString url(message->getMessage());
-                QUrl modelsURL = QUrl(url, QUrl::StrictMode);
-                QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
-                QNetworkRequest request(modelsURL);
-                QNetworkReply* reply = networkAccessManager.get(request);
-                connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
-                    QNetworkReply::NetworkError networkError = reply->error();
-                    if (networkError == QNetworkReply::NoError) {
-                        QByteArray contents = reply->readAll();
-                        replaceContentFromMessageData(contents);
-                    } else {
-                        qDebug() << "Error downloading JSON from specified file";
-                    }
-                });
-            } else {
-                qDebug() << "Cannot perform octree file replacement since current persist file path is not yet known";
-            }
-        }
-    }
-}
-
-void OctreeServer::replaceContentFromMessageData(QByteArray content) {
-    //Assume we have compressed data
-    auto compressedOctree = content;
-    QByteArray jsonOctree;
-
-    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
-    if (!wasCompressed) {
-        // the source was not compressed, assume we were sent regular JSON data
-        jsonOctree = compressedOctree;
-    }
-    // check the JSON data to verify it is an object
-    if (QJsonDocument::fromJson(jsonOctree).isObject()) {
-        if (!wasCompressed) {
-            // source was not compressed, we compress it before we write it locally
-            gzip(jsonOctree, compressedOctree);
-        }
-        // write the compressed octree data to a special file
-        auto replacementFilePath = _persistAbsoluteFilePath.append(OctreePersistThread::REPLACEMENT_FILE_EXTENSION);
-        QFile replacementFile(replacementFilePath);
-        if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
-            // we've now written our replacement file, time to take the server down so it can
-            // process it when it comes back up
-            qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
-            setFinished(true);
-        } else {
-            qWarning() << "Could not write replacement octree data to file - refusing to process";
-        }
-    } else {
-        qDebug() << "Received replacement octree file that is invalid - refusing to process";
-    }
-}
-
 bool OctreeServer::readOptionBool(const QString& optionName, const QJsonObject& settingsSectionObject, bool& result) {
     result = false; // assume it doesn't exist
     bool optionAvailable = false;
@@ -1119,7 +1041,18 @@ void OctreeServer::readConfiguration() {
             _persistFilePath = getMyDefaultPersistFilename();
         }
 
+        QDir persistPath { _persistFilePath };
+
+        if (persistPath.isRelative()) {
+            // if the domain settings passed us a relative path, make an absolute path that is relative to the
+            // default data directory
+            _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
+        } else {
+            _persistAbsoluteFilePath = persistPath.absolutePath();
+        }
+
         qDebug() << "persistFilePath=" << _persistFilePath;
+        qDebug() << "persisAbsoluteFilePath=" << _persistAbsoluteFilePath;
 
         _persistAsFileType = "json.gz";
 
@@ -1200,20 +1133,94 @@ void OctreeServer::run() {
 }
 
 void OctreeServer::domainSettingsRequestComplete() {
+    if (_state != OctreeServerState::WaitingForDomainSettings) {
+        qCWarning(octree_server) << "Received domain settings after they have already been received";
+        return;
+    }
+
+    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
+    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
+    packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
+
+    packetReceiver.registerListener(PacketType::OctreeDataFileReply, this, "handleOctreeDataFileReply");
+
+    qDebug(octree_server) << "Received domain settings";
+
+    readConfiguration();
+
+    _state = OctreeServerState::WaitingForOctreeDataNegotation;
+
+    auto nodeList = DependencyManager::get<NodeList>();
+    const DomainHandler& domainHandler = nodeList->getDomainHandler();
+
+    auto packet = NLPacket::create(PacketType::OctreeDataFileRequest, -1, true, false);
+
+    OctreeUtils::RawOctreeData data;
+    qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
+    if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
+        qCDebug(octree_server) << "Current octree data: ID(" << data.id << ") DataVersion(" << data.version << ")";
+        packet->writePrimitive(true);
+        auto id = data.id.toRfc4122();
+        packet->write(id);
+        packet->writePrimitive(data.version);
+    } else {
+        qCWarning(octree_server) << "No octree data found";
+        packet->writePrimitive(false);
+    }
+
+    qCDebug(octree_server) << "Sending request for octree data to DS";
+    nodeList->sendPacket(std::move(packet), domainHandler.getSockAddr());
+} 
+
+void OctreeServer::handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message) {
+    if (_state != OctreeServerState::WaitingForOctreeDataNegotation) {
+        qCWarning(octree_server) << "Server received ocree data file reply but is not currently negotiating.";
+        return;
+    }
+
+    bool includesNewData;
+    message->readPrimitive(&includesNewData);
+    QByteArray replaceData;
+    if (includesNewData) {
+        replaceData = message->readAll();
+        qDebug() << "Got reply to octree data file request, new data sent";
+    } else {
+        qDebug() << "Got reply to octree data file request, current entity data is sufficient";
+        
+        OctreeUtils::RawEntityData data;
+        qCDebug(octree_server) << "Reading octree data from" << _persistAbsoluteFilePath;
+        if (data.readOctreeDataInfoFromFile(_persistAbsoluteFilePath)) {
+            if (data.id.isNull()) {
+                qCDebug(octree_server) << "Current octree data has a null id, updating";
+                data.resetIdAndVersion();
+
+                QFile file(_persistAbsoluteFilePath);
+                if (file.open(QIODevice::WriteOnly)) {
+                    auto entityData = data.toGzippedByteArray();
+                    file.write(entityData);
+                    file.close();
+                } else {
+                    qCDebug(octree_server) << "Failed to update octree data";
+                }
+            }
+        }
+    }
+
+    _state = OctreeServerState::Running;
+    beginRunning(replaceData);
+}
+
+void OctreeServer::beginRunning(QByteArray replaceData) {
+    if (_state != OctreeServerState::Running) {
+        qCWarning(octree_server) << "Server is not running";
+        return;
+    }
 
     auto nodeList = DependencyManager::get<NodeList>();
 
     // we need to ask the DS about agents so we can ping/reply with them
     nodeList->addSetOfNodeTypesToNodeInterestSet({ NodeType::Agent, NodeType::EntityScriptServer });
 
-    auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
-    packetReceiver.registerListener(getMyQueryMessageType(), this, "handleOctreeQueryPacket");
-    packetReceiver.registerListener(PacketType::OctreeDataNack, this, "handleOctreeDataNackPacket");
-    packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacement");
-    packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURL");
-
-    readConfiguration();
-
     beforeRun(); // after payload has been processed
 
     connect(nodeList.data(), SIGNAL(nodeAdded(SharedNodePointer)), SLOT(nodeAdded(SharedNodePointer)));
@@ -1233,17 +1240,6 @@ void OctreeServer::domainSettingsRequestComplete() {
 
     // if we want Persistence, set up the local file and persist thread
     if (_wantPersist) {
-        // If persist filename does not exist, let's see if there is one beside the application binary
-        // If there is, let's copy it over to our target persist directory
-        QDir persistPath { _persistFilePath };
-        _persistAbsoluteFilePath = persistPath.absolutePath();
-
-        if (persistPath.isRelative()) {
-            // if the domain settings passed us a relative path, make an absolute path that is relative to the
-            // default data directory
-            _persistAbsoluteFilePath = QDir(PathUtils::getAppDataFilePath("entities/")).absoluteFilePath(_persistFilePath);
-        }
-
         static const QString ENTITY_PERSIST_EXTENSION = ".json.gz";
 
         // force the persist file to end with .json.gz
@@ -1328,7 +1324,7 @@ void OctreeServer::domainSettingsRequestComplete() {
 
         // now set up PersistThread
         _persistThread = new OctreePersistThread(_tree, _persistAbsoluteFilePath, _backupDirectoryPath, _persistInterval,
-                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType);
+                                                 _wantBackup, _settings, _debugTimestampNow, _persistAsFileType, replaceData);
         _persistThread->initialize(true);
     }
 

assignment-client/src/octree/OctreeServer.h

@@ -27,8 +27,18 @@
 #include "OctreeServerConsts.h"
 #include "OctreeInboundPacketProcessor.h"
 
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(octree_server)
+
 const int DEFAULT_PACKETS_PER_INTERVAL = 2000; // some 120,000 packets per second total
 
+enum class OctreeServerState {
+    WaitingForDomainSettings,
+    WaitingForOctreeDataNegotation,
+    Running
+};
+
 /// Handles assignments of type OctreeServer - sending octrees to various clients.
 class OctreeServer : public ThreadedAssignment, public HTTPRequestHandler {
     Q_OBJECT
@@ -36,6 +46,8 @@ public:
     OctreeServer(ReceivedMessage& message);
     ~OctreeServer();
 
+    OctreeServerState _state { OctreeServerState::WaitingForDomainSettings };
+
     /// allows setting of run arguments
     void setArguments(int argc, char** argv);
 
@@ -137,8 +149,7 @@ private slots:
     void domainSettingsRequestComplete();
     void handleOctreeQueryPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
     void handleOctreeDataNackPacket(QSharedPointer<ReceivedMessage> message, SharedNodePointer senderNode);
-    void handleOctreeFileReplacement(QSharedPointer<ReceivedMessage> message);
-    void handleOctreeFileReplacementFromURL(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeDataFileReply(QSharedPointer<ReceivedMessage> message);
     void removeSendThread();
 
 protected:
@@ -159,12 +170,12 @@ protected:
     QString getFileLoadTime();
     QString getConfiguration();
     QString getStatusLink();
+
+    void beginRunning(QByteArray replaceData);
     
     UniqueSendThread createSendThread(const SharedNodePointer& node);
     virtual UniqueSendThread newSendThread(const SharedNodePointer& node);
 
-    void replaceContentFromMessageData(QByteArray content);
-
     int _argc;
     const char** _argv;
     char** _parsedArgV;

assignment-client/src/scripts/EntityScriptServer.cpp

@@ -178,7 +178,7 @@ void EntityScriptServer::updateEntityPPS() {
     int numRunningScripts = _entitiesScriptEngine->getNumRunningEntityScripts();
     int pps;
     if (std::numeric_limits<int>::max() / _entityPPSPerScript < numRunningScripts) {
-        qWarning() << QString("Integer multiplaction would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
+        qWarning() << QString("Integer multiplication would overflow, clamping to maxint: %1 * %2").arg(numRunningScripts).arg(_entityPPSPerScript);
         pps = std::numeric_limits<int>::max();
         pps = std::min(_maxEntityPPS, pps);
     } else {

domain-server/CMakeLists.txt

@@ -24,7 +24,19 @@ symlink_or_copy_directory_beside_target(${_SHOULD_SYMLINK_RESOURCES} "${CMAKE_CU
 # link the shared hifi libraries
 include_hifi_library_headers(gpu)
 include_hifi_library_headers(graphics)
-link_hifi_libraries(embedded-webserver networking shared avatars)
+link_hifi_libraries(embedded-webserver networking shared avatars octree)
+
+target_zlib()
+
+add_dependency_external_projects(quazip)
+
+find_package(QuaZip REQUIRED)
+target_include_directories(${TARGET_NAME} SYSTEM PUBLIC ${QUAZIP_INCLUDE_DIRS})
+target_link_libraries(${TARGET_NAME} ${QUAZIP_LIBRARIES})
+
+if (WIN32)
+  add_paths_to_fixup_libs(${QUAZIP_DLL_PATH})
+endif ()
 
 # find OpenSSL
 find_package(OpenSSL REQUIRED)

domain-server/resources/describe-settings.json

@@ -1,5 +1,5 @@
 {
-  "version": 2.1,
+  "version": 2.2,
   "settings": [
     {
       "name": "metaverse",
@@ -306,7 +306,37 @@
             }
           ],
           "non-deletable-row-key": "permissions_id",
-          "non-deletable-row-values": [ "localhost", "anonymous", "logged-in" ]
+          "non-deletable-row-values": [ "localhost", "anonymous", "logged-in" ],
+          "default": [
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "anonymous"
+            },
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "friends"
+            },
+            {
+                "id_can_adjust_locks": true,
+                "id_can_connect": true,
+                "id_can_connect_past_max_capacity": true,
+                "id_can_kick": true,
+                "id_can_replace_content": true,
+                "id_can_rez": true,
+                "id_can_rez_certified": true,
+                "id_can_rez_tmp": true,
+                "id_can_rez_tmp_certified": true,
+                "id_can_write_to_asset_server": true,
+                "permissions_id": "localhost"
+            },
+            {
+                "id_can_connect": true,
+                "id_can_rez_tmp_certified": true,
+                "permissions_id": "logged-in"
+            }
+          ]
         },
         {
           "name": "group_permissions",
@@ -1321,73 +1351,6 @@
           "default": "30000",
           "advanced": true
         },
-        {
-          "name": "backups",
-          "type": "table",
-          "label": "Backup Rules",
-          "help": "In this table you can define a set of rules for how frequently to backup copies of your entites content file.",
-          "numbered": false,
-          "can_add_new_rows": true,
-          "default": [
-            {
-              "Name": "Half Hourly Rolling",
-              "backupInterval": 1800,
-              "format": ".backup.halfhourly.%N",
-              "maxBackupVersions": 5
-            },
-            {
-              "Name": "Daily Rolling",
-              "backupInterval": 86400,
-              "format": ".backup.daily.%N",
-              "maxBackupVersions": 7
-            },
-            {
-              "Name": "Weekly Rolling",
-              "backupInterval": 604800,
-              "format": ".backup.weekly.%N",
-              "maxBackupVersions": 4
-            },
-            {
-              "Name": "Thirty Day Rolling",
-              "backupInterval": 2592000,
-              "format": ".backup.thirtyday.%N",
-              "maxBackupVersions": 12
-            }
-          ],
-          "columns": [
-            {
-              "name": "Name",
-              "label": "Name",
-              "can_set": true,
-              "placeholder": "Example",
-              "default": "Example"
-            },
-            {
-              "name": "format",
-              "label": "Rule Format",
-              "can_set": true,
-              "help": "Format used to create the extension for the backup of your persisted entities. Use a format with %N to get rolling. Or use date formatting like %Y-%m-%d.%H:%M:%S.%z",
-              "placeholder": ".backup.example.%N",
-              "default": ".backup.example.%N"
-            },
-            {
-              "name": "backupInterval",
-              "label": "Backup Interval in Seconds",
-              "help": "Interval between backup checks in seconds.",
-              "placeholder": 1800,
-              "default": 1800,
-              "can_set": true
-            },
-            {
-              "name": "maxBackupVersions",
-              "label": "Max Rolled Backup Versions",
-              "help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?",
-              "placeholder": 5,
-              "default": 5,
-              "can_set": true
-            }
-          ]
-        },
         {
           "name": "NoPersist",
           "type": "checkbox",
@@ -1649,6 +1612,67 @@
         }
       ]
     },
+    {
+      "name": "automatic_content_archives",
+      "label": "Automatic Content Archives",
+      "settings": [
+        {
+          "name": "backup_rules",
+          "type": "table",
+          "label": "Rolling Backup Rules",
+          "help": "Define how frequently to create automatic content archives",
+          "numbered": false,
+          "can_add_new_rows": true,
+          "default": [
+            {
+              "Name": "Half Hourly Rolling",
+              "backupInterval": 1800,
+              "maxBackupVersions": 5
+            },
+            {
+              "Name": "Daily Rolling",
+              "backupInterval": 86400,
+              "maxBackupVersions": 7
+            },
+            {
+              "Name": "Weekly Rolling",
+              "backupInterval": 604800,
+              "maxBackupVersions": 4
+            },
+            {
+              "Name": "Thirty Day Rolling",
+              "backupInterval": 2592000,
+              "maxBackupVersions": 12
+            }
+          ],
+          "columns": [
+            {
+              "name": "Name",
+              "label": "Name",
+              "can_set": true,
+              "placeholder": "Example",
+              "default": "Example"
+            },
+            {
+              "name": "backupInterval",
+              "label": "Backup Interval in Seconds",
+              "help": "Interval between backup checks in seconds.",
+              "placeholder": 1800,
+              "default": 1800,
+              "can_set": true
+            },
+            {
+              "name": "maxBackupVersions",
+              "label": "Max Rolled Backup Versions",
+              "help": "If your backup extension format uses 'rolling', how many versions do you want us to keep?",
+              "placeholder": 5,
+              "default": 5,
+              "can_set": true
+            }
+          ]
+        }
+      ]
+    },
     {
       "name": "wizard",
       "label": "Setup Wizard",

domain-server/resources/web/base-settings-scripts.html

@@ -3,5 +3,4 @@
 <script src='/js/bootbox.min.js'></script>
 <script src='/js/form2js.min.js'></script>
 <script src='/js/bootstrap-switch.min.js'></script>
-<script src='/js/shared.js'></script>
 <script src='/js/base-settings.js'></script>

domain-server/resources/web/content/index.shtml

@@ -14,6 +14,8 @@
 
 <!--#include virtual="base-settings-scripts.html"-->
 
+<script src="js/moment-locale.min.js"></script>
+<script src="js/bootstrap-sortable.min.js"></script>
 <script src="js/content.js"></script>
 
 <!--#include virtual="page-end.html"-->

domain-server/resources/web/content/js/content.js

@@ -1,37 +1,437 @@
 $(document).ready(function(){
 
-  Settings.afterReloadActions = function() {};
+  var RESTORE_SETTINGS_UPLOAD_ID = 'restore-settings-button';
+  var RESTORE_SETTINGS_FILE_ID = 'restore-settings-file';
+  var UPLOAD_CONTENT_ALLOWED_DIV_ID = 'upload-content-allowed';
+  var UPLOAD_CONTENT_RECOVERING_DIV_ID = 'upload-content-recovering';
 
-  var frm = $('#upload-form');
-  frm.submit(function (ev) {
-    $.ajax({
-      type: frm.attr('method'),
-      url: frm.attr('action'),
-      data: new FormData($(this)[0]),
-      cache: false,
-      contentType: false,
-      processData: false,
-      success: function (data) {
-        swal({
-          title: 'Uploaded',
-          type: 'success',
-          text: 'Your Entity Server is restarting to replace its local content with the uploaded file.',
-          confirmButtonText: 'OK'
-        })
-      },
-      error: function (data) {
-        swal({
-          title: '',
-          type: 'error',
-          text: 'Your entities file could not be transferred to the Entity Server.</br>Verify that the file is a <i>.json</i> or <i>.json.gz</i> entities file and try again.',
-          html: true,
-          confirmButtonText: 'OK',
+  var isRestoring = false;
+
+  function progressBarHTML(extraClass, label) {
+    var html = "<div class='progress'>";
+    html += "<div class='" + extraClass + " progress-bar progress-bar-success progress-bar-striped active' role='progressbar' aria-valuemin='0' aria-valuemax='100'>";
+    html += label + "<span class='sr-only'></span></div></div>";
+    return html;
+  }
+
+  function setupBackupUpload() {
+    // construct the HTML needed for the settings backup panel
+    var html = "<div class='form-group'><div id='" + UPLOAD_CONTENT_ALLOWED_DIV_ID + "'>";
+
+    html += "<span class='help-block'>Upload a content archive (.zip) or entity file (.json, .json.gz) to replace the content of this domain.";
+    html += "<br/>Note: Your domain content will be replaced by the content you upload, but the existing backup files of your domain's content will not immediately be changed.</span>";
+
+    html += "<input id='restore-settings-file' name='restore-settings' type='file'>";
+    html += "<button type='button' id='" + RESTORE_SETTINGS_UPLOAD_ID + "' disabled='true' class='btn btn-primary'>Upload Content</button>";
+
+    html += "</div><div id='" + UPLOAD_CONTENT_RECOVERING_DIV_ID + "'>";
+    html += "<span class='help-block'>Restore in progress</span>";
+    html += progressBarHTML('recovery', 'Restoring');
+    html += "</div></div>";
+
+    $('#' + Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID + ' .panel-body').html(html);
+  }
+
+  // handle content archive or entity file upload
+
+  // when the selected file is changed, enable the button if there's a selected file
+  $('body').on('change', '#' + RESTORE_SETTINGS_FILE_ID, function() {
+    $('#' + RESTORE_SETTINGS_UPLOAD_ID).attr('disabled', $(this).val().length == 0);
+  });
+
+  // when the upload button is clicked, send the file to the DS
+  // and reload the page if restore was successful or
+  // show an error if not
+  $('body').on('click', '#' + RESTORE_SETTINGS_UPLOAD_ID, function(e){
+    e.preventDefault();
+
+    swalAreYouSure(
+      "Your domain content will be replaced by the uploaded content archive or entity file",
+      "Restore content",
+      function() {
+        var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
+
+        var fileFormData = new FormData();
+        fileFormData.append('restore-file', files[0]);
+
+        showSpinnerAlert("Uploading content to restore");
+
+        $.ajax({
+          url: '/content/upload',
+          type: 'POST',
+          cache: false,
+          processData: false,
+          contentType: false,
+          data: fileFormData
+        }).done(function(data, textStatus, jqXHR) {
+          isRestoring = true;
+
+          // immediately reload backup information since one should be restoring now
+          reloadBackupInformation();
+
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain content.\n"
+            + "Please ensure that the content archive or entity file is valid and try again."
+          );
         });
       }
-    });
-
-    ev.preventDefault();
-
-    showSpinnerAlert("Uploading Entities File");
+    );
   });
+
+  var GENERATE_ARCHIVE_BUTTON_ID = 'generate-archive-button';
+  var CONTENT_ARCHIVES_NORMAL_ID = 'content-archives-success';
+  var CONTENT_ARCHIVES_ERROR_ID = 'content-archives-error';
+  var AUTOMATIC_ARCHIVES_TABLE_ID = 'automatic-archives-table';
+  var AUTOMATIC_ARCHIVES_TBODY_ID = 'automatic-archives-tbody';
+  var MANUAL_ARCHIVES_TABLE_ID = 'manual-archives-table';
+  var MANUAL_ARCHIVES_TBODY_ID = 'manual-archives-tbody';
+  var AUTO_ARCHIVES_SETTINGS_LINK_ID = 'auto-archives-settings-link';
+  var ACTION_MENU_CLASS = 'action-menu';
+
+  var automaticBackups = [];
+  var manualBackups = [];
+
+  function setupContentArchives() {
+    // construct the HTML needed for the content archives panel
+    var html = "<div id='" + CONTENT_ARCHIVES_NORMAL_ID + "'><div class='form-group'>";
+    html += "<label class='control-label'>Automatic Content Archives</label>";
+    html += "<span class='help-block'>Your domain server makes regular archives of the content in your domain. In the list below, you can see and download all of your backups of domain content and content settings."
+    html += "<a href='/settings/#automatic_content_archives' id='" + AUTO_ARCHIVES_SETTINGS_LINK_ID + "'>Click here to manage automatic content archive intervals.</a></span>";
+    html += "</div>";
+    html += "<table class='table sortable' id='" + AUTOMATIC_ARCHIVES_TABLE_ID + "'>";
+
+    var backups_table_head = "<thead><tr class='gray-tr'><th>Archive Name</th><th data-defaultsort='desc'>Archive Date</th>"
+      + "<th data-defaultsort='disabled'></th><th class='" + ACTION_MENU_CLASS + "' data-defaultsort='disabled'>Actions</th>"
+      + "</tr></thead>";
+
+    html += backups_table_head;
+    html += "<tbody id='" + AUTOMATIC_ARCHIVES_TBODY_ID + "'></tbody></table>";
+    html += "<div class='form-group'>";
+    html += "<label class='control-label'>Manual Content Archives</label>";
+    html += "<span class='help-block'>You can generate and download an archive of your domain content right now. You can also download, delete and restore any archive listed.</span>";
+    html += "<button type='button' id='" + GENERATE_ARCHIVE_BUTTON_ID + "' class='btn btn-primary'>Generate New Archive</button>";
+    html += "</div>";
+    html += "<table class='table sortable' id='" + MANUAL_ARCHIVES_TABLE_ID + "'>";
+    html += backups_table_head;
+    html += "<tbody id='" + MANUAL_ARCHIVES_TBODY_ID + "'></tbody></table></div>";
+
+    html += "<div class='form-group' id='" + CONTENT_ARCHIVES_ERROR_ID + "' style='display:none;'>"
+      + "<span class='help-block'>There was a problem loading your list of automatic and manual content archives. "
+      + "Please reload the page to try again.</span></div>";
+
+    // put the base HTML in the content archives panel
+    $('#' + Settings.CONTENT_ARCHIVES_PANEL_ID + ' .panel-body').html(html);
+  }
+
+  var BACKUP_RESTORE_LINK_CLASS = 'restore-backup';
+  var BACKUP_DOWNLOAD_LINK_CLASS = 'download-backup';
+  var BACKUP_DELETE_LINK_CLASS = 'delete-backup';
+  var ACTIVE_BACKUP_ROW_CLASS = 'active-backup';
+  var CORRUPTED_ROW_CLASS = 'danger';
+
+  function reloadBackupInformation() {
+    // make a GET request to get backup information to populate the table
+    $.ajax({
+      url: '/api/backups',
+      cache: false
+    }).done(function(data) {
+
+      // split the returned data into manual and automatic manual backups
+      var splitBackups = _.partition(data.backups, function(value, index) {
+        return value.isManualBackup;
+      });
+
+      if (isRestoring && !data.status.isRecovering) {
+        // we were recovering and we finished - the DS is going to restart so show the restart modal
+        showRestartModal();
+        return;
+      }
+
+      isRestoring = data.status.isRecovering;
+
+      manualBackups = splitBackups[0];
+      automaticBackups = splitBackups[1];
+
+      // populate the backups tables with the backups
+      function createBackupTableRow(backup) {
+        return "<tr data-backup-id='" + backup.id + "' data-backup-name='" + backup.name + "'>"
+          + "<td data-value='" + backup.name.toLowerCase() + "'>" + backup.name + "</td><td data-value='" + backup.createdAtMillis + "'>"
+          + moment(backup.createdAtMillis).format('lll')
+          + "</td><td class='backup-status'></td><td class='" + ACTION_MENU_CLASS + "'>"
+          + "<div class='dropdown'><div class='dropdown-toggle' data-toggle='dropdown' aria-expanded='false'><span class='glyphicon glyphicon-option-vertical'></span></div>"
+          + "<ul class='dropdown-menu dropdown-menu-right'>"
+          + "<li><a class='" + BACKUP_RESTORE_LINK_CLASS + "' href='#'>Restore from here</a></li><li class='divider'></li>"
+          + "<li><a class='" + BACKUP_DOWNLOAD_LINK_CLASS + "' href='/api/backups/" + backup.id + "'>Download</a></li><li class='divider'></li>"
+          + "<li><a class='" + BACKUP_DELETE_LINK_CLASS + "' href='#' target='_blank'>Delete</a></li></ul></div></td>";
+      }
+
+      function updateProgressBars($progressBar, value) {
+        $progressBar.attr('aria-valuenow', value).attr('style', 'width: ' + value + '%');
+        $progressBar.find('.sr-only').html(value + "% Complete");
+      }
+
+      // before we add any new rows and update existing ones
+      // remove our flag for active rows
+      $('.' + ACTIVE_BACKUP_ROW_CLASS).removeClass(ACTIVE_BACKUP_ROW_CLASS);
+
+      function updateOrAddTableRow(backup, tableBodyID) {
+        // check for a backup with this ID
+        var $backupRow = $("tr[data-backup-id='" + backup.id + "']");
+
+        if ($backupRow.length == 0) {
+          // create a new row and then add it to the table
+          $backupRow = $(createBackupTableRow(backup));
+          $('#' + tableBodyID).append($backupRow);
+        }
+
+        // update the row status column depending on if it is available or recovering
+        if (!backup.isAvailable) {
+          // add a progress bar to the status row for availability
+          $backupRow.find('td.backup-status').html(progressBarHTML('availability', 'Archiving'));
+
+          // set the value of the progress bar based on availability progress
+          updateProgressBars($backupRow.find('.progress-bar'), backup.availabilityProgress * 100);
+        } else if (backup.id == data.status.recoveringBackupId) {
+          // add a progress bar to the status row for recovery
+          $backupRow.find('td.backup-status').html(progressBarHTML('recovery', 'Restoring'));
+        } else if (backup.isCorrupted) {
+          // add text for corrupted status to row
+          $backupRow.find('td.backup-status').html('<span>Corrupted</span>');
+        } else {
+          // no special status for this row, use an empty status column
+          $backupRow.find('td.backup-status').html('');
+        }
+
+        // color the row red if it is corrupted
+        $backupRow.toggleClass(CORRUPTED_ROW_CLASS, backup.isCorrupted);
+
+        // disable restore if the backup is corrupted
+        $backupRow.find('a.' + BACKUP_RESTORE_LINK_CLASS).parent().toggleClass('disabled', backup.isCorrupted);
+
+        // toggle the dropdown menu depending on if the row is available
+        $backupRow.find('td.' + ACTION_MENU_CLASS + ' .dropdown').toggle(backup.isAvailable);
+
+        $backupRow.addClass(ACTIVE_BACKUP_ROW_CLASS);
+      }
+
+      if (automaticBackups.length > 0) {
+        for (var backupIndex in automaticBackups) {
+          updateOrAddTableRow(automaticBackups[backupIndex], AUTOMATIC_ARCHIVES_TBODY_ID);
+        }
+      }
+
+      if (manualBackups.length > 0) {
+        for (var backupIndex in manualBackups) {
+          updateOrAddTableRow(manualBackups[backupIndex], MANUAL_ARCHIVES_TBODY_ID);
+        }
+      }
+
+      // at this point, any rows that no longer have the ACTIVE_BACKUP_ROW_CLASS
+      // are deleted backups, so we remove them from the table
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID + ' tbody tr:not(.' + ACTIVE_BACKUP_ROW_CLASS + ')').remove();
+
+      // check if the restore action on all rows should be enabled or disabled
+      $('tr:not(.' + CORRUPTED_ROW_CLASS + ') .' + BACKUP_RESTORE_LINK_CLASS).parent().toggleClass('disabled', data.status.isRecovering);
+
+      // hide or show the manual content upload file and button depending on our recovering status
+      $('#' + UPLOAD_CONTENT_ALLOWED_DIV_ID).toggle(!data.status.isRecovering);
+      $('#' + UPLOAD_CONTENT_RECOVERING_DIV_ID).toggle(data.status.isRecovering);
+
+      // update the progress bars for current restore status
+      if (data.status.isRecovering) {
+        updateProgressBars($('.recovery.progress-bar'), data.status.recoveryProgress * 100);
+      }
+
+      // tell bootstrap sortable to update for the new rows
+      $.bootstrapSortable({ applyLast: true });
+
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID).toggle(true);
+      $('#' + CONTENT_ARCHIVES_ERROR_ID).toggle(false);
+
+    }).fail(function(){
+      // we've hit the very rare case where we couldn't load the list of backups from the domain server
+
+      // set our backups to empty
+      automaticBackups = [];
+      manualBackups = [];
+
+      // replace the content archives panel with a simple error message
+      // stating that the user should reload the page
+      $('#' + CONTENT_ARCHIVES_NORMAL_ID).toggle(false);
+      $('#' + CONTENT_ARCHIVES_ERROR_ID).toggle(true);
+
+    }).always(function(){
+      // toggle showing or hiding the tables depending on if they have entries
+      $('#' + AUTOMATIC_ARCHIVES_TABLE_ID).toggle(automaticBackups.length > 0);
+      $('#' + MANUAL_ARCHIVES_TABLE_ID).toggle(manualBackups.length > 0);
+    });
+  }
+
+  // handle click in table to restore a given content backup
+  $('body').on('click', '.' + BACKUP_RESTORE_LINK_CLASS, function(e) {
+    // stop the default behaviour
+    e.preventDefault();
+
+    // if this is a disabled link, don't proceed with the restore
+    if ($(this).parent().hasClass('disabled')) {
+      return false;
+    }
+
+    // grab the name of this backup so we can show it in alerts
+    var backupName = $(this).closest('tr').attr('data-backup-name');
+
+    // grab the ID of this backup in case we need to send a POST
+    var backupID = $(this).closest('tr').attr('data-backup-id');
+
+    // make sure the user knows what is about to happen
+    swalAreYouSure(
+      "Your domain content will be replaced by the content archive " + backupName,
+      "Restore content",
+      function() {
+        // show a spinner while we send off our request
+        showSpinnerAlert("Starting restore of " +  backupName);
+
+        // setup an AJAX POST to request content restore
+        $.post('/api/backups/recover/' + backupID).done(function(data, textStatus, jqXHR) {
+          isRestoring = true;
+
+          // immediately reload our backup information since one should be restoring now
+          reloadBackupInformation();
+
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain content.\n"
+            + "If the problem persists, the content archive may be corrupted."
+          );
+        });
+      }
+    )
+  });
+
+  // handle click in table to delete a given content backup
+  $('body').on('click', '.' + BACKUP_DELETE_LINK_CLASS, function(e){
+    // stop the default behaviour
+    e.preventDefault();
+
+    // grab the name of this backup so we can show it in alerts
+    var backupName = $(this).closest('tr').attr('data-backup-name');
+
+    // grab the ID of this backup in case we need to send the DELETE request
+    var backupID = $(this).closest('tr').attr('data-backup-id');
+
+    // make sure the user knows what is about to happen
+    swalAreYouSure(
+      "The content archive " + backupName + " will be deleted and will no longer be available for restore or download from this page.",
+      "Delete content archive",
+      function() {
+        // show a spinner while we send off our request
+        showSpinnerAlert("Deleting content archive " +  backupName);
+
+        // setup an AJAX DELETE to request content archive delete
+        $.ajax({
+          url: '/api/backups/' + backupID,
+          type: 'DELETE'
+        }).done(function(data, textStatus, jqXHR) {
+          swal.close();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was an unexpected error deleting the content archive"
+          );
+        }).always(function(){
+          // reload the list of content archives in case we deleted a backup
+          // or it's no longer an available backup for some other reason
+          reloadBackupInformation();
+        });
+      }
+    )
+  });
+
+  // handle click on automatic content archive settings link
+  $('body').on('click', '#' + AUTO_ARCHIVES_SETTINGS_LINK_ID, function(e) {
+    if (Settings.pendingChanges > 0) {
+      // don't follow the link right away, make sure the user knows they are about to leave
+      // the page and lose changes
+      e.preventDefault();
+
+      var settingsLink = $(this).attr('href');
+
+      swalAreYouSure(
+        "You have pending changes to content settings that have not been saved. They will be lost if you leave the page to manage automatic content archive intervals.",
+        "Proceed without Saving",
+        function() {
+          // user wants to drop their changes, switch pages
+          window.location =  settingsLink;
+        }
+      );
+    }
+  });
+
+  // handle click on manual archive creation button
+  $('body').on('click', '#' + GENERATE_ARCHIVE_BUTTON_ID, function(e) {
+    e.preventDefault();
+
+    // show a sweet alert to ask the user to provide a name for their content archive
+    swal({
+      title: "Generate a content archive",
+      type: "input",
+      text: "This will capture the state of all the content in your domain right now, which you can save as a backup and restore from later.",
+      confirmButtonText: "Generate Archive",
+      showCancelButton: true,
+      closeOnConfirm: false,
+      inputPlaceholder: 'Archive Name'
+    }, function(inputValue){
+      if (inputValue === false) {
+        return false;
+      }
+
+      if (inputValue === "") {
+        swal.showInputError("Please give the content archive a name.")
+        return false;
+      }
+
+      var MANUAL_ARCHIVE_NAME_REGEX = /^[a-zA-Z0-9\-_ ]+$/;
+      if (!MANUAL_ARCHIVE_NAME_REGEX.test(inputValue)) {
+        swal.showInputError("Valid characters include A-z, 0-9, ' ', '_', and '-'.");
+        return false;
+      }
+
+      // post the provided archive name to ask the server to kick off a manual backup
+      $.ajax({
+        type: 'POST',
+        url: '/api/backups',
+        data: {
+          'name': inputValue
+        }
+      }).done(function(data) {
+        // since we successfully setup a new content archive, reload the table of archives
+        // which should show that this archive is pending creation
+        swal.close();
+        reloadBackupInformation();
+      }).fail(function(jqXHR, textStatus, errorThrown) {
+        showErrorMessage(
+          "Error",
+          "There was an unexpected error creating the manual content archive"
+        )
+      });
+    });
+  });
+
+  Settings.extraGroupsAtIndex = Settings.extraContentGroupsAtIndex;
+
+  Settings.afterReloadActions = function() {
+    setupBackupUpload();
+    setupContentArchives();
+
+    // load the latest backups immediately
+    reloadBackupInformation();
+
+    // setup a timer to reload them every 5 seconds
+    setInterval(reloadBackupInformation, 5000);
+  };
 });

domain-server/resources/web/css/bootstrap-sortable.css

@@ -0,0 +1,110 @@
+/**
+ * adding sorting ability to HTML tables with Bootstrap styling
+ * @summary HTML tables sorting ability
+ * @version 2.0.0
+ * @requires tinysort, moment.js, jQuery
+ * @license MIT
+ * @author Matus Brlit (drvic10k)
+ * @copyright Matus Brlit (drvic10k), bootstrap-sortable contributors
+ */
+
+table.sortable span.sign {
+    display: block;
+    position: absolute;
+    top: 50%;
+    right: 5px;
+    font-size: 12px;
+    margin-top: -10px;
+    color: #bfbfc1;
+}
+
+table.sortable th:after {
+    display: block;
+    position: absolute;
+    top: 50%;
+    right: 5px;
+    font-size: 12px;
+    margin-top: -10px;
+    color: #bfbfc1;
+}
+
+table.sortable th.arrow:after {
+    content: '';
+}
+
+table.sortable span.arrow, span.reversed, th.arrow.down:after, th.reversedarrow.down:after, th.arrow.up:after, th.reversedarrow.up:after {
+    border-style: solid;
+    border-width: 5px;
+    font-size: 0;
+    border-color: #ccc transparent transparent transparent;
+    line-height: 0;
+    height: 0;
+    width: 0;
+    margin-top: -2px;
+}
+
+    table.sortable span.arrow.up, th.arrow.up:after {
+        border-color: transparent transparent #ccc transparent;
+        margin-top: -7px;
+    }
+
+table.sortable span.reversed, th.reversedarrow.down:after {
+    border-color: transparent transparent #ccc transparent;
+    margin-top: -7px;
+}
+
+    table.sortable span.reversed.up, th.reversedarrow.up:after {
+        border-color: #ccc transparent transparent transparent;
+        margin-top: -2px;
+    }
+
+table.sortable span.az:before, th.az.down:after {
+    content: "a .. z";
+}
+
+table.sortable span.az.up:before, th.az.up:after {
+    content: "z .. a";
+}
+
+table.sortable th.az.nosort:after, th.AZ.nosort:after, th._19.nosort:after, th.month.nosort:after {
+    content: "..";
+}
+
+table.sortable span.AZ:before, th.AZ.down:after {
+    content: "A .. Z";
+}
+
+table.sortable span.AZ.up:before, th.AZ.up:after {
+    content: "Z .. A";
+}
+
+table.sortable span._19:before, th._19.down:after {
+    content: "1 .. 9";
+}
+
+table.sortable span._19.up:before, th._19.up:after {
+    content: "9 .. 1";
+}
+
+table.sortable span.month:before, th.month.down:after {
+    content: "jan .. dec";
+}
+
+table.sortable span.month.up:before, th.month.up:after {
+    content: "dec .. jan";
+}
+
+table.sortable>thead th:not([data-defaultsort=disabled]) {
+    cursor: pointer;
+    position: relative;
+    top: 0;
+    left: 0;
+}
+
+table.sortable>thead th:hover:not([data-defaultsort=disabled]) {
+    background: #efefef;
+}
+
+table.sortable>thead th div.mozilla {
+    position: relative;
+}

domain-server/resources/web/css/style.css

@@ -355,21 +355,31 @@ table .headers + .headers td {
   }
 }
 
-ul.nav li.dropdown ul.dropdown-menu {
+ul.dropdown-menu {
   padding: 0px 0px;
 }
 
-ul.nav li.dropdown li a {
+ul.dropdown-menu li a {
   padding-top: 7px;
   padding-bottom: 7px;
 }
 
-ul.nav li.dropdown li a:hover {
+ul.dropdown-menu li a:hover {
   color: white;
   background-color: #337ab7;
 }
 
-ul.nav li.dropdown ul.dropdown-menu .divider {
+table ul.dropdown-menu li:first-child a:hover {
+  border-top-left-radius: 4px;
+  border-top-right-radius: 4px;
+}
+
+ul.dropdown-menu li:last-child a:hover {
+  border-bottom-left-radius: 4px;
+  border-bottom-right-radius: 4px;
+}
+
+ul.dropdown-menu .divider {
   margin: 0px 0;
 }
 
@@ -434,3 +444,42 @@ ul.nav li.dropdown ul.dropdown-menu .divider {
 .save-button-text {
   pointer-events: none;
 }
+
+#content_archives .panel-body {
+  padding: 0;
+}
+
+#content_archives .panel-body .form-group {
+  padding: 15px;
+}
+
+#content_archives .panel-body th, #content_archives .panel-body td {
+  padding: 8px 15px;
+}
+
+#content_archives table {
+  border-top: 1px solid #ddd;
+  border-bottom: 1px solid #ddd;
+}
+
+tr.gray-tr {
+  background-color: #f5f5f5;
+}
+
+table .action-menu {
+  text-align: right;
+  width: 90px;
+}
+
+.dropdown-toggle span.glyphicon-option-vertical {
+  font-size: 110%;
+  cursor: pointer;
+  border-radius: 50%;
+  background-color: #F5F5F5;
+  padding: 4px 4px 4px 6px;
+}
+
+.dropdown.open span.glyphicon-option-vertical {
+  background-color: #337AB7;
+  color: white;
+}

domain-server/resources/web/footer.html

@@ -1,4 +1,5 @@
 </div>
 <script src='/js/jquery-2.1.4.min.js'></script>
 <script src='/js/bootstrap.min.js'></script>
+<script src='/js/shared.js'></script>
 <script src='/js/domain-server.js'></script>

domain-server/resources/web/header.html

@@ -9,6 +9,7 @@
     <link href="/css/style.css" rel="stylesheet" media="screen">
     <link href="/css/sweetalert.css" rel="stylesheet" media="screen">
     <link href="/css/bootstrap-switch.min.css" rel="stylesheet" media="screen">
+    <link href="/css/bootstrap-sortable.css" rel="stylesheet" media="screen">
 
     <script src='/js/sweetalert.min.js'></script>
   </head>

domain-server/resources/web/js/base-settings.js

@@ -106,8 +106,12 @@ function reloadSettings(callback) {
   $.getJSON(Settings.endpoint, function(data){
     _.extend(data, viewHelpers);
 
-    for (var spliceIndex in Settings.extraGroups) {
-      data.descriptions.splice(spliceIndex, 0, Settings.extraGroups[spliceIndex]);
+    for (var spliceIndex in Settings.extraGroupsAtIndex) {
+      data.descriptions.splice(spliceIndex, 0, Settings.extraGroupsAtIndex[spliceIndex]);
+    }
+
+    for (var endGroupIndex in Settings.extraGroupsAtEnd) {
+      data.descriptions.push(Settings.extraGroupsAtEnd[endGroupIndex]);
     }
 
     $('#panels').html(Settings.panelsTemplate(data));
@@ -122,6 +126,8 @@ function reloadSettings(callback) {
 
     $('[data-toggle="tooltip"]').tooltip();
 
+    Settings.pendingChanges = 0;
+
     // call the callback now that settings are loaded
     callback(true);
   }).fail(function() {
@@ -257,7 +263,7 @@ $(document).ready(function(){
     }
   });
 
-  $('#' + Settings.FORM_ID).on('change keyup paste', '.' + Settings.TRIGGER_CHANGE_CLASS , function(e){
+  $('#' + Settings.FORM_ID).on('change input propertychange', '.' + Settings.TRIGGER_CHANGE_CLASS , function(e){
     // this input was changed, add the changed data attribute to it
     $(this).attr('data-changed', true);
 
@@ -676,11 +682,11 @@ function makeTableHiddenInputs(setting, initialValues, categoryValue) {
     } else {
       html +=
         "<td " + (col.hidden ? "style='display: none;'" : "") + " class='" + Settings.DATA_COL_CLASS + "' " +
-            "name='" + col.name + "'>" +
-          "<input type='text' style='display: none;' class='form-control' placeholder='" + (col.placeholder ? col.placeholder : "") + "' " +
-                 "value='" + (defaultValue || "") + "' data-default='" + (defaultValue || "") + "'" +
-                 (col.readonly ? " readonly" : "") + ">" +
-        "</td>";
+          "name='" + col.name + "'>" +
+          "<input type='text' style='display: none;' class='form-control " + Settings.TRIGGER_CHANGE_CLASS +
+          "' placeholder='" + (col.placeholder ? col.placeholder : "") + "' " +
+          "value='" + (defaultValue || "") + "' data-default='" + (defaultValue || "") + "'" +
+          (col.readonly ? " readonly" : "") + ">" + "</td>";
     }
   })
 
@@ -801,6 +807,8 @@ function badgeForDifferences(changedElement) {
     }
   });
 
+  Settings.pendingChanges = totalChanges;
+
   if (totalChanges == 0) {
     totalChanges = ""
   }
@@ -830,7 +838,7 @@ function addTableRow(row) {
     var keyInput = row.children(".key").children("input");
 
     // whenever the keyInput changes, re-badge for differences
-    keyInput.on('change keyup paste', function(e){
+    keyInput.on('change input propertychange', function(e){
       // update siblings in the row to have the correct name
       var currentKey = $(this).val();
 

domain-server/resources/web/js/domain-server.js

@@ -28,7 +28,7 @@ function settingsGroupAnchor(base, html_id) {
 }
 
 $(document).ready(function(){
-  var url = window.location;
+  var url = location.protocol + '//' + location.host+location.pathname;
 
   // Will only work if string in href matches with location
   $('ul.nav a[href="'+ url +'"]').parent().addClass('active');
@@ -39,22 +39,49 @@ $(document).ready(function(){
   }).parent().addClass('active');
 
   $('body').on('click', '#restart-server', function(e) {
-    swal( {
-      title: "Are you sure?",
-      text: "This will restart your domain server, causing your domain to be briefly offline.",
-      type: "warning",
-      html: true,
-      showCancelButton: true
-    }, function() {
-      $.get("/restart");
-      showRestartModal();
-    });
+    swalAreYouSure(
+      "This will restart your domain server, causing your domain to be briefly offline.",
+      "Restart",
+      function() {
+        swal.close();
+        $.get("/restart");
+        showRestartModal();
+      }
+    )
     return false;
   });
 
   var $contentDropdown = $('#content-settings-nav-dropdown');
   var $settingsDropdown = $('#domain-settings-nav-dropdown');
 
+  // define extra groups to add to setting panels, with their splice index
+  Settings.extraContentGroupsAtIndex = {
+    0: {
+      html_id: Settings.CONTENT_ARCHIVES_PANEL_ID,
+      label: 'Content Archives'
+    },
+    1: {
+      html_id: Settings.UPLOAD_CONTENT_BACKUP_PANEL_ID,
+      label: 'Upload Content'
+    }
+  };
+
+  Settings.extraContentGroupsAtEnd = [];
+
+  Settings.extraDomainGroupsAtIndex = {
+    1: {
+      html_id: 'places',
+      label: 'Places'
+    }
+  }
+
+  Settings.extraDomainGroupsAtEnd = [
+    {
+      html_id: 'settings_backup',
+      label: 'Settings Backup / Restore'
+    }
+  ]
+
   // for pages that have the settings dropdowns
   if ($contentDropdown.length && $settingsDropdown.length) {
     // make a JSON request to get the dropdown menus for content and settings
@@ -65,6 +92,15 @@ $(document).ready(function(){
         return "<li class='setting-group'><a href='" + settingsGroupAnchor(base, html_id) + "'>" + group.label + "<span class='badge'></span></a></li>";
       }
 
+      // add the dummy settings groups that get populated via JS
+      for (var spliceIndex in Settings.extraContentGroupsAtIndex) {
+        data.content_settings.splice(spliceIndex, 0, Settings.extraContentGroupsAtIndex[spliceIndex]);
+      }
+
+      for (var endIndex in Settings.extraContentGroupsAtEnd) {
+        data.content_settings.push(Settings.extraContentGroupsAtEnd[endIndex]);
+      }
+
       $.each(data.content_settings, function(index, group){
         if (index > 0) {
           $contentDropdown.append("<li role='separator' class='divider'></li>");
@@ -73,25 +109,22 @@ $(document).ready(function(){
         $contentDropdown.append(makeGroupDropdownElement(group, "/content/"));
       });
 
+      // add the dummy settings groups that get populated via JS
+      for (var spliceIndex in Settings.extraDomainGroupsAtIndex) {
+        data.domain_settings.splice(spliceIndex, 0, Settings.extraDomainGroupsAtIndex[spliceIndex]);
+      }
+
+      for (var endIndex in Settings.extraDomainGroupsAtEnd) {
+        data.domain_settings.push(Settings.extraDomainGroupsAtEnd[endIndex]);
+      }
+
       $.each(data.domain_settings, function(index, group){
         if (index > 0) {
           $settingsDropdown.append("<li role='separator' class='divider'></li>");
         }
 
         $settingsDropdown.append(makeGroupDropdownElement(group, "/settings/"));
-
-        // for domain settings, we add a dummy "Places" group that we fill
-        // via the API - add it to the dropdown menu in the right spot
-        // which is after "Metaverse / Networking"
-        if (group.name == "metaverse") {
-          $settingsDropdown.append("<li role='separator' class='divider'></li>");
-          $settingsDropdown.append(makeGroupDropdownElement({ html_id: 'places', label: 'Places' }, "/settings/"));
-        }
       });
-
-      // append a link for the "Settings Backup" panel
-      $settingsDropdown.append("<li role='separator' class='divider'></li>");
-      $settingsDropdown.append(makeGroupDropdownElement({ html_id: 'settings_backup', label: 'Settings Backup'}, "/settings"));
     });
   }
 });

domain-server/resources/web/js/shared.js

@@ -42,7 +42,9 @@ Object.assign(Settings, {
   ADD_PLACE_BTN_ID: 'add-place-btn',
   FORM_ID: 'settings-form',
   INVALID_ROW_CLASS: 'invalid-input',
-  DATA_ROW_INDEX: 'data-row-index'
+  DATA_ROW_INDEX: 'data-row-index',
+  CONTENT_ARCHIVES_PANEL_ID: 'content_archives',
+  UPLOAD_CONTENT_BACKUP_PANEL_ID: 'upload_content'
 });
 
 var URLs = {
@@ -96,6 +98,17 @@ var DOMAIN_ID_TYPE_TEMP = 1;
 var DOMAIN_ID_TYPE_FULL = 2;
 var DOMAIN_ID_TYPE_UNKNOWN = 3;
 
+function swalAreYouSure(text, confirmButtonText, callback) {
+  swal({
+    title: "Are you sure?",
+    text: text,
+    type: "warning",
+    showCancelButton: true,
+    confirmButtonText: confirmButtonText,
+    closeOnConfirm: false
+  }, callback);
+}
+
 function domainIDIsSet() {
   if (typeof Settings.data.values.metaverse !== 'undefined' &&
     typeof Settings.data.values.metaverse.id !== 'undefined') {
@@ -164,7 +177,7 @@ function getDomainFromAPI(callback) {
   if (callback === undefined) {
     callback = function() {};
   }
-  
+
   if (!domainIDIsSet()) {
     callback({ status: 'fail' });
     return null;

domain-server/resources/web/settings/js/settings.js

@@ -14,17 +14,8 @@ $(document).ready(function(){
       return b;
   })(window.location.search.substr(1).split('&'));
 
-  // define extra groups to add to description, with their splice index
-  Settings.extraGroups = {
-    1: {
-      html_id: 'places',
-      label: 'Places'
-    },
-    "-1": {
-      html_id: 'settings_backup',
-      label: 'Settings Backup'
-    }
-  }
+  Settings.extraGroupsAtEnd = Settings.extraDomainGroupsAtEnd;
+  Settings.extraGroupsAtIndex = Settings.extraDomainGroupsAtIndex;
 
   Settings.afterReloadActions = function() {
     // append the domain selection modal
@@ -103,20 +94,17 @@ $(document).ready(function(){
       var password = formJSON["security"]["http_password"];
 
       if ((password == sha256_digest("")) && (username == undefined || (username && username.length != 0))) {
-        swal({
-          title: "Are you sure?",
-          text: "You have entered a blank password with a non-blank username. Are you sure you want to require a blank password?",
-          type: "warning",
-          showCancelButton: true,
-          confirmButtonColor: "#5cb85c",
-          confirmButtonText: "Yes!",
-          closeOnConfirm: true
-        },
-        function () {
+        swalAreYouSure(
+          "You have entered a blank password with a non-blank username. Are you sure you want to require a blank password?",
+          "Use blank password",
+          function() {
+            swal.close();
+            
             formJSON["security"]["http_password"] = "";
 
             postSettings(formJSON);
-        });
+          }
+        );
 
         return;
       }
@@ -643,7 +631,6 @@ $(document).ready(function(){
     autoNetworkingEl.after(form);
   }
 
-
   function setupPlacesTable() {
     // create a dummy table using our view helper
     var placesTableSetting = {
@@ -1043,32 +1030,38 @@ $(document).ready(function(){
   $('body').on('click', '#' + RESTORE_SETTINGS_UPLOAD_ID, function(e){
     e.preventDefault();
 
-    var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
+    swalAreYouSure(
+      "Your domain settings will be replaced by the uploaded settings",
+      "Restore settings",
+      function() {
+        var files = $('#' + RESTORE_SETTINGS_FILE_ID).prop('files');
 
-    var fileFormData = new FormData();
-    fileFormData.append('restore-file', files[0]);
+        var fileFormData = new FormData();
+        fileFormData.append('restore-file', files[0]);
 
-    showSpinnerAlert("Restoring Settings");
+        showSpinnerAlert("Restoring Settings");
 
-    $.ajax({
-      url: '/settings/restore',
-      type: 'POST',
-      processData: false,
-      contentType: false,
-      dataType: 'json',
-      data: fileFormData
-    }).done(function(data, textStatus, jqXHR) {
-      swal.close();
-      showRestartModal();
-    }).fail(function(jqXHR, textStatus, errorThrown) {
-      showErrorMessage(
-        "Error",
-        "There was a problem restoring domain settings.\n"
-        + "Please ensure that your current domain settings are valid and try again."
-      );
+        $.ajax({
+          url: '/settings/restore',
+          type: 'POST',
+          processData: false,
+          contentType: false,
+          dataType: 'json',
+          data: fileFormData
+        }).done(function(data, textStatus, jqXHR) {
+          swal.close();
+          showRestartModal();
+        }).fail(function(jqXHR, textStatus, errorThrown) {
+          showErrorMessage(
+            "Error",
+            "There was a problem restoring domain settings.\n"
+            + "Please ensure that your current domain settings are valid and try again."
+          );
 
-      reloadSettings();
-    });
+          reloadSettings();
+        });
+      }
+    );
   });
 
   $('body').on('change', '#' + RESTORE_SETTINGS_FILE_ID, function() {
@@ -1089,7 +1082,7 @@ $(document).ready(function(){
     html += "<div class='form-group'>";
     html += "<label class='control-label'>Upload a Settings Configuration</label>";
     html += "<span class='help-block'>Upload a settings configuration to quickly configure this domain";
-    html += "<br/>Note: Your domain's settings will be replaced by the settings you upload</span>";
+    html += "<br/>Note: Your domain settings will be replaced by the settings you upload</span>";
 
     html += "<input id='restore-settings-file' name='restore-settings' type='file'>";
     html += "<button type='button' id='" + RESTORE_SETTINGS_UPLOAD_ID + "' disabled='true' class='btn btn-primary'>Upload Domain Settings</button>";
@@ -1097,8 +1090,5 @@ $(document).ready(function(){
     html += "</div>";
 
     $('#settings_backup .panel-body').html(html);
-
-    // add an upload button to the footer to kick off the upload form
-
   }
 });

domain-server/resources/web/wizard/index.shtml

@@ -261,6 +261,5 @@
 <script src='/js/underscore-min.js'></script>
 <script src='/js/bootbox.min.js'></script>
 <script src='/js/sha256.js'></script>
-<script src='/js/shared.js'></script>
 <script src='js/wizard.js'></script>
 <!--#include virtual="page-end.html"-->

domain-server/resources/web/wizard/js/wizard.js

@@ -396,10 +396,12 @@ function savePermissions() {
 
   var admins = $('#admin-usernames').val().split(',');
 
-  var existingAdmins = Settings.data.values.security.permissions.map(function(value) {
-    return value.permissions_id;
-  });
-  admins = admins.concat(existingAdmins);
+  if (Settings.data.values.security.permissions) {
+    var existingAdmins = Settings.data.values.security.permissions.map(function(value) {
+      return value.permissions_id;
+    });
+    admins = admins.concat(existingAdmins);
+  }
 
   // Filter out unique values
   admins = _.uniq(admins.map(function(username) {

domain-server/src/AssetsBackupHandler.cpp

@@ -0,0 +1,605 @@
+//
+//  AssetsBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Clement Brisset on 1/12/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "AssetsBackupHandler.h"
+
+#include <QJsonDocument>
+#include <QDate>
+#include <QtCore/QLoggingCategory>
+
+#include <quazip5/quazipfile.h>
+#include <quazip5/quazipdir.h>
+
+#include <AssetClient.h>
+#include <AssetRequest.h>
+#include <AssetUpload.h>
+#include <MappingRequest.h>
+#include <PathUtils.h>
+
+using namespace std;
+
+static const QString ASSETS_DIR { "/assets/" };
+static const QString MAPPINGS_FILE { "mappings.json" };
+static const QString ZIP_ASSETS_FOLDER { "files" };
+static const chrono::minutes MAX_REFRESH_TIME { 5 };
+
+Q_DECLARE_LOGGING_CATEGORY(asset_backup)
+Q_LOGGING_CATEGORY(asset_backup, "hifi.asset-backup");
+
+AssetsBackupHandler::AssetsBackupHandler(const QString& backupDirectory) :
+    _assetsDirectory(backupDirectory + ASSETS_DIR)
+{
+    // Make sure the asset directory exists.
+    QDir(_assetsDirectory).mkpath(".");
+
+    refreshAssetsOnDisk();
+
+    setupRefreshTimer();
+}
+
+void AssetsBackupHandler::setupRefreshTimer() {
+    _mappingsRefreshTimer.setTimerType(Qt::CoarseTimer);
+    _mappingsRefreshTimer.setSingleShot(true);
+    QObject::connect(&_mappingsRefreshTimer, &QTimer::timeout, this, &AssetsBackupHandler::refreshMappings);
+
+    auto nodeList = DependencyManager::get<LimitedNodeList>();
+    QObject::connect(nodeList.data(), &LimitedNodeList::nodeActivated, this, [this](SharedNodePointer node) {
+        if (node->getType() == NodeType::AssetServer) {
+            // run immediately for the first time.
+            _mappingsRefreshTimer.start(0);
+        }
+    });
+    QObject::connect(nodeList.data(), &LimitedNodeList::nodeKilled, this, [this](SharedNodePointer node) {
+        if (node->getType() == NodeType::AssetServer) {
+            _mappingsRefreshTimer.stop();
+        }
+    });
+}
+
+void AssetsBackupHandler::refreshAssetsOnDisk() {
+    QDir assetsDir { _assetsDirectory };
+    auto assetNames = assetsDir.entryList(QDir::Files);
+
+    // store all valid hashes
+    copy_if(begin(assetNames), end(assetNames),
+            inserter(_assetsOnDisk, begin(_assetsOnDisk)),
+            AssetUtils::isValidHash);
+
+}
+
+void AssetsBackupHandler::refreshAssetsInBackups() {
+    _assetsInBackups.clear();
+    for (const auto& backup : _backups) {
+        for (const auto& mapping : backup.mappings) {
+            _assetsInBackups.insert(mapping.second);
+        }
+    }
+}
+
+void AssetsBackupHandler::checkForMissingAssets() {
+    vector<AssetUtils::AssetHash> missingAssets;
+    set_difference(begin(_assetsInBackups), end(_assetsInBackups),
+                   begin(_assetsOnDisk), end(_assetsOnDisk),
+                   back_inserter(missingAssets));
+    if (missingAssets.size() > 0) {
+        qCWarning(asset_backup) << "Found" << missingAssets.size() << "backup assets missing from disk.";
+    }
+}
+
+void AssetsBackupHandler::checkForAssetsToDelete() {
+    vector<AssetUtils::AssetHash> deprecatedAssets;
+    set_difference(begin(_assetsOnDisk), end(_assetsOnDisk),
+                   begin(_assetsInBackups), end(_assetsInBackups),
+                   back_inserter(deprecatedAssets));
+
+    if (deprecatedAssets.size() > 0) {
+        qCDebug(asset_backup) << "Found" << deprecatedAssets.size() << "backup assets to delete from disk.";
+        const auto noCorruptedBackups = none_of(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+            return backup.corruptedBackup;
+        });
+        if (noCorruptedBackups) {
+            for (const auto& hash : deprecatedAssets) {
+                auto success = QFile::remove(_assetsDirectory + hash);
+                if (success) {
+                    _assetsOnDisk.erase(hash);
+                } else {
+                    qCWarning(asset_backup) << "Could not delete asset:" << hash;
+                }
+            }
+        } else {
+            qCWarning(asset_backup) << "Some backups did not load properly, aborting delete operation for safety.";
+        }
+    }
+}
+
+bool AssetsBackupHandler::isCorruptedBackup(const QString& backupName) {
+    auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& value) {
+        return value.name == backupName;
+    });
+
+    if (it == end(_backups)) {
+        return false;
+    }
+
+    return it->corruptedBackup;
+}
+
+std::pair<bool, float> AssetsBackupHandler::isAvailable(const QString& backupName) {
+    const auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        return { true, 1.0f };
+    }
+
+    int mappingsMissing = 0;
+    for (const auto& mapping : it->mappings) {
+        if (_assetsLeftToRequest.find(mapping.second) != end(_assetsLeftToRequest)) {
+            ++mappingsMissing;
+        }
+    }
+
+    if (mappingsMissing == 0) {
+        return { true, 1.0f };
+    }
+
+    float progress = (float)it->mappings.size();
+    progress -= (float)mappingsMissing;
+    progress /= it->mappings.size();
+
+    return { false, progress };
+}
+
+std::pair<bool, float> AssetsBackupHandler::getRecoveryStatus() {
+    if (_assetsLeftToUpload.empty() &&
+        _mappingsLeftToSet.empty() &&
+        _mappingsLeftToDelete.empty() &&
+        _mappingRequestsInFlight == 0) {
+        return { false, 1.0f };
+    }
+
+    float progress = (float)_numRestoreOperations;
+    progress -= (float)_assetsLeftToUpload.size();
+    progress -= (float)_mappingRequestsInFlight;
+    progress /= (float)_numRestoreOperations;
+
+    return { true, progress };
+}
+
+void AssetsBackupHandler::loadBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    _backups.emplace_back(backupName, AssetUtils::Mappings(), false);
+    auto& backup = _backups.back();
+
+    if (!zip.setCurrentFile(MAPPINGS_FILE)) {
+        qCCritical(asset_backup) << "Failed to find" << MAPPINGS_FILE << "while loading backup";
+        qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QFile::ReadOnly)) {
+        qCCritical(asset_backup) << "Could not unzip backup file for load:" << MAPPINGS_FILE;
+        qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    QJsonParseError error;
+    auto document = QJsonDocument::fromJson(zipFile.readAll(), &error);
+    if (document.isNull() || !document.isObject()) {
+        qCCritical(asset_backup) << "Could not parse backup file to JSON object for load:" << MAPPINGS_FILE;
+        qCCritical(asset_backup) << "    Error:" << error.errorString();
+        backup.corruptedBackup = true;
+        return;
+    }
+
+    auto jsonObject = document.object();
+    for (auto it = begin(jsonObject); it != end(jsonObject); ++it) {
+        const auto& assetPath = it.key();
+        const auto& assetHash = it.value().toString();
+
+        if (!AssetUtils::isValidHash(assetHash)) {
+            qCCritical(asset_backup) << "Corrupted mapping in loading backup file" << backupName << ":" << it.key();
+            backup.corruptedBackup = true;
+            continue;
+        }
+
+        backup.mappings[assetPath] = assetHash;
+        _assetsInBackups.insert(assetHash);
+    }
+}
+
+void AssetsBackupHandler::loadingComplete() {
+    checkForMissingAssets();
+    checkForAssetsToDelete();
+}
+
+void AssetsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is already an operation in progress.";
+        return;
+    }
+
+    if (_lastMappingsRefresh.time_since_epoch().count() == 0) {
+        qCWarning(asset_backup) << "Current mappings not yet loaded.";
+        return;
+    }
+
+    if ((p_high_resolution_clock::now() - _lastMappingsRefresh) > MAX_REFRESH_TIME) {
+        qCWarning(asset_backup) << "Backing up asset mappings that might be stale.";
+    }
+
+    AssetUtils::Mappings mappings;
+
+    QJsonObject jsonObject;
+    for (const auto& mapping : _currentMappings) {
+        mappings[mapping.first] = mapping.second;
+        _assetsInBackups.insert(mapping.second);
+        jsonObject.insert(mapping.first, mapping.second);
+    }
+    QJsonDocument document(jsonObject);
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(MAPPINGS_FILE))) {
+        qCDebug(asset_backup) << "Could not open zip file:" << zipFile.getZipError();
+        return;
+    }
+    zipFile.write(document.toJson());
+    zipFile.close();
+    if (zipFile.getZipError() != UNZ_OK) {
+        qCDebug(asset_backup) << "Could not close zip file: " << zipFile.getZipError();
+        return;
+    }
+    _backups.emplace_back(backupName, mappings, false);
+    qDebug() << "Created asset backup:" << backupName;
+}
+
+void AssetsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is already a backup/restore in progress.";
+        return;
+    }
+
+    if (_lastMappingsRefresh.time_since_epoch().count() == 0) {
+        qCWarning(asset_backup) << "Current mappings not yet loaded.";
+        return;
+    }
+
+    if ((p_high_resolution_clock::now() - _lastMappingsRefresh) > MAX_REFRESH_TIME) {
+        qCWarning(asset_backup) << "Recovering while current asset mappings might be stale.";
+    }
+
+    auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        loadBackup(backupName, zip);
+
+        QuaZipDir zipDir { &zip, ZIP_ASSETS_FOLDER };
+
+        auto assetNames = zipDir.entryList(QDir::Files);
+        for (const auto& asset : assetNames) {
+            if (AssetUtils::isValidHash(asset)) {
+                if (!zip.setCurrentFile(zipDir.filePath(asset))) {
+                    qCCritical(asset_backup) << "Failed to find" << asset << "while recovering backup";
+                    qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+                    continue;
+                }
+
+                QuaZipFile zipFile { &zip };
+                if (!zipFile.open(QFile::ReadOnly)) {
+                    qCCritical(asset_backup) << "Could not unzip asset file:" << asset;
+                    qCCritical(asset_backup) << "    Error:" << zip.getZipError();
+                    continue;
+                }
+
+                writeAssetFile(asset, zipFile.readAll());
+            }
+        }
+
+        // iterator is end() and has been invalidated in the `loadBackup` call
+        // grab the new iterator
+        it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+            return backup.name == backupName;
+        });
+
+        if (it == end(_backups)) {
+            qCCritical(asset_backup) << "Failed to recover backup:" << backupName;
+            return;
+        }
+    }
+
+    const auto& newMappings = it->mappings;
+    computeServerStateDifference(_currentMappings, newMappings);
+
+    restoreAllAssets();
+}
+
+void AssetsBackupHandler::deleteBackup(const QString& backupName) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is a backup/restore in progress.";
+        return;
+    }
+
+    const auto it = remove_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        qCDebug(asset_backup) << "Could not find backup" << backupName << "to delete.";
+        return;
+    }
+
+    _backups.erase(it, end(_backups));
+
+    refreshAssetsInBackups();
+    checkForAssetsToDelete();
+    qDebug() << "Deleted asset backup:" << backupName;
+}
+
+void AssetsBackupHandler::consolidateBackup(const QString& backupName, QuaZip& zip) {
+    Q_ASSERT(QThread::currentThread() == thread());
+
+    if (operationInProgress()) {
+        qCWarning(asset_backup) << "There is a backup/restore in progress.";
+        return;
+    }
+
+    const auto it = find_if(begin(_backups), end(_backups), [&](const AssetServerBackup& backup) {
+        return backup.name == backupName;
+    });
+    if (it == end(_backups)) {
+        qCDebug(asset_backup) << "Could not find backup" << backupName << "to consolidate.";
+        return;
+    }
+
+    for (const auto& mapping : it->mappings) {
+        const auto& hash = mapping.second;
+
+        QDir assetsDir { _assetsDirectory };
+        QFile file { assetsDir.filePath(hash) };
+        if (!file.open(QFile::ReadOnly)) {
+            qCCritical(asset_backup) << "Could not open asset file" << file.fileName();
+            continue;
+        }
+
+        QuaZipFile zipFile { &zip };
+        if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(ZIP_ASSETS_FOLDER + "/" + hash))) {
+            qCDebug(asset_backup) << "Could not open zip file:" << zipFile.getZipError();
+            continue;
+        }
+        zipFile.write(file.readAll());
+        zipFile.close();
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCDebug(asset_backup) << "Could not close zip file: " << zipFile.getZipError();
+            continue;
+        }
+    }
+
+}
+
+void AssetsBackupHandler::refreshMappings() {
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto request = assetClient->createGetAllMappingsRequest();
+
+    QObject::connect(request, &GetAllMappingsRequest::finished, this, [this](GetAllMappingsRequest* request) {
+        if (request->getError() == MappingRequest::NoError) {
+            const auto& mappings = request->getMappings();
+
+            // Clear existing mappings
+            _currentMappings.clear();
+
+            // Set new mapping, but ignore baked assets
+            for (const auto& mapping : mappings) {
+                if (!mapping.first.startsWith(AssetUtils::HIDDEN_BAKED_CONTENT_FOLDER)) {
+                    _currentMappings.insert({ mapping.first, mapping.second.hash });
+                }
+            }
+            _lastMappingsRefresh = p_high_resolution_clock::now();
+
+            downloadMissingFiles(_currentMappings);
+        } else {
+            qCCritical(asset_backup) << "Could not refresh asset server mappings.";
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        request->deleteLater();
+
+        // Launch next mappings request
+        static constexpr int MAPPINGS_REFRESH_INTERVAL = 30 * 1000;
+        _mappingsRefreshTimer.start(MAPPINGS_REFRESH_INTERVAL);
+    });
+
+    request->start();
+}
+
+void AssetsBackupHandler::downloadMissingFiles(const AssetUtils::Mappings& mappings) {
+    auto wasEmpty = _assetsLeftToRequest.empty();
+
+    for (const auto& mapping : mappings) {
+        const auto& hash = mapping.second;
+        if (_assetsOnDisk.find(hash) == end(_assetsOnDisk)) {
+            _assetsLeftToRequest.insert(hash);
+        }
+    }
+
+    // If we were empty, that means no download chain was already going, start one.
+    if (wasEmpty) {
+        downloadNextMissingFile();
+    }
+}
+
+void AssetsBackupHandler::downloadNextMissingFile() {
+    if (_assetsLeftToRequest.empty()) {
+        return;
+    }
+    auto hash = *begin(_assetsLeftToRequest);
+
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto assetRequest = assetClient->createRequest(hash);
+
+    QObject::connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) {
+        if (request->getError() == AssetRequest::NoError) {
+            qCDebug(asset_backup) << "Backing up asset" << request->getHash();
+
+            bool success = writeAssetFile(request->getHash(), request->getData());
+            if (!success) {
+                qCCritical(asset_backup) << "Failed to write asset file" << request->getHash();
+            }
+        } else {
+            qCCritical(asset_backup) << "Failed to backup asset" << request->getHash();
+        }
+
+        _assetsLeftToRequest.erase(request->getHash());
+        downloadNextMissingFile();
+
+        request->deleteLater();
+    });
+
+    assetRequest->start();
+}
+
+bool AssetsBackupHandler::writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data) {
+    QDir assetsDir { _assetsDirectory };
+    QFile file { assetsDir.filePath(hash) };
+    if (!file.open(QFile::WriteOnly)) {
+        qCCritical(asset_backup) << "Could not open asset file for write:" << file.fileName();
+        return false;
+    }
+
+    auto bytesWritten = file.write(data);
+    if (bytesWritten != data.size()) {
+        qCCritical(asset_backup) << "Could not write data to file" << file.fileName();
+        file.remove();
+        return false;
+    }
+
+    _assetsOnDisk.insert(hash);
+
+    return true;
+}
+
+void AssetsBackupHandler::computeServerStateDifference(const AssetUtils::Mappings& currentMappings,
+                                                       const AssetUtils::Mappings& newMappings) {
+    _mappingsLeftToSet.reserve((int)newMappings.size());
+    _assetsLeftToUpload.reserve((int)newMappings.size());
+    _mappingsLeftToDelete.reserve((int)currentMappings.size());
+
+    set<AssetUtils::AssetHash> currentAssets;
+    for (const auto& currentMapping : currentMappings) {
+        const auto& currentPath = currentMapping.first;
+        const auto& currentHash = currentMapping.second;
+
+        if (newMappings.find(currentPath) == end(newMappings)) {
+            _mappingsLeftToDelete.push_back(currentPath);
+        }
+        currentAssets.insert(currentHash);
+    }
+
+    for (const auto& newMapping : newMappings) {
+        const auto& newPath = newMapping.first;
+        const auto& newHash = newMapping.second;
+
+        auto it = currentMappings.find(newPath);
+        if (it == end(currentMappings) || it->second != newHash) {
+            _mappingsLeftToSet.push_back({ newPath, newHash });
+        }
+        if (currentAssets.find(newHash) == end(currentAssets)) {
+            _assetsLeftToUpload.push_back(newHash);
+        }
+    }
+
+    _numRestoreOperations = (int)_assetsLeftToUpload.size() + (int)_mappingsLeftToSet.size();
+    if (!_mappingsLeftToDelete.empty()) {
+        ++_numRestoreOperations;
+    }
+
+    qCDebug(asset_backup) << "Mappings to set:" << _mappingsLeftToSet.size();
+    qCDebug(asset_backup) << "Mappings to del:" << _mappingsLeftToDelete.size();
+    qCDebug(asset_backup) << "Assets to upload:" << _assetsLeftToUpload.size();
+}
+
+void AssetsBackupHandler::restoreAllAssets() {
+    restoreNextAsset();
+}
+
+void AssetsBackupHandler::restoreNextAsset() {
+    if (_assetsLeftToUpload.empty()) {
+        updateMappings();
+        return;
+    }
+
+    auto hash = _assetsLeftToUpload.back();
+    _assetsLeftToUpload.pop_back();
+
+    auto assetFilename = _assetsDirectory + hash;
+
+    auto assetClient = DependencyManager::get<AssetClient>();
+    auto request = assetClient->createUpload(assetFilename);
+
+    QObject::connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) {
+        if (request->getError() != AssetUpload::NoError) {
+            qCCritical(asset_backup) << "Failed to restore asset:" << request->getFilename();
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        restoreNextAsset();
+
+        request->deleteLater();
+    });
+
+    request->start();
+}
+
+void AssetsBackupHandler::updateMappings() {
+    auto assetClient = DependencyManager::get<AssetClient>();
+    for (const auto& mapping : _mappingsLeftToSet) {
+        auto request = assetClient->createSetMappingRequest(mapping.first, mapping.second);
+        QObject::connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) {
+            if (request->getError() != MappingRequest::NoError) {
+                qCCritical(asset_backup) << "Failed to set mapping:" << request->getPath();
+                qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+            }
+
+            --_mappingRequestsInFlight;
+
+            request->deleteLater();
+        });
+
+        request->start();
+        ++_mappingRequestsInFlight;
+    }
+    _mappingsLeftToSet.clear();
+
+    auto request = assetClient->createDeleteMappingsRequest(_mappingsLeftToDelete);
+    QObject::connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) {
+        if (request->getError() != MappingRequest::NoError) {
+            qCCritical(asset_backup) << "Failed to delete mappings";
+            qCCritical(asset_backup) << "    Error:" << request->getErrorString();
+        }
+
+        --_mappingRequestsInFlight;
+
+        request->deleteLater();
+    });
+    _mappingsLeftToDelete.clear();
+
+    request->start();
+    ++_mappingRequestsInFlight;
+}

domain-server/src/AssetsBackupHandler.h

@@ -0,0 +1,98 @@
+//
+//  AssetsBackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 1/12/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_AssetsBackupHandler_h
+#define hifi_AssetsBackupHandler_h
+
+#include <set>
+#include <map>
+
+#include <QObject>
+#include <QTimer>
+#include <QJsonDocument>
+#include <QJsonObject>
+
+#include <AssetUtils.h>
+#include <ReceivedMessage.h>
+#include <PortableHighResolutionClock.h>
+
+#include "BackupHandler.h"
+
+class AssetsBackupHandler : public QObject, public BackupHandlerInterface {
+    Q_OBJECT
+
+public:
+    AssetsBackupHandler(const QString& backupDirectory);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override;
+    std::pair<bool, float> getRecoveryStatus() override;
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override;
+    void loadingComplete() override;
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+    void deleteBackup(const QString& backupName) override;
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override;
+    bool isCorruptedBackup(const QString& backupName) override;
+
+    bool operationInProgress() { return getRecoveryStatus().first; }
+
+private:
+    void setupRefreshTimer();
+    void refreshMappings();
+
+    void refreshAssetsInBackups();
+    void refreshAssetsOnDisk();
+    void checkForMissingAssets();
+    void checkForAssetsToDelete();
+
+    void downloadMissingFiles(const AssetUtils::Mappings& mappings);
+    void downloadNextMissingFile();
+    bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data);
+
+    void computeServerStateDifference(const AssetUtils::Mappings& currentMappings,
+                                      const AssetUtils::Mappings& newMappings);
+    void restoreAllAssets();
+    void restoreNextAsset();
+    void updateMappings();
+
+    QString _assetsDirectory;
+
+    QTimer _mappingsRefreshTimer;
+    p_high_resolution_clock::time_point _lastMappingsRefresh;
+    AssetUtils::Mappings _currentMappings;
+
+    struct AssetServerBackup {
+        AssetServerBackup(const QString& pName, AssetUtils::Mappings pMappings, bool pCorruptedBackup) :
+            name(pName), mappings(pMappings), corruptedBackup(pCorruptedBackup) {}
+
+        QString name;
+        AssetUtils::Mappings mappings;
+        bool corruptedBackup;
+    };
+
+    // Internal storage for backups on disk
+    std::vector<AssetServerBackup> _backups;
+    std::set<AssetUtils::AssetHash> _assetsInBackups;
+    std::set<AssetUtils::AssetHash> _assetsOnDisk;
+
+    // Internal storage for backup in progress
+    std::set<AssetUtils::AssetHash> _assetsLeftToRequest;
+
+    // Internal storage for restore in progress
+    std::vector<AssetUtils::AssetHash> _assetsLeftToUpload;
+    std::vector<std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash>> _mappingsLeftToSet;
+    AssetUtils::AssetPathList _mappingsLeftToDelete;
+    int _mappingRequestsInFlight { 0 };
+    int _numRestoreOperations { 0 }; // Used to compute a restore progress.
+};
+
+#endif /* hifi_AssetsBackupHandler_h */

domain-server/src/BackupHandler.h

@@ -0,0 +1,40 @@
+//
+//  BackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/5/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_BackupHandler_h
+#define hifi_BackupHandler_h
+
+#include <memory>
+
+#include <QString>
+
+class QuaZip;
+
+class BackupHandlerInterface {
+public:
+    virtual ~BackupHandlerInterface() = default;
+
+    virtual std::pair<bool, float> isAvailable(const QString& backupName) = 0;
+
+    // Returns whether a recovery is ongoing and a progress between 0 and 1 if one is.
+    virtual std::pair<bool, float> getRecoveryStatus() = 0;
+
+    virtual void loadBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void loadingComplete() = 0;
+    virtual void createBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void recoverBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual void deleteBackup(const QString& backupName) = 0;
+    virtual void consolidateBackup(const QString& backupName, QuaZip& zip) = 0;
+    virtual bool isCorruptedBackup(const QString& backupName) = 0;
+};
+using BackupHandlerPointer = std::unique_ptr<BackupHandlerInterface>;
+
+#endif /* hifi_BackupHandler_h */

domain-server/src/BackupSupervisor.cpp

@@ -1,400 +0,0 @@
-//
-//  BackupSupervisor.cpp
-//  domain-server/src
-//
-//  Created by Clement Brisset on 1/12/18.
-//  Copyright 2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#include "BackupSupervisor.h"
-
-#include <QJsonDocument>
-#include <QDate>
-
-#include <AssetClient.h>
-#include <AssetRequest.h>
-#include <AssetUpload.h>
-#include <MappingRequest.h>
-#include <PathUtils.h>
-
-const QString BACKUPS_DIR = "backups/";
-const QString ASSETS_DIR = "files/";
-const QString MAPPINGS_PREFIX = "mappings-";
-
-using namespace std;
-
-BackupSupervisor::BackupSupervisor() {
-    _backupsDirectory = PathUtils::getAppDataPath() + BACKUPS_DIR;
-    QDir backupDir { _backupsDirectory };
-    if (!backupDir.exists()) {
-        backupDir.mkpath(".");
-    }
-
-    _assetsDirectory = PathUtils::getAppDataPath() + BACKUPS_DIR + ASSETS_DIR;
-    QDir assetsDir { _assetsDirectory };
-    if (!assetsDir.exists()) {
-        assetsDir.mkpath(".");
-    }
-
-    loadAllBackups();
-}
-
-void BackupSupervisor::loadAllBackups() {
-    _backups.clear();
-    _assetsInBackups.clear();
-    _assetsOnDisk.clear();
-    _allBackupsLoadedSuccessfully = true;
-
-    QDir assetsDir { _assetsDirectory };
-    auto assetNames = assetsDir.entryList(QDir::Files);
-    qDebug() << "Loading" << assetNames.size() << "assets.";
-
-    // store all valid hashes
-    copy_if(begin(assetNames), end(assetNames),
-            inserter(_assetsOnDisk, begin(_assetsOnDisk)), AssetUtils::isValidHash);
-
-    QDir backupsDir { _backupsDirectory };
-    auto files = backupsDir.entryList({ MAPPINGS_PREFIX + "*.json" }, QDir::Files);
-    qDebug() << "Loading" << files.size() << "backups.";
-
-    for (const auto& fileName : files) {
-        auto filePath = backupsDir.filePath(fileName);
-        auto success = loadBackup(filePath);
-        if (!success) {
-            qCritical() << "Failed to load backup file" << filePath;
-            _allBackupsLoadedSuccessfully = false;
-        }
-    }
-
-    vector<AssetUtils::AssetHash> missingAssets;
-    set_difference(begin(_assetsInBackups), end(_assetsInBackups),
-                   begin(_assetsOnDisk), end(_assetsOnDisk),
-                   back_inserter(missingAssets));
-    if (missingAssets.size() > 0) {
-        qWarning() << "Found" << missingAssets.size() << "assets missing.";
-    }
-
-    vector<AssetUtils::AssetHash> deprecatedAssets;
-    set_difference(begin(_assetsOnDisk), end(_assetsOnDisk),
-                   begin(_assetsInBackups), end(_assetsInBackups),
-                   back_inserter(deprecatedAssets));
-
-    if (deprecatedAssets.size() > 0) {
-        qDebug() << "Found" << deprecatedAssets.size() << "assets to delete.";
-        if (_allBackupsLoadedSuccessfully) {
-            for (const auto& hash : deprecatedAssets) {
-                QFile::remove(_assetsDirectory + hash);
-            }
-        } else {
-            qWarning() << "Some backups did not load properly, aborting deleting for safety.";
-        }
-    }
-}
-
-bool BackupSupervisor::loadBackup(const QString& backupFile) {
-    _backups.push_back({ backupFile.toStdString(), {}, false });
-    auto& backup = _backups.back();
-
-    QFile file { backupFile };
-    if (!file.open(QFile::ReadOnly)) {
-        qCritical() << "Could not open backup file:" << backupFile;
-        backup.corruptedBackup = true;
-        return false;
-    }
-    QJsonParseError error;
-    auto document = QJsonDocument::fromJson(file.readAll(), &error);
-    if (document.isNull() || !document.isObject()) {
-        qCritical() << "Could not parse backup file to JSON object:" << backupFile;
-        qCritical() << "    Error:" << error.errorString();
-        backup.corruptedBackup = true;
-        return false;
-    }
-
-    auto jsonObject = document.object();
-    for (auto it = begin(jsonObject); it != end(jsonObject); ++it) {
-        const auto& assetPath = it.key();
-        const auto& assetHash = it.value().toString();
-
-        if (!AssetUtils::isValidHash(assetHash)) {
-            qCritical() << "Corrupted mapping in backup file" << backupFile << ":" << it.key();
-            backup.corruptedBackup = true;
-            return false;
-        }
-
-        backup.mappings[assetPath] = assetHash;
-        _assetsInBackups.insert(assetHash);
-    }
-
-    _backups.push_back(backup);
-    return true;
-}
-
-void BackupSupervisor::backupAssetServer() {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is already a backup/restore in progress.";
-        return;
-    }
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createGetAllMappingsRequest();
-
-    connect(request, &GetAllMappingsRequest::finished, this, [this](GetAllMappingsRequest* request) {
-        qDebug() << "Got" << request->getMappings().size() << "mappings!";
-
-        if (request->getError() != MappingRequest::NoError) {
-            qCritical() << "Could not complete backup.";
-            qCritical() << "    Error:" << request->getErrorString();
-            finishBackup();
-            request->deleteLater();
-            return;
-        }
-
-        if (!writeBackupFile(request->getMappings())) {
-            finishBackup();
-            request->deleteLater();
-            return;
-        }
-
-        assert(!_backups.empty());
-        const auto& mappings = _backups.back().mappings;
-        backupMissingFiles(mappings);
-
-        request->deleteLater();
-    });
-
-    startBackup();
-    request->start();
-}
-
-void BackupSupervisor::backupMissingFiles(const AssetUtils::Mappings& mappings) {
-    _assetsLeftToRequest.reserve(mappings.size());
-    for (auto& mapping : mappings) {
-        const auto& hash = mapping.second;
-        if (_assetsOnDisk.find(hash) == end(_assetsOnDisk)) {
-            _assetsLeftToRequest.push_back(hash);
-        }
-    }
-
-    backupNextMissingFile();
-}
-
-void BackupSupervisor::backupNextMissingFile() {
-    if (_assetsLeftToRequest.empty()) {
-        finishBackup();
-        return;
-    }
-
-    auto hash = _assetsLeftToRequest.back();
-    _assetsLeftToRequest.pop_back();
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto assetRequest = assetClient->createRequest(hash);
-
-    connect(assetRequest, &AssetRequest::finished, this, [this](AssetRequest* request) {
-        if (request->getError() == AssetRequest::NoError) {
-            qDebug() << "Got" << request->getHash();
-
-            bool success = writeAssetFile(request->getHash(), request->getData());
-            if (!success) {
-                qCritical() << "Failed to write asset file" << request->getHash();
-            }
-        } else {
-            qCritical() << "Failed to backup asset" << request->getHash();
-        }
-
-        backupNextMissingFile();
-
-        request->deleteLater();
-    });
-
-    assetRequest->start();
-}
-
-bool BackupSupervisor::writeBackupFile(const AssetUtils::AssetMappings& mappings) {
-    auto filename = MAPPINGS_PREFIX + QDateTime::currentDateTimeUtc().toString(Qt::ISODate) + ".json";
-    QFile file { PathUtils::getAppDataPath() + BACKUPS_DIR + filename };
-    if (!file.open(QFile::WriteOnly)) {
-        qCritical() << "Could not open backup file" << file.fileName();
-        return false;
-    }
-
-    AssetServerBackup backup;
-    QJsonObject jsonObject;
-    for (auto& mapping : mappings) {
-        backup.mappings[mapping.first] = mapping.second.hash;
-        _assetsInBackups.insert(mapping.second.hash);
-        jsonObject.insert(mapping.first, mapping.second.hash);
-    }
-
-    QJsonDocument document(jsonObject);
-    file.write(document.toJson());
-
-    backup.filePath = file.fileName().toStdString();
-    _backups.push_back(backup);
-
-    return true;
-}
-
-bool BackupSupervisor::writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data) {
-    QDir assetsDir { _assetsDirectory };
-    QFile file { assetsDir.filePath(hash) };
-    if (!file.open(QFile::WriteOnly)) {
-        qCritical() << "Could not open backup file" << file.fileName();
-        return false;
-    }
-
-    file.write(data);
-
-    _assetsOnDisk.insert(hash);
-
-    return true;
-}
-
-void BackupSupervisor::restoreAssetServer(int backupIndex) {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is already a backup/restore in progress.";
-        return;
-    }
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createGetAllMappingsRequest();
-
-    connect(request, &GetAllMappingsRequest::finished, this, [this, backupIndex](GetAllMappingsRequest* request) {
-        if (request->getError() == MappingRequest::NoError) {
-            const auto& newMappings = _backups.at(backupIndex).mappings;
-            computeServerStateDifference(request->getMappings(), newMappings);
-
-            restoreAllAssets();
-        } else {
-            finishRestore();
-        }
-
-        request->deleteLater();
-    });
-
-    startRestore();
-    request->start();
-}
-
-void BackupSupervisor::computeServerStateDifference(const AssetUtils::AssetMappings& currentMappings,
-                                                    const AssetUtils::Mappings& newMappings) {
-    _mappingsLeftToSet.reserve((int)newMappings.size());
-    _assetsLeftToUpload.reserve((int)newMappings.size());
-    _mappingsLeftToDelete.reserve((int)currentMappings.size());
-
-    set<AssetUtils::AssetHash> currentAssets;
-    for (const auto& currentMapping : currentMappings) {
-        const auto& currentPath = currentMapping.first;
-        const auto& currentHash = currentMapping.second.hash;
-
-        if (newMappings.find(currentPath) == end(newMappings)) {
-            _mappingsLeftToDelete.push_back(currentPath);
-        }
-        currentAssets.insert(currentHash);
-    }
-
-    for (const auto& newMapping : newMappings) {
-        const auto& newPath = newMapping.first;
-        const auto& newHash = newMapping.second;
-
-        auto it = currentMappings.find(newPath);
-        if (it == end(currentMappings) || it->second.hash != newHash) {
-            _mappingsLeftToSet.push_back({ newPath, newHash });
-        }
-        if (currentAssets.find(newHash) == end(currentAssets)) {
-            _assetsLeftToUpload.push_back(newHash);
-        }
-    }
-
-    qDebug() << "Mappings to set:" << _mappingsLeftToSet.size();
-    qDebug() << "Mappings to del:" << _mappingsLeftToDelete.size();
-    qDebug() << "Assets to upload:" << _assetsLeftToUpload.size();
-}
-
-void BackupSupervisor::restoreAllAssets() {
-    restoreNextAsset();
-}
-
-void BackupSupervisor::restoreNextAsset() {
-    if (_assetsLeftToUpload.empty()) {
-        updateMappings();
-        return;
-    }
-
-    auto hash = _assetsLeftToUpload.back();
-    _assetsLeftToUpload.pop_back();
-
-    auto assetFilename = _assetsDirectory + hash;
-
-    auto assetClient = DependencyManager::get<AssetClient>();
-    auto request = assetClient->createUpload(assetFilename);
-
-    connect(request, &AssetUpload::finished, this, [this](AssetUpload* request) {
-        if (request->getError() != AssetUpload::NoError) {
-            qCritical() << "Failed to restore asset:" << request->getFilename();
-            qCritical() << "    Error:" << request->getErrorString();
-        }
-
-        restoreNextAsset();
-
-        request->deleteLater();
-    });
-
-    request->start();
-}
-
-void BackupSupervisor::updateMappings() {
-    auto assetClient = DependencyManager::get<AssetClient>();
-    for (const auto& mapping : _mappingsLeftToSet) {
-        auto request = assetClient->createSetMappingRequest(mapping.first, mapping.second);
-        connect(request, &SetMappingRequest::finished, this, [this](SetMappingRequest* request) {
-            if (request->getError() != MappingRequest::NoError) {
-                qCritical() << "Failed to set mapping:" << request->getPath();
-                qCritical() << "    Error:" << request->getErrorString();
-            }
-
-            if (--_mappingRequestsInFlight == 0) {
-                finishRestore();
-            }
-
-            request->deleteLater();
-        });
-
-        request->start();
-        ++_mappingRequestsInFlight;
-    }
-    _mappingsLeftToSet.clear();
-
-    auto request = assetClient->createDeleteMappingsRequest(_mappingsLeftToDelete);
-    connect(request, &DeleteMappingsRequest::finished, this, [this](DeleteMappingsRequest* request) {
-        if (request->getError() != MappingRequest::NoError) {
-            qCritical() << "Failed to delete mappings";
-            qCritical() << "    Error:" << request->getErrorString();
-        }
-
-        if (--_mappingRequestsInFlight == 0) {
-            finishRestore();
-        }
-
-        request->deleteLater();
-    });
-    _mappingsLeftToDelete.clear();
-
-    request->start();
-    ++_mappingRequestsInFlight;
-}
-bool BackupSupervisor::deleteBackup(int backupIndex) {
-    if (backupInProgress() || restoreInProgress()) {
-        qWarning() << "There is a backup/restore in progress.";
-        return false;
-    }
-    const auto& filePath = _backups.at(backupIndex).filePath;
-    auto success = QFile::remove(filePath.c_str());
-
-    loadAllBackups();
-
-    return success;
-}

domain-server/src/BackupSupervisor.h

@@ -1,85 +0,0 @@
-//
-//  BackupSupervisor.h
-//  domain-server/src
-//
-//  Created by Clement Brisset on 1/12/18.
-//  Copyright 2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#ifndef hifi_BackupSupervisor_h
-#define hifi_BackupSupervisor_h
-
-#include <set>
-#include <map>
-
-#include <QObject>
-
-#include <AssetUtils.h>
-
-#include <ReceivedMessage.h>
-
-struct AssetServerBackup {
-    std::string filePath;
-    AssetUtils::Mappings mappings;
-    bool corruptedBackup;
-};
-
-class BackupSupervisor : public QObject {
-    Q_OBJECT
-
-public:
-    BackupSupervisor();
-
-    void backupAssetServer();
-    void restoreAssetServer(int backupIndex);
-    bool deleteBackup(int backupIndex);
-
-    const std::vector<AssetServerBackup>& getBackups() const { return _backups; };
-
-    bool backupInProgress() const { return _backupInProgress; }
-    bool restoreInProgress() const { return _restoreInProgress; }
-
-private:
-    void loadAllBackups();
-    bool loadBackup(const QString& backupFile);
-
-    void startBackup() { _backupInProgress = true; }
-    void finishBackup() { _backupInProgress = false; }
-    void backupMissingFiles(const AssetUtils::Mappings& mappings);
-    void backupNextMissingFile();
-    bool writeBackupFile(const AssetUtils::AssetMappings& mappings);
-    bool writeAssetFile(const AssetUtils::AssetHash& hash, const QByteArray& data);
-
-    void startRestore() { _restoreInProgress = true; }
-    void finishRestore() { _restoreInProgress = false; }
-    void computeServerStateDifference(const AssetUtils::AssetMappings& currentMappings,
-                                      const AssetUtils::Mappings& newMappings);
-    void restoreAllAssets();
-    void restoreNextAsset();
-    void updateMappings();
-
-    QString _backupsDirectory;
-    QString _assetsDirectory;
-
-    // Internal storage for backups on disk
-    bool _allBackupsLoadedSuccessfully { false };
-    std::vector<AssetServerBackup> _backups;
-    std::set<AssetUtils::AssetHash> _assetsInBackups;
-    std::set<AssetUtils::AssetHash> _assetsOnDisk;
-
-    // Internal storage for backup in progress
-    bool _backupInProgress { false };
-    std::vector<AssetUtils::AssetHash> _assetsLeftToRequest;
-
-    // Internal storage for restore in progress
-    bool _restoreInProgress { false };
-    std::vector<AssetUtils::AssetHash> _assetsLeftToUpload;
-    std::vector<std::pair<AssetUtils::AssetPath, AssetUtils::AssetHash>> _mappingsLeftToSet;
-    AssetUtils::AssetPathList _mappingsLeftToDelete;
-    int _mappingRequestsInFlight { 0 };
-};
-
-#endif /* hifi_BackupSupervisor_h */

domain-server/src/ContentSettingsBackupHandler.cpp

@@ -0,0 +1,74 @@
+//
+//  ContentSettingsBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Stephen Birarda on 2/15/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "ContentSettingsBackupHandler.h"
+
+#include <quazip5/quazip.h>
+#include <quazip5/quazipfile.h>
+
+ContentSettingsBackupHandler::ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager) :
+    _settingsManager(domainServerSettingsManager)
+{
+}
+
+static const QString CONTENT_SETTINGS_BACKUP_FILENAME = "content-settings.json";
+
+void ContentSettingsBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+
+    // grab the content settings as JSON, excluding default values and values hidden from backup
+    QJsonObject contentSettingsJSON = _settingsManager.settingsResponseObjectForType(
+        "", // include all settings types
+        DomainServerSettingsManager::Authenticated, DomainServerSettingsManager::NoDomainSettings,
+        DomainServerSettingsManager::IncludeContentSettings, DomainServerSettingsManager::NoDefaultSettings,
+        DomainServerSettingsManager::ForBackup
+    );
+
+    // make a QJsonDocument using the object
+    QJsonDocument contentSettingsDocument { contentSettingsJSON };
+
+    QuaZipFile zipFile { &zip };
+
+    if (zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(CONTENT_SETTINGS_BACKUP_FILENAME))) {
+        if (zipFile.write(contentSettingsDocument.toJson()) == -1) {
+            qCritical().nospace() << "Failed to write to " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+
+        zipFile.close();
+
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCritical().nospace() << "Failed to zip " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+    } else {
+        qCritical().nospace() << "Failed to open " << CONTENT_SETTINGS_BACKUP_FILENAME << ": " << zipFile.getZipError();
+    }
+}
+
+void ContentSettingsBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    if (!zip.setCurrentFile(CONTENT_SETTINGS_BACKUP_FILENAME)) {
+        qWarning() << "Failed to find" << CONTENT_SETTINGS_BACKUP_FILENAME << "while recovering backup";
+        return;
+    }
+
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::ReadOnly)) {
+        qCritical() << "Failed to open" << CONTENT_SETTINGS_BACKUP_FILENAME << "in backup";
+        return;
+    }
+
+    auto rawData = zipFile.readAll();
+    zipFile.close();
+
+    QJsonDocument jsonDocument = QJsonDocument::fromJson(rawData);
+
+    if (!_settingsManager.restoreSettingsFromObject(jsonDocument.object(), ContentSettings)) {
+        qCritical() << "Failed to restore settings from" << CONTENT_SETTINGS_BACKUP_FILENAME << "in content archive";
+    }
+}

domain-server/src/ContentSettingsBackupHandler.h

@@ -0,0 +1,43 @@
+//
+//  ContentSettingsBackupHandler.h
+//  domain-server/src
+//
+//  Created by Stephen Birarda on 2/15/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_ContentSettingsBackupHandler_h
+#define hifi_ContentSettingsBackupHandler_h
+
+#include "BackupHandler.h"
+#include "DomainServerSettingsManager.h"
+
+class ContentSettingsBackupHandler : public BackupHandlerInterface {
+public:
+    ContentSettingsBackupHandler(DomainServerSettingsManager& domainServerSettingsManager);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override { return { true, 1.0f }; }
+    std::pair<bool, float> getRecoveryStatus() override { return { false, 1.0f }; }
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override {}
+
+    void loadingComplete() override {}
+
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+
+    void deleteBackup(const QString& backupName) override {}
+
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override {}
+
+    bool isCorruptedBackup(const QString& backupName) override { return false; }
+
+private:
+    DomainServerSettingsManager& _settingsManager;
+};
+
+#endif // hifi_ContentSettingsBackupHandler_h

domain-server/src/DomainContentBackupManager.cpp

@@ -0,0 +1,599 @@
+//
+//  DomainContentBackupManager.cpp
+//  libraries/domain-server/src
+//
+//  Created by Ryan Huffman on 1/01/18.
+//  Adapted from OctreePersistThread
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "DomainContentBackupManager.h"
+
+#include <chrono>
+#include <thread>
+
+#include <cstdio>
+#include <fstream>
+#include <time.h>
+
+#include <QBuffer>
+#include <QDateTime>
+#include <QDebug>
+#include <QDir>
+#include <QDirIterator>
+#include <QFile>
+#include <QJsonArray>
+#include <QJsonObject>
+#include <QJsonDocument>
+
+#include <quazip5/quazip.h>
+
+#include <NumericalConstants.h>
+#include <PerfStat.h>
+#include <PathUtils.h>
+#include <shared/QtHelpers.h>
+
+#include "DomainServer.h"
+
+const std::chrono::seconds DomainContentBackupManager::DEFAULT_PERSIST_INTERVAL { 30 };
+
+// Backup format looks like: daily_backup-TIMESTAMP.zip
+static const QString DATETIME_FORMAT { "yyyy-MM-dd_HH-mm-ss" };
+static const QString DATETIME_FORMAT_RE { "\\d{4}-\\d{2}-\\d{2}_\\d{2}-\\d{2}-\\d{2}" };
+static const QString AUTOMATIC_BACKUP_PREFIX { "autobackup-" };
+static const QString MANUAL_BACKUP_PREFIX { "backup-" };
+static const QString MANUAL_BACKUP_NAME_RE { "[a-zA-Z0-9\\-_ ]+" };
+
+void DomainContentBackupManager::addBackupHandler(BackupHandlerPointer handler) {
+    _backupHandlers.push_back(std::move(handler));
+}
+
+DomainContentBackupManager::DomainContentBackupManager(const QString& backupDirectory,
+                                                       const QVariantList& backupRules,
+                                                       std::chrono::milliseconds persistInterval,
+                                                       bool debugTimestampNow) :
+    _backupDirectory(backupDirectory), _persistInterval(persistInterval), _lastCheck(p_high_resolution_clock::now())
+{
+
+    setObjectName("DomainContentBackupManager");
+
+    // Make sure the backup directory exists.
+    QDir(_backupDirectory).mkpath(".");
+
+    parseBackupRules(backupRules);
+}
+
+void DomainContentBackupManager::parseBackupRules(const QVariantList& backupRules) {
+    qCDebug(domain_server) << "BACKUP RULES:";
+
+    for (const QVariant& value : backupRules) {
+        QVariantMap map = value.toMap();
+
+        int interval = map["backupInterval"].toInt();
+        int count = map["maxBackupVersions"].toInt();
+        auto name = map["Name"].toString();
+        auto format = name.toLower();
+        QRegExp matchDisallowedCharacters { "[^a-zA-Z0-9\\-_]+" };
+        format.replace(matchDisallowedCharacters, "_");
+
+        qCDebug(domain_server) << "    Name:" << name;
+        qCDebug(domain_server) << "        format:" << format;
+        qCDebug(domain_server) << "        interval:" << interval;
+        qCDebug(domain_server) << "        count:" << count;
+
+        BackupRule newRule = { name, interval, format, count, 0 };
+
+        newRule.lastBackupSeconds = getMostRecentBackupTimeInSecs(format);
+
+        if (newRule.lastBackupSeconds > 0) {
+            auto now = QDateTime::currentSecsSinceEpoch();
+            auto sinceLastBackup = now - newRule.lastBackupSeconds;
+            qCDebug(domain_server).noquote() << "        lastBackup:" <<  formatSecTime(sinceLastBackup) << "ago";
+        } else {
+            qCDebug(domain_server) << "        lastBackup: NEVER";
+        }
+
+        _backupRules.push_back(newRule);
+    }
+}
+
+void DomainContentBackupManager::refreshBackupRules() {
+    for (auto& backup : _backupRules) {
+        backup.lastBackupSeconds = getMostRecentBackupTimeInSecs(backup.extensionFormat);
+    }
+}
+
+int64_t DomainContentBackupManager::getMostRecentBackupTimeInSecs(const QString& format) {
+    int64_t mostRecentBackupInSecs = 0;
+
+    QString mostRecentBackupFileName;
+    QDateTime mostRecentBackupTime;
+
+    bool recentBackup = getMostRecentBackup(format, mostRecentBackupFileName, mostRecentBackupTime);
+
+    if (recentBackup) {
+        mostRecentBackupInSecs = mostRecentBackupTime.toSecsSinceEpoch();
+    }
+
+    return mostRecentBackupInSecs;
+}
+
+void DomainContentBackupManager::setup() {
+    auto backups = getAllBackups();
+    for (auto& backup : backups) {
+        QFile backupFile { backup.absolutePath };
+        if (!backupFile.open(QIODevice::ReadOnly)) {
+            qCritical() << "Could not open file:" << backup.absolutePath;
+            qCritical() << "    ERROR:" << backupFile.errorString();
+            continue;
+        }
+
+        QuaZip zip { &backupFile };
+        if (!zip.open(QuaZip::mdUnzip)) {
+            qCritical() << "Could not open backup archive:" << backup.absolutePath;
+            qCritical() << "    ERROR:" << zip.getZipError();
+            continue;
+        }
+
+        for (auto& handler : _backupHandlers) {
+            handler->loadBackup(backup.id, zip);
+        }
+
+        zip.close();
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->loadingComplete();
+    }
+}
+
+bool DomainContentBackupManager::process() {
+    if (isStillRunning()) {
+        constexpr int64_t MSECS_TO_USECS = 1000;
+        constexpr int64_t USECS_TO_SLEEP = 10 * MSECS_TO_USECS;  // every 10ms
+        std::this_thread::sleep_for(std::chrono::microseconds(USECS_TO_SLEEP));
+
+        if (_isRecovering) {
+            bool isStillRecovering = any_of(begin(_backupHandlers), end(_backupHandlers), [](const BackupHandlerPointer& handler) {
+                return handler->getRecoveryStatus().first;
+            });
+
+            if (!isStillRecovering) {
+                _isRecovering = false;
+                _recoveryFilename = "";
+                emit recoveryCompleted();
+            }
+        }
+
+        auto now = p_high_resolution_clock::now();
+        auto sinceLastSave = now - _lastCheck;
+        if (sinceLastSave > _persistInterval) {
+            _lastCheck = now;
+
+            if (!_isRecovering) {
+                backup();
+            }
+        }
+    }
+
+    return isStillRunning();
+}
+
+void DomainContentBackupManager::shutdown() {
+    // Destroy handlers on the correct thread so that they can cleanup timers
+    _backupHandlers.clear();
+}
+
+void DomainContentBackupManager::aboutToFinish() {
+    _stopThread = true;
+}
+
+bool DomainContentBackupManager::getMostRecentBackup(const QString& format,
+                                                     QString& mostRecentBackupFileName,
+                                                     QDateTime& mostRecentBackupTime) {
+    QRegExp formatRE { AUTOMATIC_BACKUP_PREFIX + QRegExp::escape(format) + "\\-(" + DATETIME_FORMAT_RE + ")" + "\\.zip" };
+
+    QStringList filters;
+    filters << AUTOMATIC_BACKUP_PREFIX + format + "*.zip";
+
+    bool bestBackupFound = false;
+    QString bestBackupFile;
+    QDateTime bestBackupFileTime;
+
+    // Iterate over all of the backup files in the persist location
+    QDirIterator dirIterator(_backupDirectory, filters, QDir::Files | QDir::NoSymLinks, QDirIterator::NoIteratorFlags);
+    while (dirIterator.hasNext()) {
+        dirIterator.next();
+        auto fileName = dirIterator.fileInfo().fileName();
+
+        if (formatRE.exactMatch(fileName)) {
+            auto datetime = formatRE.cap(1);
+            auto createdAt = QDateTime::fromString(datetime, DATETIME_FORMAT);
+
+            if (!createdAt.isValid()) {
+                qDebug() << "Skipping backup with invalid timestamp: " << datetime;
+                continue;
+            }
+
+            qDebug() << "Checking " << dirIterator.fileInfo().filePath();
+
+            // Based on last modified date, track the most recently modified file as the best backup
+            if (createdAt > bestBackupFileTime) {
+                bestBackupFound = true;
+                bestBackupFile = dirIterator.filePath();
+                bestBackupFileTime = createdAt;
+            }
+        } else {
+            qDebug() << "NO match: " << fileName << formatRE;
+        }
+    }
+
+    // If we found a backup then return the results
+    if (bestBackupFound) {
+        mostRecentBackupFileName = bestBackupFile;
+        mostRecentBackupTime = bestBackupFileTime;
+    }
+    return bestBackupFound;
+}
+
+void DomainContentBackupManager::deleteBackup(MiniPromise::Promise promise, const QString& backupName) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "deleteBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, backupName));
+        return;
+    }
+
+    if (_isRecovering && backupName == _recoveryFilename) {
+        promise->resolve({
+            { "success", false }
+        });
+        return;
+    }
+
+    QDir backupDir { _backupDirectory };
+    QFile backupFile { backupDir.filePath(backupName) };
+    auto success = backupFile.remove();
+
+    refreshBackupRules();
+
+    for (auto& handler : _backupHandlers) {
+        handler->deleteBackup(backupName);
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+bool DomainContentBackupManager::recoverFromBackupZip(const QString& backupName, QuaZip& zip) {
+    if (!zip.open(QuaZip::Mode::mdUnzip)) {
+        qWarning() << "Failed to unzip file: " << backupName;
+        return false;
+    } else {
+        _isRecovering = true;
+        _recoveryFilename = backupName;
+
+        for (auto& handler : _backupHandlers) {
+            handler->recoverBackup(backupName, zip);
+        }
+
+        qDebug() << "Successfully started recovering from " << backupName;
+        return true;
+    }
+}
+
+void DomainContentBackupManager::recoverFromBackup(MiniPromise::Promise promise, const QString& backupName) {
+    if (_isRecovering) {
+        promise->resolve({
+            { "success", false }
+        });
+        return;
+    };
+
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "recoverFromBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, backupName));
+        return;
+    }
+
+    qDebug() << "Recovering from" << backupName;
+
+    bool success { false };
+    QDir backupDir { _backupDirectory };
+    auto backupFilePath { backupDir.filePath(backupName) };
+    QFile backupFile { backupFilePath };
+    if (backupFile.open(QIODevice::ReadOnly)) {
+        QuaZip zip { &backupFile };
+
+        success = recoverFromBackupZip(backupName, zip);
+
+        backupFile.close();
+    } else {
+        success = false;
+        qWarning() << "Failed to open backup file for reading: " << backupFilePath;
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+void DomainContentBackupManager::recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup) {
+
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "recoverFromUploadedBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(QByteArray, uploadedBackup));
+        return;
+    }
+
+    qDebug() << "Recovering from uploaded content archive";
+
+    // create a buffer and then a QuaZip from that buffer
+    QBuffer uploadedBackupBuffer { &uploadedBackup };
+    QuaZip uploadedZip { &uploadedBackupBuffer };
+
+    QString backupName = MANUAL_BACKUP_PREFIX + "uploaded.zip";
+    bool success = recoverFromBackupZip(backupName, uploadedZip);
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+std::vector<BackupItemInfo> DomainContentBackupManager::getAllBackups() {
+
+    QDir backupDir { _backupDirectory };
+    auto matchingFiles =
+            backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + "*.zip", MANUAL_BACKUP_PREFIX + "*.zip" },
+                                    QDir::Files | QDir::NoSymLinks, QDir::Name);
+    QString prefixFormat = "(" + QRegExp::escape(AUTOMATIC_BACKUP_PREFIX) + "|" + QRegExp::escape(MANUAL_BACKUP_PREFIX) + ")";
+    QString nameFormat = "(.+)";
+    QString dateTimeFormat = "(" + DATETIME_FORMAT_RE + ")";
+    QRegExp backupNameFormat { prefixFormat + nameFormat + "-" + dateTimeFormat + "\\.zip" };
+
+    std::vector<BackupItemInfo> backups;
+
+    for (const auto& fileInfo : matchingFiles) {
+        auto fileName = fileInfo.fileName();
+        if (backupNameFormat.exactMatch(fileName)) {
+            auto type = backupNameFormat.cap(1);
+            auto name = backupNameFormat.cap(2);
+            auto dateTime = backupNameFormat.cap(3);
+            auto createdAt = QDateTime::fromString(dateTime, DATETIME_FORMAT);
+            if (!createdAt.isValid()) {
+                qDebug().nospace() << "Skipping backup (" << fileName << ") with invalid timestamp: " << dateTime;
+                continue;
+            }
+
+            backups.emplace_back(fileInfo.fileName(), name, fileInfo.absoluteFilePath(), createdAt,
+                                 type == MANUAL_BACKUP_PREFIX);
+        }
+    }
+
+    return backups;
+}
+
+void DomainContentBackupManager::getAllBackupsAndStatus(MiniPromise::Promise promise) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "getAllBackupsAndStatus", Q_ARG(MiniPromise::Promise, promise));
+        return;
+    }
+
+    auto backups = getAllBackups();
+
+    QVariantList variantBackups;
+
+    for (auto& backup : backups) {
+        bool isAvailable { true };
+        bool isCorrupted { false };
+        float availabilityProgress { 0.0f };
+        for (auto& handler : _backupHandlers) {
+            bool handlerIsAvailable { true };
+            float progress { 0.0f };
+            std::tie(handlerIsAvailable, progress) = handler->isAvailable(backup.id);
+            isAvailable &= handlerIsAvailable;
+            availabilityProgress += progress / _backupHandlers.size();
+
+            isCorrupted = isCorrupted || handler->isCorruptedBackup(backup.id);
+        }
+        variantBackups.push_back(QVariantMap({
+            { "id", backup.id },
+            { "name", backup.name },
+            { "createdAtMillis", backup.createdAt.toMSecsSinceEpoch() },
+            { "isAvailable", isAvailable },
+            { "availabilityProgress", availabilityProgress },
+            { "isManualBackup", backup.isManualBackup },
+            { "isCorrupted", isCorrupted }
+        }));
+    }
+
+    float recoveryProgress = 0.0f;
+    bool isRecovering = _isRecovering.load();
+    if (_isRecovering) {
+        for (auto& handler : _backupHandlers) {
+            float progress = handler->getRecoveryStatus().second;
+            recoveryProgress += progress / _backupHandlers.size();
+        }
+    }
+
+    QVariantMap status { 
+        { "isRecovering", isRecovering },
+        { "recoveringBackupId", _recoveryFilename },
+        { "recoveryProgress", recoveryProgress }
+    };
+
+    QVariantMap info {
+        { "backups", variantBackups },
+        { "status", status }
+    };
+
+    promise->resolve(info);
+}
+
+void DomainContentBackupManager::removeOldBackupVersions(const BackupRule& rule) {
+    QDir backupDir { _backupDirectory };
+    if (backupDir.exists() && rule.maxBackupVersions > 0) {
+        qCDebug(domain_server) << "Rolling old backup versions for rule" << rule.name;
+
+        auto matchingFiles =
+                backupDir.entryInfoList({ AUTOMATIC_BACKUP_PREFIX + rule.extensionFormat + "*.zip" }, QDir::Files | QDir::NoSymLinks, QDir::Name);
+
+        int backupsToDelete = matchingFiles.length() - rule.maxBackupVersions;
+        if (backupsToDelete <= 0) {
+            qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, no backups need to be deleted";
+        } else {
+            qCDebug(domain_server) << "Found" << matchingFiles.length() << "backups, deleting " << backupsToDelete << "backup(s)";
+            for (int i = 0; i < backupsToDelete; ++i) {
+                auto fileInfo = matchingFiles[i].absoluteFilePath();
+                QFile backupFile(fileInfo);
+                if (backupFile.remove()) {
+                    qCDebug(domain_server) << "Removed old backup: " << backupFile.fileName();
+                } else {
+                    qCDebug(domain_server) << "Failed to remove old backup: " << backupFile.fileName();
+                }
+            }
+            qCDebug(domain_server) << "Done removing old backup versions";
+        }
+    } else {
+        qCDebug(domain_server) << "Rolling backups for rule" << rule.name << "."
+                                << " Max Rolled Backup Versions less than 1 [" << rule.maxBackupVersions << "]."
+                                << " No need to roll backups";
+    }
+}
+
+void DomainContentBackupManager::backup() {
+    auto nowDateTime = QDateTime::currentDateTime();
+    auto nowSeconds = nowDateTime.toSecsSinceEpoch();
+
+    for (BackupRule& rule : _backupRules) {
+        auto secondsSinceLastBackup = nowSeconds - rule.lastBackupSeconds;
+
+        qCDebug(domain_server) << "Checking [" << rule.name << "] - Time since last backup [" << secondsSinceLastBackup
+                                << "] "
+                                << "compared to backup interval [" << rule.intervalSeconds << "]...";
+
+        if (secondsSinceLastBackup > rule.intervalSeconds) {
+            qCDebug(domain_server) << "Time since last backup [" << secondsSinceLastBackup << "] for rule [" << rule.name
+                                    << "] exceeds backup interval [" << rule.intervalSeconds << "] doing backup now...";
+
+            bool success;
+            QString path;
+            std::tie(success, path) =  createBackup(AUTOMATIC_BACKUP_PREFIX, rule.extensionFormat);
+            if (!success) {
+                qCWarning(domain_server) << "Failed to create backup for" << rule.name << "at" << path;
+                continue;
+            }
+
+            qDebug() << "Created backup: " << path;
+
+            rule.lastBackupSeconds = nowSeconds;
+
+            removeOldBackupVersions(rule);
+        } else {
+            qCDebug(domain_server) << "Backup not needed for this rule [" << rule.name << "]...";
+        }
+    }
+}
+
+void DomainContentBackupManager::consolidateBackup(MiniPromise::Promise promise, QString fileName) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "consolidateBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(QString, fileName));
+        return;
+    }
+
+    QDir backupDir { _backupDirectory };
+    if (!backupDir.exists()) {
+        qCritical() << "Backup directory does not exist, bailing consolidation of backup";
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    auto filePath = backupDir.absoluteFilePath(fileName);
+
+    auto copyFilePath = QDir::tempPath() + "/" + fileName;
+
+    {
+        QFile copyFile(copyFilePath);
+        copyFile.remove();
+        copyFile.close();
+    }
+    auto copySuccess = QFile::copy(filePath, copyFilePath);
+    if (!copySuccess) {
+        qCritical() << "Failed to create copy of backup.";
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    QuaZip zip(copyFilePath);
+    if (!zip.open(QuaZip::mdAdd)) {
+        qCritical() << "Could not open backup archive:" << filePath;
+        qCritical() << "    ERROR:" << zip.getZipError();
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->consolidateBackup(fileName, zip);
+    }
+
+    zip.close();
+
+    if (zip.getZipError() != UNZ_OK) {
+        qCritical() << "Failed to consolidate backup: " << zip.getZipError();
+        promise->resolve({ { "success", false } });
+        return;
+    }
+
+    promise->resolve({
+        { "success", true },
+        { "backupFilePath", copyFilePath }
+    });
+}
+
+void DomainContentBackupManager::createManualBackup(MiniPromise::Promise promise, const QString& name) {
+    if (QThread::currentThread() != thread()) {
+        QMetaObject::invokeMethod(this, "createManualBackup", Q_ARG(MiniPromise::Promise, promise),
+                                  Q_ARG(const QString&, name));
+        return;
+    }
+
+
+    QRegExp nameRE { MANUAL_BACKUP_NAME_RE };
+    bool success;
+
+    if (!nameRE.exactMatch(name)) {
+        qDebug() << "Cannot create manual backup with invalid name: " << name;
+        success = false;
+    } else {
+        QString path;
+        std::tie(success, path) = createBackup(MANUAL_BACKUP_PREFIX, name);
+    }
+
+    promise->resolve({
+        { "success", success }
+    });
+}
+
+std::pair<bool, QString> DomainContentBackupManager::createBackup(const QString& prefix, const QString& name) {
+    auto timestamp = QDateTime::currentDateTime().toString(DATETIME_FORMAT);
+    auto fileName = prefix + name + "-" + timestamp + ".zip";
+    auto path = _backupDirectory + "/" + fileName;
+    QuaZip zip(path);
+    if (!zip.open(QuaZip::mdAdd)) {
+        qCWarning(domain_server) << "Failed to open zip file at " << path;
+        qCWarning(domain_server) << "    ERROR:" << zip.getZipError();
+        return { false, path };
+    }
+
+    for (auto& handler : _backupHandlers) {
+        handler->createBackup(fileName, zip);
+    }
+
+    zip.close();
+
+    return { true, path };
+}

domain-server/src/DomainContentBackupManager.h

@@ -0,0 +1,106 @@
+//
+//  DomainContentBackupManager.h
+//  libraries/domain-server/src
+//
+//  Created by Ryan Huffman on 1/01/18.
+//  Adapted from OctreePersistThread
+//  Copyright 2018 High Fidelity, Inc.
+//
+//
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_DomainContentBackupManager_h
+#define hifi_DomainContentBackupManager_h
+
+#include <QString>
+#include <QVector>
+#include <QDateTime>
+
+#include <GenericThread.h>
+
+#include "BackupHandler.h"
+
+#include <shared/MiniPromises.h>
+
+#include <PortableHighResolutionClock.h>
+
+struct BackupItemInfo {
+    BackupItemInfo(QString pId, QString pName, QString pAbsolutePath, QDateTime pCreatedAt, bool pIsManualBackup) :
+        id(pId), name(pName), absolutePath(pAbsolutePath), createdAt(pCreatedAt), isManualBackup(pIsManualBackup) { };
+
+    QString id;
+    QString name;
+    QString absolutePath;
+    QDateTime createdAt;
+    bool isManualBackup;
+};
+
+class DomainContentBackupManager : public GenericThread {
+    Q_OBJECT
+public:
+    class BackupRule {
+    public:
+        QString name;
+        int intervalSeconds;
+        QString extensionFormat;
+        int maxBackupVersions;
+        qint64 lastBackupSeconds;
+    };
+
+    static const std::chrono::seconds DEFAULT_PERSIST_INTERVAL;
+
+    DomainContentBackupManager(const QString& rootBackupDirectory,
+                               const QVariantList& settings,
+                               std::chrono::milliseconds persistInterval = DEFAULT_PERSIST_INTERVAL,
+                               bool debugTimestampNow = false);
+
+    std::vector<BackupItemInfo> getAllBackups();
+    void addBackupHandler(BackupHandlerPointer handler);
+    void aboutToFinish();  /// call this to inform the persist thread that the owner is about to finish to support final persist
+    void replaceData(QByteArray data);
+
+public slots:
+    void getAllBackupsAndStatus(MiniPromise::Promise promise);
+    void createManualBackup(MiniPromise::Promise promise, const QString& name);
+    void recoverFromBackup(MiniPromise::Promise promise, const QString& backupName);
+    void recoverFromUploadedBackup(MiniPromise::Promise promise, QByteArray uploadedBackup);
+    void deleteBackup(MiniPromise::Promise promise, const QString& backupName);
+    void consolidateBackup(MiniPromise::Promise promise, QString fileName);
+
+signals:
+    void loadCompleted();
+    void recoveryCompleted();
+
+protected:
+    /// Implements generic processing behavior for this thread.
+    virtual void setup() override;
+    virtual bool process() override;
+    virtual void shutdown() override;
+
+    void backup();
+    void removeOldBackupVersions(const BackupRule& rule);
+    void refreshBackupRules();
+    bool getMostRecentBackup(const QString& format, QString& mostRecentBackupFileName, QDateTime& mostRecentBackupTime);
+    int64_t getMostRecentBackupTimeInSecs(const QString& format);
+    void parseBackupRules(const QVariantList& backupRules);
+
+    std::pair<bool, QString> createBackup(const QString& prefix, const QString& name);
+
+    bool recoverFromBackupZip(const QString& backupName, QuaZip& backupZip);
+
+private:
+    const QString _backupDirectory;
+    std::vector<BackupHandlerPointer> _backupHandlers;
+    std::chrono::milliseconds _persistInterval { 0 };
+
+    std::atomic<bool> _isRecovering { false };
+    QString _recoveryFilename { };
+
+    p_high_resolution_clock::time_point _lastCheck;
+    std::vector<BackupRule> _backupRules;
+};
+
+#endif  // hifi_DomainContentBackupManager_h

domain-server/src/DomainGatekeeper.cpp

@@ -435,10 +435,11 @@ SharedNodePointer DomainGatekeeper::processAgentConnectRequest(const NodeConnect
     if (!userPerms.can(NodePermissions::Permission::canConnectPastMaxCapacity) && !isWithinMaxCapacity()) {
         // we can't allow this user to connect because we are at max capacity
         QString redirectOnMaxCapacity;
-        const QVariant* redirectOnMaxCapacityVariant =
-            valueForKeyPath(_server->_settingsManager.getSettingsMap(), MAXIMUM_USER_CAPACITY_REDIRECT_LOCATION);
-        if (redirectOnMaxCapacityVariant && redirectOnMaxCapacityVariant->canConvert<QString>()) {
-            redirectOnMaxCapacity = redirectOnMaxCapacityVariant->toString();
+
+        QVariant redirectOnMaxCapacityVariant =
+            _server->_settingsManager.valueForKeyPath(MAXIMUM_USER_CAPACITY_REDIRECT_LOCATION);
+        if (redirectOnMaxCapacityVariant.canConvert<QString>()) {
+            redirectOnMaxCapacity = redirectOnMaxCapacityVariant.toString();
             qDebug() << "Redirection domain:" << redirectOnMaxCapacity;
         }
 
@@ -610,9 +611,9 @@ bool DomainGatekeeper::verifyUserSignature(const QString& username,
 
 bool DomainGatekeeper::isWithinMaxCapacity() {
     // find out what our maximum capacity is
-    const QVariant* maximumUserCapacityVariant =
-        valueForKeyPath(_server->_settingsManager.getSettingsMap(), MAXIMUM_USER_CAPACITY);
-    unsigned int maximumUserCapacity = maximumUserCapacityVariant ? maximumUserCapacityVariant->toUInt() : 0;
+    QVariant maximumUserCapacityVariant =
+        _server->_settingsManager.valueForKeyPath(MAXIMUM_USER_CAPACITY);
+    unsigned int maximumUserCapacity = maximumUserCapacityVariant.isValid() ? maximumUserCapacityVariant.toUInt() : 0;
 
     if (maximumUserCapacity > 0) {
         unsigned int connectedUsers = _server->countConnectedUsers();

domain-server/src/DomainMetadata.cpp

@@ -84,21 +84,22 @@ void DomainMetadata::descriptorsChanged() {
     // get descriptors
     assert(_metadata[DESCRIPTORS].canConvert<QVariantMap>());
     auto& state = *static_cast<QVariantMap*>(_metadata[DESCRIPTORS].data());
-    auto& settings = static_cast<DomainServer*>(parent())->_settingsManager.getSettingsMap();
-    auto& descriptors = static_cast<DomainServer*>(parent())->_settingsManager.getDescriptorsMap();
+
+    static const QString DESCRIPTORS_GROUP_KEYPATH = "descriptors";
+    auto descriptorsMap = static_cast<DomainServer*>(parent())->_settingsManager.valueForKeyPath(DESCRIPTORS).toMap();
 
     // copy simple descriptors (description/maturity)
-    state[Descriptors::DESCRIPTION] = descriptors[Descriptors::DESCRIPTION]; 
-    state[Descriptors::MATURITY] = descriptors[Descriptors::MATURITY];
+    state[Descriptors::DESCRIPTION] = descriptorsMap[Descriptors::DESCRIPTION];
+    state[Descriptors::MATURITY] = descriptorsMap[Descriptors::MATURITY];
 
     // copy array descriptors (hosts/tags)
-    state[Descriptors::HOSTS] = descriptors[Descriptors::HOSTS].toList();
-    state[Descriptors::TAGS] = descriptors[Descriptors::TAGS].toList();
+    state[Descriptors::HOSTS] = descriptorsMap[Descriptors::HOSTS].toList();
+    state[Descriptors::TAGS] = descriptorsMap[Descriptors::TAGS].toList();
 
     // parse capacity
     static const QString CAPACITY = "security.maximum_user_capacity";
-    const QVariant* capacityVariant = valueForKeyPath(settings, CAPACITY);
-    unsigned int capacity = capacityVariant ? capacityVariant->toUInt() : 0;
+    QVariant capacityVariant = static_cast<DomainServer*>(parent())->_settingsManager.valueForKeyPath(CAPACITY);
+    unsigned int capacity = capacityVariant.isValid() ? capacityVariant.toUInt() : 0;
     state[Descriptors::CAPACITY] = capacity;
 
 #if DEV_BUILD || PR_BUILD

domain-server/src/DomainServer.cpp

@@ -24,6 +24,7 @@
 #include <QTimer>
 #include <QUrlQuery>
 #include <QCommandLineParser>
+#include <QUuid>
 
 #include <AccountManager.h>
 #include <AssetClient.h>
@@ -44,10 +45,20 @@
 #include <Trace.h>
 #include <StatTracker.h>
 
+#include "AssetsBackupHandler.h"
+#include "ContentSettingsBackupHandler.h"
 #include "DomainServerNodeData.h"
+#include "EntitiesBackupHandler.h"
 #include "NodeConnectionData.h"
 
+#include <Gzip.h>
+
+#include <OctreeDataUtils.h>
+
+Q_LOGGING_CATEGORY(domain_server, "hifi.domain_server")
+
 const QString ACCESS_TOKEN_KEY_PATH = "metaverse.access_token";
+const QString DomainServer::REPLACEMENT_FILE_EXTENSION = ".replace";
 
 int const DomainServer::EXIT_CODE_REBOOT = 234923;
 
@@ -64,8 +75,8 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection,
                                               std::initializer_list<QString> optionalData,
                                               bool requireAccessToken) {
 
-    auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-    if (accessTokenVariant == nullptr && requireAccessToken) {
+    auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+    if (!accessTokenVariant.isValid() && requireAccessToken) {
         connection->respond(HTTPConnection::StatusCode400, "User access token has not been set");
         return true;
     }
@@ -101,8 +112,8 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection,
     req.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
     req.setHeader(QNetworkRequest::ContentTypeHeader, "application/json");
 
-    if (accessTokenVariant != nullptr) {
-        auto accessTokenHeader = QString("Bearer ") + accessTokenVariant->toString();
+    if (accessTokenVariant.isValid()) {
+        auto accessTokenHeader = QString("Bearer ") + accessTokenVariant.toString();
         req.setRawHeader("Authorization", accessTokenHeader.toLatin1());
     }
 
@@ -280,6 +291,27 @@ DomainServer::DomainServer(int argc, char* argv[]) :
             qDebug() << "Ignoring subnet in whitelist, invalid ip portion: " << subnet;
         }
     }
+
+    if (QDir(getEntitiesDirPath()).mkpath(".")) {
+        qCDebug(domain_server) << "Created entities data directory";
+    }
+    maybeHandleReplacementEntityFile();
+
+
+    static const QString BACKUP_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
+    auto backupRulesVariant = _settingsManager.valueOrDefaultValueForKeyPath(BACKUP_RULES_KEYPATH);
+
+    _contentManager.reset(new DomainContentBackupManager(getContentBackupDir(), backupRulesVariant.toList()));
+
+    connect(_contentManager.get(), &DomainContentBackupManager::started, _contentManager.get(), [this](){
+        _contentManager->addBackupHandler(BackupHandlerPointer(new EntitiesBackupHandler(getEntitiesFilePath(), getEntitiesReplacementFilePath())));
+        _contentManager->addBackupHandler(BackupHandlerPointer(new AssetsBackupHandler(getContentBackupDir())));
+        _contentManager->addBackupHandler(BackupHandlerPointer(new ContentSettingsBackupHandler(_settingsManager)));
+    });
+
+    _contentManager->initialize(true);
+
+    connect(_contentManager.get(), &DomainContentBackupManager::recoveryCompleted, this, &DomainServer::restart);
 }
 
 void DomainServer::parseCommandLine() {
@@ -345,6 +377,11 @@ void DomainServer::parseCommandLine() {
 DomainServer::~DomainServer() {
     qInfo() << "Domain Server is shutting down.";
 
+    if (_contentManager) {
+        _contentManager->aboutToFinish();
+        _contentManager->terminate();
+    }
+
     // cleanup the AssetClient thread
     DependencyManager::destroy<AssetClient>();
     _assetClientThread.quit();
@@ -377,8 +414,8 @@ bool DomainServer::optionallyReadX509KeyAndCertificate() {
     const QString X509_PRIVATE_KEY_OPTION = "key";
     const QString X509_KEY_PASSPHRASE_ENV = "DOMAIN_SERVER_KEY_PASSPHRASE";
 
-    QString certPath = _settingsManager.getSettingsMap().value(X509_CERTIFICATE_OPTION).toString();
-    QString keyPath = _settingsManager.getSettingsMap().value(X509_PRIVATE_KEY_OPTION).toString();
+    QString certPath = _settingsManager.valueForKeyPath(X509_CERTIFICATE_OPTION).toString();
+    QString keyPath = _settingsManager.valueForKeyPath(X509_PRIVATE_KEY_OPTION).toString();
 
     if (!certPath.isEmpty() && !keyPath.isEmpty()) {
         // the user wants to use the following cert and key for HTTPS
@@ -421,8 +458,7 @@ bool DomainServer::optionallySetupOAuth() {
     const QString OAUTH_CLIENT_SECRET_ENV = "DOMAIN_SERVER_CLIENT_SECRET";
     const QString REDIRECT_HOSTNAME_OPTION = "hostname";
 
-    const QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-    _oauthProviderURL = QUrl(settingsMap.value(OAUTH_PROVIDER_URL_OPTION).toString());
+    _oauthProviderURL = QUrl(_settingsManager.valueForKeyPath(OAUTH_PROVIDER_URL_OPTION).toString());
 
     // if we don't have an oauth provider URL then we default to the default node auth url
     if (_oauthProviderURL.isEmpty()) {
@@ -432,9 +468,9 @@ bool DomainServer::optionallySetupOAuth() {
     auto accountManager = DependencyManager::get<AccountManager>();
     accountManager->setAuthURL(_oauthProviderURL);
 
-    _oauthClientID = settingsMap.value(OAUTH_CLIENT_ID_OPTION).toString();
+    _oauthClientID = _settingsManager.valueForKeyPath(OAUTH_CLIENT_ID_OPTION).toString();
     _oauthClientSecret = QProcessEnvironment::systemEnvironment().value(OAUTH_CLIENT_SECRET_ENV);
-    _hostname = settingsMap.value(REDIRECT_HOSTNAME_OPTION).toString();
+    _hostname = _settingsManager.valueForKeyPath(REDIRECT_HOSTNAME_OPTION).toString();
 
     if (!_oauthClientID.isEmpty()) {
         if (_oauthProviderURL.isEmpty()
@@ -459,11 +495,11 @@ static const QString METAVERSE_DOMAIN_ID_KEY_PATH = "metaverse.id";
 
 void DomainServer::getTemporaryName(bool force) {
     // check if we already have a domain ID
-    const QVariant* idValueVariant = valueForKeyPath(_settingsManager.getSettingsMap(), METAVERSE_DOMAIN_ID_KEY_PATH);
+    QVariant idValueVariant = _settingsManager.valueForKeyPath(METAVERSE_DOMAIN_ID_KEY_PATH);
 
     qInfo() << "Requesting temporary domain name";
-    if (idValueVariant) {
-        qDebug() << "A domain ID is already present in domain-server settings:" << idValueVariant->toString();
+    if (idValueVariant.isValid()) {
+        qDebug() << "A domain ID is already present in domain-server settings:" << idValueVariant.toString();
         if (force) {
             qDebug() << "Requesting temporary domain name to replace current ID:" << getID();
         } else {
@@ -503,9 +539,6 @@ void DomainServer::handleTempDomainSuccess(QNetworkReply& requestReply) {
         auto settingsDocument = QJsonDocument::fromJson(newSettingsJSON.toUtf8());
         _settingsManager.recurseJSONObjectAndOverwriteSettings(settingsDocument.object(), DomainSettings);
 
-        // store the new ID and auto networking setting on disk
-        _settingsManager.persistToFile();
-
         // store the new token to the account info
         auto accountManager = DependencyManager::get<AccountManager>();
         accountManager->setTemporaryDomain(id, key);
@@ -607,8 +640,6 @@ void DomainServer::setupNodeListAndAssignments() {
     QVariant localPortValue = _settingsManager.valueOrDefaultValueForKeyPath(CUSTOM_LOCAL_PORT_OPTION);
     int domainServerPort = localPortValue.toInt();
 
-    QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-
     int domainServerDTLSPort = INVALID_PORT;
 
     if (_isUsingDTLS) {
@@ -616,8 +647,9 @@ void DomainServer::setupNodeListAndAssignments() {
 
         const QString CUSTOM_DTLS_PORT_OPTION = "dtls-port";
 
-        if (settingsMap.contains(CUSTOM_DTLS_PORT_OPTION)) {
-            domainServerDTLSPort = (unsigned short) settingsMap.value(CUSTOM_DTLS_PORT_OPTION).toUInt();
+        auto dtlsPortVariant = _settingsManager.valueForKeyPath(CUSTOM_DTLS_PORT_OPTION);
+        if (dtlsPortVariant.isValid()) {
+            domainServerDTLSPort = (unsigned short) dtlsPortVariant.toUInt();
         }
     }
 
@@ -647,9 +679,9 @@ void DomainServer::setupNodeListAndAssignments() {
         nodeList->setSessionUUID(_overridingDomainID);
         isMetaverseDomain = true; // assume metaverse domain
     } else {
-        const QVariant* idValueVariant = valueForKeyPath(settingsMap, METAVERSE_DOMAIN_ID_KEY_PATH);
-        if (idValueVariant) {
-            nodeList->setSessionUUID(idValueVariant->toString());
+        QVariant idValueVariant = _settingsManager.valueForKeyPath(METAVERSE_DOMAIN_ID_KEY_PATH);
+        if (idValueVariant.isValid()) {
+            nodeList->setSessionUUID(idValueVariant.toString());
             isMetaverseDomain = true; // if we have an ID, we'll assume we're a metaverse domain
         } else {
             nodeList->setSessionUUID(QUuid::createUuid()); // Use random UUID
@@ -691,13 +723,18 @@ void DomainServer::setupNodeListAndAssignments() {
     packetReceiver.registerListener(PacketType::ICEServerHeartbeatDenied, this, "processICEServerHeartbeatDenialPacket");
     packetReceiver.registerListener(PacketType::ICEServerHeartbeatACK, this, "processICEServerHeartbeatACK");
 
+    packetReceiver.registerListener(PacketType::OctreeDataFileRequest, this, "processOctreeDataRequestMessage");
+    packetReceiver.registerListener(PacketType::OctreeDataPersist, this, "processOctreeDataPersistMessage");
+
+    packetReceiver.registerListener(PacketType::OctreeFileReplacement, this, "handleOctreeFileReplacementRequest");
+    packetReceiver.registerListener(PacketType::OctreeFileReplacementFromUrl, this, "handleOctreeFileReplacementFromURLRequest");
+
     // set a custom packetVersionMatch as the verify packet operator for the udt::Socket
     nodeList->setPacketFilterOperator(&DomainServer::isPacketVerified);
 
     _assetClientThread.setObjectName("AssetClient Thread");
     auto assetClient = DependencyManager::set<AssetClient>();
     assetClient->moveToThread(&_assetClientThread);
-    QObject::connect(&_assetClientThread, &QThread::started, assetClient.data(), &AssetClient::init);
     _assetClientThread.start();
 
     // add whatever static assignments that have been parsed to the queue
@@ -712,10 +749,10 @@ bool DomainServer::resetAccountManagerAccessToken() {
         QString accessToken = QProcessEnvironment::systemEnvironment().value(ENV_ACCESS_TOKEN_KEY);
 
         if (accessToken.isEmpty()) {
-            const QVariant* accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
+            QVariant accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
 
-            if (accessTokenVariant && accessTokenVariant->canConvert(QMetaType::QString)) {
-                accessToken = accessTokenVariant->toString();
+            if (accessTokenVariant.canConvert(QMetaType::QString)) {
+                accessToken = accessTokenVariant.toString();
             } else {
                 qWarning() << "No access token is present. Some operations that use the metaverse API will fail.";
                 qDebug() << "Set an access token via the web interface, in your user config"
@@ -846,31 +883,26 @@ void DomainServer::updateICEServerAddresses() {
 }
 
 void DomainServer::parseAssignmentConfigs(QSet<Assignment::Type>& excludedTypes) {
-    const QString ASSIGNMENT_CONFIG_REGEX_STRING = "config-([\\d]+)";
-    QRegExp assignmentConfigRegex(ASSIGNMENT_CONFIG_REGEX_STRING);
-
-    const QVariantMap& settingsMap = _settingsManager.getSettingsMap();
+    const QString ASSIGNMENT_CONFIG_PREFIX = "config-";
 
     // scan for assignment config keys
-    QStringList variantMapKeys = settingsMap.keys();
-    int configIndex = variantMapKeys.indexOf(assignmentConfigRegex);
+    for (int i = 0; i < Assignment::AllTypes; ++i) {
+        QVariant assignmentConfigVariant = _settingsManager.valueOrDefaultValueForKeyPath(ASSIGNMENT_CONFIG_PREFIX + QString::number(i));
 
-    while (configIndex != -1) {
-        // figure out which assignment type this matches
-        Assignment::Type assignmentType = (Assignment::Type) assignmentConfigRegex.cap(1).toInt();
+        if (assignmentConfigVariant.isValid()) {
+            // figure out which assignment type this matches
+            Assignment::Type assignmentType = static_cast<Assignment::Type>(i);
 
-        if (assignmentType < Assignment::AllTypes && !excludedTypes.contains(assignmentType)) {
-            QVariant mapValue = settingsMap[variantMapKeys[configIndex]];
-            QVariantList assignmentList = mapValue.toList();
+            if (!excludedTypes.contains(assignmentType)) {
+                QVariantList assignmentList = assignmentConfigVariant.toList();
 
-            if (assignmentType != Assignment::AgentType) {
-                createStaticAssignmentsForType(assignmentType, assignmentList);
+                if (assignmentType != Assignment::AgentType) {
+                    createStaticAssignmentsForType(assignmentType, assignmentList);
+                }
+
+                excludedTypes.insert(assignmentType);
             }
-
-            excludedTypes.insert(assignmentType);
         }
-
-        configIndex = variantMapKeys.indexOf(assignmentConfigRegex, configIndex + 1);
     }
 }
 
@@ -882,10 +914,10 @@ void DomainServer::addStaticAssignmentToAssignmentHash(Assignment* newAssignment
 
 void DomainServer::populateStaticScriptedAssignmentsFromSettings() {
     const QString PERSISTENT_SCRIPTS_KEY_PATH = "scripts.persistent_scripts";
-    const QVariant* persistentScriptsVariant = valueForKeyPath(_settingsManager.getSettingsMap(), PERSISTENT_SCRIPTS_KEY_PATH);
+    QVariant persistentScriptsVariant = _settingsManager.valueOrDefaultValueForKeyPath(PERSISTENT_SCRIPTS_KEY_PATH);
 
-    if (persistentScriptsVariant) {
-        QVariantList persistentScriptsList = persistentScriptsVariant->toList();
+    if (persistentScriptsVariant.isValid()) {
+        QVariantList persistentScriptsList = persistentScriptsVariant.toList();
         foreach(const QVariant& persistentScriptVariant, persistentScriptsList) {
             QVariantMap persistentScript = persistentScriptVariant.toMap();
 
@@ -1695,10 +1727,96 @@ void DomainServer::sendHeartbeatToIceServer() {
     } else {
         qDebug() << "Not sending ice-server heartbeat since there is no selected ice-server.";
         qDebug() << "Waiting for" << _iceServerAddr << "host lookup response";
-
     }
 }
 
+void DomainServer::processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message) {
+    qDebug() << "Received octree data persist message";
+    auto data = message->readAll();
+    auto filePath = getEntitiesFilePath();
+
+    QDir dir(getEntitiesDirPath());
+    if (!dir.exists()) {
+        qCDebug(domain_server) << "Creating entities content directory:" << dir.absolutePath();
+        dir.mkpath(".");
+    }
+
+    QFile f(filePath);
+    if (f.open(QIODevice::WriteOnly)) {
+        f.write(data);
+        OctreeUtils::RawEntityData entityData;
+        if (entityData.readOctreeDataInfoFromData(data)) {
+            qCDebug(domain_server) << "Wrote new entities file" << entityData.id << entityData.version;
+        } else {
+            qCDebug(domain_server) << "Failed to read new octree data info";
+        }
+    } else {
+        qCDebug(domain_server) << "Failed to write new entities file:" << filePath;
+    }
+}
+
+QString DomainServer::getContentBackupDir() {
+    return PathUtils::getAppDataFilePath("backups");
+}
+
+QString DomainServer::getEntitiesDirPath() {
+    return PathUtils::getAppDataFilePath("entities");
+}
+
+QString DomainServer::getEntitiesFilePath() {
+    return PathUtils::getAppDataFilePath("entities/models.json.gz");
+}
+
+QString DomainServer::getEntitiesReplacementFilePath() {
+    return getEntitiesFilePath().append(REPLACEMENT_FILE_EXTENSION);
+}
+
+void DomainServer::processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message) {
+    qDebug() << "Got request for octree data from " << message->getSenderSockAddr();
+
+    maybeHandleReplacementEntityFile();
+
+    bool remoteHasExistingData { false };
+    QUuid id;
+    int version;
+    message->readPrimitive(&remoteHasExistingData);
+    if (remoteHasExistingData) {
+        constexpr size_t UUID_SIZE_BYTES = 16;
+        auto idData = message->read(UUID_SIZE_BYTES);
+        id = QUuid::fromRfc4122(idData);
+        message->readPrimitive(&version);
+        qCDebug(domain_server) << "Entity server does have existing data: ID(" << id << ") DataVersion(" << version << ")";
+    } else {
+        qCDebug(domain_server) << "Entity server does not have existing data";
+    }
+    auto entityFilePath = getEntitiesFilePath();
+
+    auto reply = NLPacketList::create(PacketType::OctreeDataFileReply, QByteArray(), true, true);
+    OctreeUtils::RawEntityData data;
+    if (data.readOctreeDataInfoFromFile(entityFilePath)) {
+        if (data.id == id && data.version <= version) {
+            qCDebug(domain_server) << "ES has sufficient octree data, not sending data";
+            reply->writePrimitive(false);
+        } else {
+            qCDebug(domain_server) << "Sending newer octree data to ES: ID(" << data.id << ") DataVersion(" << data.version << ")";
+            QFile file(entityFilePath);
+            if (file.open(QIODevice::ReadOnly)) {
+                reply->writePrimitive(true);
+                reply->write(file.readAll());
+            } else {
+                qCDebug(domain_server) << "Unable to load entity file";
+                reply->writePrimitive(false);
+            }
+        }
+    } else {
+        qCDebug(domain_server) << "Domain server does not have valid octree data";
+        reply->writePrimitive(false);
+    }
+
+    auto nodeList = DependencyManager::get<LimitedNodeList>();
+    nodeList->sendPacketList(std::move(reply), message->getSenderSockAddr());
+}
+
 void DomainServer::processNodeJSONStatsPacket(QSharedPointer<ReceivedMessage> packetList, SharedNodePointer sendingNode) {
     auto nodeData = static_cast<DomainServerNodeData*>(sendingNode->getLinkedData());
     if (nodeData) {
@@ -1808,23 +1926,25 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
     const QString URI_ASSIGNMENT = "/assignment";
     const QString URI_NODES = "/nodes";
     const QString URI_SETTINGS = "/settings";
-    const QString URI_ENTITY_FILE_UPLOAD = "/content/upload";
+    const QString URI_CONTENT_UPLOAD = "/content/upload";
     const QString URI_RESTART = "/restart";
     const QString URI_API_PLACES = "/api/places";
     const QString URI_API_DOMAINS = "/api/domains";
     const QString URI_API_DOMAINS_ID = "/api/domains/";
+    const QString URI_API_BACKUPS = "/api/backups";
+    const QString URI_API_BACKUPS_ID = "/api/backups/";
+    const QString URI_API_BACKUPS_RECOVER = "/api/backups/recover/";
 
     const QString UUID_REGEX_STRING = "[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}";
 
     auto nodeList = DependencyManager::get<LimitedNodeList>();
 
-    auto getSetting = [this](QString keyPath, QVariant& value) -> bool {
-        QVariantMap& settingsMap = _settingsManager.getSettingsMap();
-        QVariant* var = valueForKeyPath(settingsMap, keyPath);
-        if (var == nullptr) {
+    auto getSetting = [this](QString keyPath, QVariant value) -> bool {
+
+        value = _settingsManager.valueForKeyPath(keyPath);
+        if (!value.isValid()) {
             return false;
         }
-        value = *var;
         return true;
     };
 
@@ -1892,8 +2012,8 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
     if (connection->requestOperation() == QNetworkAccessManager::GetOperation) {
         const QString URI_WIZARD = "/wizard/";
         const QString WIZARD_COMPLETED_ONCE_KEY_PATH = "wizard.completed_once";
-        const QVariant* wizardCompletedOnce = valueForKeyPath(_settingsManager.getSettingsMap(), WIZARD_COMPLETED_ONCE_KEY_PATH);
-        const bool completedOnce = wizardCompletedOnce && wizardCompletedOnce->toBool();
+        QVariant wizardCompletedOnce = _settingsManager.valueForKeyPath(WIZARD_COMPLETED_ONCE_KEY_PATH);
+        const bool completedOnce = wizardCompletedOnce.isValid() && wizardCompletedOnce.toBool();
 
         if (url.path() != URI_WIZARD && url.path().endsWith('/') && !completedOnce) {
             // First visit, redirect to the wizard
@@ -1997,6 +2117,37 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             // send the response
             connection->respond(HTTPConnection::StatusCode200, nodesDocument.toJson(), qPrintable(JSON_MIME_TYPE));
 
+            return true;
+        } else if (url.path() == URI_API_BACKUPS) {
+            auto deferred = makePromise("getAllBackupsAndStatus");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonDocument docJSON(QJsonObject::fromVariantMap(result));
+
+                connection->respond(HTTPConnection::StatusCode200, docJSON.toJson(), JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->getAllBackupsAndStatus(deferred);
+            return true;
+        } else if (url.path().startsWith(URI_API_BACKUPS_ID)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
+            auto deferred = makePromise("consolidateBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                if (success) {
+                    auto path = result["backupFilePath"].toString();
+                    auto file { std::unique_ptr<QFile>(new QFile(path)) };
+                    if (file->open(QIODevice::ReadOnly)) {
+                        connection->respond(HTTPConnection::StatusCode200, std::move(file));
+                    } else {
+                        qCritical(domain_server) << "Unable to load consolidated backup at:" << path << result;
+                        connection->respond(HTTPConnection::StatusCode500, "Error opening backup");
+                    }
+                } else {
+                    connection->respond(HTTPConnection::StatusCode400);
+                }
+            });
+            _contentManager->consolidateBackup(deferred, id);
+
             return true;
         } else if (url.path() == URI_RESTART) {
             connection->respond(HTTPConnection::StatusCode200);
@@ -2084,17 +2235,52 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             connection->respond(HTTPConnection::StatusCode200);
 
             return true;
-        } else if (url.path() == URI_ENTITY_FILE_UPLOAD) {
+        } else if (url.path() == URI_CONTENT_UPLOAD) {
             // this is an entity file upload, ask the HTTPConnection to parse the data
             QList<FormData> formData = connection->parseFormData();
 
             if (formData.size() > 0 && formData[0].second.size() > 0) {
-                // invoke our method to hand the new octree file off to the octree server
-                QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
-                                          Qt::QueuedConnection, Q_ARG(QByteArray, formData[0].second));
+                auto& firstFormData = formData[0];
+
+                // check the file extension to see what kind of file this is
+                // to make sure we handle this filetype for a content restore
+                auto dispositionValue = QString(firstFormData.first.value("Content-Disposition"));
+                auto formDataFilenameRegex = QRegExp("filename=\"(.+)\"");
+                auto matchIndex = formDataFilenameRegex.indexIn(dispositionValue);
+
+                QString uploadedFilename = "";
+                if (matchIndex != -1) {
+                    uploadedFilename = formDataFilenameRegex.cap(1);
+                }
+
+                if (uploadedFilename.endsWith(".json", Qt::CaseInsensitive)
+                    || uploadedFilename.endsWith(".json.gz", Qt::CaseInsensitive)) {
+                    // invoke our method to hand the new octree file off to the octree server
+                    QMetaObject::invokeMethod(this, "handleOctreeFileReplacement",
+                                              Qt::QueuedConnection, Q_ARG(QByteArray, firstFormData.second));
+
+                    // respond with a 200 for success
+                    connection->respond(HTTPConnection::StatusCode200);
+                } else if (uploadedFilename.endsWith(".zip", Qt::CaseInsensitive)) {
+                    auto deferred = makePromise("recoverFromUploadedBackup");
+
+                    deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                        QJsonObject rootJSON;
+                        auto success = result["success"].toBool();
+                        rootJSON["success"] = success;
+                        QJsonDocument docJSON(rootJSON);
+                        connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                            JSON_MIME_TYPE.toUtf8());
+                    });
+
+                    _contentManager->recoverFromUploadedBackup(deferred, firstFormData.second);
+
+                    return true;
+                } else {
+                    // we don't have handling for this filetype, send back a 400 for failure
+                    connection->respond(HTTPConnection::StatusCode400);
+                }
 
-                // respond with a 200 for success
-                connection->respond(HTTPConnection::StatusCode200);
             } else {
                 // respond with a 400 for failure
                 connection->respond(HTTPConnection::StatusCode400);
@@ -2102,16 +2288,50 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
             return true;
 
+        } else if (url.path() == URI_API_BACKUPS) {
+            auto params = connection->parseUrlEncodedForm();
+            auto it = params.find("name");
+            if (it == params.end()) {
+                connection->respond(HTTPConnection::StatusCode400, "Bad request, missing `name`");
+                return true;
+            }
+
+            auto deferred = makePromise("createManualBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->createManualBackup(deferred, it.value());
+
+            return true;
+
         } else if (url.path() == "/domain_settings") {
-            auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-            if (!accessTokenVariant) {
+            auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+            if (!accessTokenVariant.isValid()) {
                 connection->respond(HTTPConnection::StatusCode400);
                 return true;
             }
 
         } else if (url.path() == URI_API_DOMAINS) {
-
             return forwardMetaverseAPIRequest(connection, "/api/v1/domains", "domain", { "label" });
+
+        } else if (url.path().startsWith(URI_API_BACKUPS_RECOVER)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_RECOVER).length());
+            auto deferred = makePromise("recoverFromBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->recoverFromBackup(deferred, id);
+            return true;
         }
     } else if (connection->requestOperation() == QNetworkAccessManager::PutOperation) {
         if (url.path() == URI_API_DOMAINS) {
@@ -2124,8 +2344,8 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
             return forwardMetaverseAPIRequest(connection, "/api/v1/domains/" + domainID, "domain",
                                               { }, { "network_address", "network_port", "label" });
         }  else if (url.path() == URI_API_PLACES) {
-            auto accessTokenVariant = valueForKeyPath(_settingsManager.getSettingsMap(), ACCESS_TOKEN_KEY_PATH);
-            if (!accessTokenVariant->isValid()) {
+            auto accessTokenVariant = _settingsManager.valueForKeyPath(ACCESS_TOKEN_KEY_PATH);
+            if (!accessTokenVariant.isValid()) {
                 connection->respond(HTTPConnection::StatusCode400, "User access token has not been set");
                 return true;
             }
@@ -2173,7 +2393,7 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
 
             QUrl url { NetworkingConstants::METAVERSE_SERVER_URL().toString() + "/api/v1/places/" + place_id };
 
-            url.setQuery("access_token=" + accessTokenVariant->toString());
+            url.setQuery("access_token=" + accessTokenVariant.toString());
 
             QNetworkRequest req(url);
             req.setHeader(QNetworkRequest::UserAgentHeader, HIGH_FIDELITY_USER_AGENT);
@@ -2200,7 +2420,22 @@ bool DomainServer::handleHTTPRequest(HTTPConnection* connection, const QUrl& url
         QRegExp allNodesDeleteRegex(ALL_NODE_DELETE_REGEX_STRING);
         QRegExp nodeDeleteRegex(NODE_DELETE_REGEX_STRING);
 
-        if (nodeDeleteRegex.indexIn(url.path()) != -1) {
+        if (url.path().startsWith(URI_API_BACKUPS_ID)) {
+            auto id = url.path().mid(QString(URI_API_BACKUPS_ID).length());
+            auto deferred = makePromise("deleteBackup");
+            deferred->then([connection, JSON_MIME_TYPE](QString error, QVariantMap result) {
+                QJsonObject rootJSON;
+                auto success = result["success"].toBool();
+                rootJSON["success"] = success;
+                QJsonDocument docJSON(rootJSON);
+                connection->respond(success ? HTTPConnection::StatusCode200 : HTTPConnection::StatusCode400, docJSON.toJson(),
+                                    JSON_MIME_TYPE.toUtf8());
+            });
+            _contentManager->deleteBackup(deferred, id);
+
+            return true;
+
+        } else if (nodeDeleteRegex.indexIn(url.path()) != -1) {
             // this is a request to DELETE one node by UUID
 
             // pull the captured string, if it exists
@@ -2353,10 +2588,11 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
 
     const QByteArray UNAUTHENTICATED_BODY = "You do not have permission to access this domain-server.";
 
-    QVariantMap& settingsMap = _settingsManager.getSettingsMap();
+    QVariant adminUsersVariant = _settingsManager.valueForKeyPath(ADMIN_USERS_CONFIG_KEY);
+    QVariant adminRolesVariant = _settingsManager.valueForKeyPath(ADMIN_ROLES_CONFIG_KEY);
 
     if (!_oauthProviderURL.isEmpty()
-        && (settingsMap.contains(ADMIN_USERS_CONFIG_KEY) || settingsMap.contains(ADMIN_ROLES_CONFIG_KEY))) {
+        && (adminUsersVariant.isValid() || adminRolesVariant.isValid())) {
         QString cookieString = connection->requestHeaders().value(HTTP_COOKIE_HEADER_KEY);
 
         const QString COOKIE_UUID_REGEX_STRING = HIFI_SESSION_COOKIE_KEY + "=([\\d\\w-]+)($|;)";
@@ -2367,7 +2603,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             cookieUUID = cookieUUIDRegex.cap(1);
         }
 
-        if (valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)) {
+        if (_settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).isValid()) {
             qDebug() << "Config file contains web admin settings for OAuth and basic HTTP authentication."
                 << "These cannot be combined - using OAuth for authentication.";
         }
@@ -2377,13 +2613,13 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             DomainServerWebSessionData sessionData = _cookieSessionHash.value(cookieUUID);
             QString profileUsername = sessionData.getUsername();
 
-            if (settingsMap.value(ADMIN_USERS_CONFIG_KEY).toStringList().contains(profileUsername)) {
+            if (_settingsManager.valueForKeyPath(ADMIN_USERS_CONFIG_KEY).toStringList().contains(profileUsername)) {
                 // this is an authenticated user
                 return true;
             }
 
             // loop the roles of this user and see if they are in the admin-roles array
-            QStringList adminRolesArray = settingsMap.value(ADMIN_ROLES_CONFIG_KEY).toStringList();
+            QStringList adminRolesArray = _settingsManager.valueForKeyPath(ADMIN_ROLES_CONFIG_KEY).toStringList();
 
             if (!adminRolesArray.isEmpty()) {
                 foreach(const QString& userRole, sessionData.getRoles()) {
@@ -2428,7 +2664,7 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
             // we don't know about this user yet, so they are not yet authenticated
             return false;
         }
-    } else if (valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)) {
+    } else if (_settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).isValid()) {
         // config file contains username and password combinations for basic auth
         const QByteArray BASIC_AUTH_HEADER_KEY = "Authorization";
 
@@ -2447,10 +2683,10 @@ bool DomainServer::isAuthenticatedRequest(HTTPConnection* connection, const QUrl
                     QString headerPassword = credentialList[1];
 
                     // we've pulled a username and password - now check if there is a match in our basic auth hash
-                    QString settingsUsername = valueForKeyPath(settingsMap, BASIC_AUTH_USERNAME_KEY_PATH)->toString();
-                    const QVariant* settingsPasswordVariant = valueForKeyPath(settingsMap, BASIC_AUTH_PASSWORD_KEY_PATH);
+                    QString settingsUsername = _settingsManager.valueForKeyPath(BASIC_AUTH_USERNAME_KEY_PATH).toString();
+                    QVariant settingsPasswordVariant = _settingsManager.valueForKeyPath(BASIC_AUTH_PASSWORD_KEY_PATH);
 
-                    QString settingsPassword = settingsPasswordVariant ? settingsPasswordVariant->toString() : "";
+                    QString settingsPassword = settingsPasswordVariant.isValid() ? settingsPasswordVariant.toString() : "";
                     QString hexHeaderPassword = headerPassword.isEmpty() ?
                         "" : QCryptographicHash::hash(headerPassword.toUtf8(), QCryptographicHash::Sha256).toHex();
                         
@@ -2587,13 +2823,14 @@ ReplicationServerInfo serverInformationFromSettings(QVariantMap serverMap, Repli
 }
 
 void DomainServer::updateReplicationNodes(ReplicationServerDirection direction) {
-    auto settings = _settingsManager.getSettingsMap();
 
-    if (settings.contains(BROADCASTING_SETTINGS_KEY)) {
+    auto broadcastSettingsVariant = _settingsManager.valueForKeyPath(BROADCASTING_SETTINGS_KEY);
+
+    if (broadcastSettingsVariant.isValid()) {
         auto nodeList = DependencyManager::get<LimitedNodeList>();
         std::vector<HifiSockAddr> replicationNodesInSettings;
 
-        auto replicationSettings = settings.value(BROADCASTING_SETTINGS_KEY).toMap();
+        auto replicationSettings = broadcastSettingsVariant.toMap();
 
         QString serversKey = direction == Upstream ? "upstream_servers" : "downstream_servers";
         QString replicationDirection = direction == Upstream ? "upstream" : "downstream";
@@ -2669,13 +2906,12 @@ void DomainServer::updateUpstreamNodes() {
 
 void DomainServer::updateReplicatedNodes() {
     // Make sure we have downstream nodes in our list
-    auto settings = _settingsManager.getSettingsMap();
-
     static const QString REPLICATED_USERS_KEY = "users";
     _replicatedUsernames.clear();
-    
-    if (settings.contains(BROADCASTING_SETTINGS_KEY)) {
-        auto replicationSettings = settings.value(BROADCASTING_SETTINGS_KEY).toMap();
+
+    auto replicationVariant = _settingsManager.valueForKeyPath(BROADCASTING_SETTINGS_KEY);
+    if (replicationVariant.isValid()) {
+        auto replicationSettings = replicationVariant.toMap();
         if (replicationSettings.contains(REPLICATED_USERS_KEY)) {
             auto usersSettings = replicationSettings.value(REPLICATED_USERS_KEY).toList();
             for (auto& username : usersSettings) {
@@ -2863,17 +3099,17 @@ void DomainServer::processPathQueryPacket(QSharedPointer<ReceivedMessage> messag
 
         // check out paths in the _configMap to see if we have a match
         auto keypath = QString(PATHS_SETTINGS_KEYPATH_FORMAT).arg(SETTINGS_PATHS_KEY).arg(pathQuery);
-        const QVariant* pathMatch = valueForKeyPath(_settingsManager.getSettingsMap(), keypath);
+        QVariant pathMatch = _settingsManager.valueForKeyPath(keypath);
 
-        if (pathMatch || pathQuery == INDEX_PATH) {
+        if (pathMatch.isValid() || pathQuery == INDEX_PATH) {
             // we got a match, respond with the resulting viewpoint
             auto nodeList = DependencyManager::get<LimitedNodeList>();
 
             QString responseViewpoint;
 
             // if we didn't match the path BUT this is for the index path then send back our default
-            if (pathMatch) {
-                responseViewpoint = pathMatch->toMap()[PATH_VIEWPOINT_KEY].toString();
+            if (pathMatch.isValid()) {
+                responseViewpoint = pathMatch.toMap()[PATH_VIEWPOINT_KEY].toString();
             } else {
                 const QString DEFAULT_INDEX_PATH = "/0,0,0/0,0,0,1";
                 responseViewpoint = DEFAULT_INDEX_PATH;
@@ -3105,19 +3341,101 @@ void DomainServer::setupGroupCacheRefresh() {
     }
 }
 
-void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
-    // enumerate the nodes and find any octree type servers with active sockets
+void DomainServer::maybeHandleReplacementEntityFile() {
+    const auto replacementFilePath = getEntitiesReplacementFilePath();
+    OctreeUtils::RawEntityData data;
+    if (!data.readOctreeDataInfoFromFile(replacementFilePath)) {
+        qCWarning(domain_server) << "Replacement file could not be read, it either doesn't exist or is invalid.";
+    } else {
+        qCDebug(domain_server) << "Replacing existing entity date with replacement file";
 
-    auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
-    limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
-        return node->getType() == NodeType::EntityServer && node->getActiveSocket();
-    }, [&octreeFile, limitedNodeList](const SharedNodePointer& octreeNode) {
-        // setup a packet to send to this octree server with the new octree file data
-        auto octreeFilePacketList = NLPacketList::create(PacketType::OctreeFileReplacement, QByteArray(), true, true);
-        octreeFilePacketList->write(octreeFile);
+        QFile replacementFile(replacementFilePath);
+        if (!replacementFile.remove()) {
+            // If we can't remove the replacement file, we are at risk of getting into a state where
+            // we continually replace the primary entity file with the replacement entity file.
+            qCWarning(domain_server) << "Unable to remove replacement file, bailing";
+        } else {
+            data.resetIdAndVersion();
+            auto gzippedData = data.toGzippedByteArray();
 
-        qDebug() << "Sending an octree file replacement of" << octreeFile.size() << "bytes to" << octreeNode;
-
-        limitedNodeList->sendPacketList(std::move(octreeFilePacketList), *octreeNode);
-    });
+            QFile currentFile(getEntitiesFilePath());
+            if (!currentFile.open(QIODevice::WriteOnly)) {
+                qCWarning(domain_server)
+                    << "Failed to update entities data file with replacement file, unable to open entities file for writing";
+            } else {
+                currentFile.write(gzippedData);
+            }
+        }
+    }
+}
+
+void DomainServer::handleOctreeFileReplacement(QByteArray octreeFile) {
+    //Assume we have compressed data
+    auto compressedOctree = octreeFile;
+    QByteArray jsonOctree;
+
+    bool wasCompressed = gunzip(compressedOctree, jsonOctree);
+    if (!wasCompressed) {
+        // the source was not compressed, assume we were sent regular JSON data
+        jsonOctree = compressedOctree;
+    }
+
+    OctreeUtils::RawEntityData data;
+    if (data.readOctreeDataInfoFromData(jsonOctree)) {
+        data.resetIdAndVersion();
+
+        gzip(data.toByteArray(), compressedOctree);
+
+        // write the compressed octree data to a special file
+        auto replacementFilePath = getEntitiesReplacementFilePath();
+        QFile replacementFile(replacementFilePath);
+        if (replacementFile.open(QIODevice::WriteOnly) && replacementFile.write(compressedOctree) != -1) {
+            // we've now written our replacement file, time to take the server down so it can
+            // process it when it comes back up
+            qInfo() << "Wrote octree replacement file to" << replacementFilePath << "- stopping server";
+
+            QMetaObject::invokeMethod(this, "restart", Qt::QueuedConnection);
+        } else {
+            qWarning() << "Could not write replacement octree data to file - refusing to process";
+        }
+    } else {
+        qDebug() << "Received replacement octree file that is invalid - refusing to process";
+    }
+}
+
+void DomainServer::handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message) {
+    qInfo() << "Received request to replace content from a url";
+    auto node = DependencyManager::get<LimitedNodeList>()->findNodeWithAddr(message->getSenderSockAddr());
+    if (node) {
+        qDebug() << "Found node: " << node->getCanReplaceContent();
+    }
+    if (node->getCanReplaceContent()) {
+        // Convert message data into our URL
+        QString url(message->getMessage());
+        QUrl modelsURL = QUrl(url, QUrl::StrictMode);
+        QNetworkAccessManager& networkAccessManager = NetworkAccessManager::getInstance();
+        QNetworkRequest request(modelsURL);
+        QNetworkReply* reply = networkAccessManager.get(request);
+
+        qDebug() << "Downloading JSON from: " << modelsURL;
+
+        connect(reply, &QNetworkReply::finished, [this, reply, modelsURL]() {
+            QNetworkReply::NetworkError networkError = reply->error();
+            if (networkError == QNetworkReply::NoError) {
+                handleOctreeFileReplacement(reply->readAll());
+            } else {
+                qDebug() << "Error downloading JSON from specified file: " << modelsURL;
+            }
+        });
+    }
+}
+
+
+
+
+void DomainServer::handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message) {
+    auto node = DependencyManager::get<NodeList>()->nodeWithUUID(message->getSourceID());
+    if (node->getCanReplaceContent()) {
+        handleOctreeFileReplacement(message->readAll());
+    }
 }

domain-server/src/DomainServer.h

@@ -26,15 +26,20 @@
 #include <HTTPSConnection.h>
 #include <LimitedNodeList.h>
 
-#include "BackupSupervisor.h"
+#include "AssetsBackupHandler.h"
 #include "DomainGatekeeper.h"
 #include "DomainMetadata.h"
 #include "DomainServerSettingsManager.h"
 #include "DomainServerWebSessionData.h"
 #include "WalletTransaction.h"
+#include "DomainContentBackupManager.h"
 
 #include "PendingAssignedNodeData.h"
 
+#include <QLoggingCategory>
+
+Q_DECLARE_LOGGING_CATEGORY(domain_server)
+
 typedef QSharedPointer<Assignment> SharedAssignmentPointer;
 typedef QMultiHash<QUuid, WalletTransaction*> TransactionHash;
 
@@ -65,6 +70,8 @@ public:
     bool handleHTTPRequest(HTTPConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
     bool handleHTTPSRequest(HTTPSConnection* connection, const QUrl& url, bool skipSubHandler = false) override;
 
+    static const QString REPLACEMENT_FILE_EXTENSION;
+
 public slots:
     /// Called by NodeList to inform us a node has been added
     void nodeAdded(SharedNodePointer node);
@@ -84,6 +91,13 @@ private slots:
     void processICEServerHeartbeatDenialPacket(QSharedPointer<ReceivedMessage> message);
     void processICEServerHeartbeatACK(QSharedPointer<ReceivedMessage> message);
 
+    void handleOctreeFileReplacementFromURLRequest(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeFileReplacementRequest(QSharedPointer<ReceivedMessage> message);
+    void handleOctreeFileReplacement(QByteArray octreeFile);
+
+    void processOctreeDataRequestMessage(QSharedPointer<ReceivedMessage> message);
+    void processOctreeDataPersistMessage(QSharedPointer<ReceivedMessage> message);
+
     void setupPendingAssignmentCredits();
     void sendPendingTransactionsToServer();
 
@@ -91,8 +105,7 @@ private slots:
     void sendHeartbeatToMetaverse() { sendHeartbeatToMetaverse(QString()); }
     void sendHeartbeatToIceServer();
 
-    void handleConnectedNode(SharedNodePointer newNode);
-
+    void handleConnectedNode(SharedNodePointer newNode); 
     void handleTempDomainSuccess(QNetworkReply& requestReply);
     void handleTempDomainError(QNetworkReply& requestReply);
 
@@ -109,8 +122,6 @@ private slots:
     void handleSuccessfulICEServerAddressUpdate(QNetworkReply& requestReply);
     void handleFailedICEServerAddressUpdate(QNetworkReply& requestReply);
 
-    void handleOctreeFileReplacement(QByteArray octreeFile);
-
     void updateReplicatedNodes();
     void updateDownstreamNodes();
     void updateUpstreamNodes();
@@ -127,6 +138,13 @@ private:
     const QUuid& getID();
     void parseCommandLine();
 
+    QString getContentBackupDir();
+    QString getEntitiesDirPath();
+    QString getEntitiesFilePath();
+    QString getEntitiesReplacementFilePath();
+
+    void maybeHandleReplacementEntityFile();
+
     void setupNodeListAndAssignments();
     bool optionallySetupOAuth();
     bool optionallyReadX509KeyAndCertificate();
@@ -252,6 +270,8 @@ private:
     bool _sendICEServerAddressToMetaverseAPIInProgress { false };
     bool _sendICEServerAddressToMetaverseAPIRedo { false };
 
+    std::unique_ptr<DomainContentBackupManager> _contentManager { nullptr };
+
     QHash<QUuid, QPointer<HTTPSConnection>> _pendingOAuthConnections;
 
     QThread _assetClientThread;

domain-server/src/DomainServerSettingsManager.cpp

@@ -19,6 +19,7 @@
 #include <QtCore/QJsonArray>
 #include <QtCore/QJsonObject>
 #include <QtCore/QStandardPaths>
+#include <QtCore/QThread>
 #include <QtCore/QUrl>
 #include <QtCore/QUrlQuery>
 
@@ -32,9 +33,13 @@
 #include <SettingHelpers.h>
 #include <AvatarData.h> //for KillAvatarReason
 #include <FingerprintUtils.h>
+
 #include "DomainServerNodeData.h"
 
 const QString SETTINGS_DESCRIPTION_RELATIVE_PATH = "/resources/describe-settings.json";
+const QString SETTINGS_PATH = "/settings";
+const QString SETTINGS_PATH_JSON = SETTINGS_PATH + ".json";
+const QString CONTENT_SETTINGS_PATH_JSON = "/content-settings.json";
 
 const QString DESCRIPTION_SETTINGS_KEY = "settings";
 const QString SETTING_DEFAULT_KEY = "default";
@@ -187,6 +192,9 @@ void DomainServerSettingsManager::processSettingsRequestPacket(QSharedPointer<Re
 }
 
 void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList) {
+    // since we're called from the DomainServerSettingsManager constructor, we don't take a write lock here
+    // even though we change the underlying config map
+
     _argumentList = argumentList;
 
     _configMap.loadConfig(_argumentList);
@@ -390,6 +398,7 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
             _standardAgentPermissions[NodePermissions::standardNameLocalhost]->set(NodePermissions::Permission::canRezTemporaryCertifiedEntities);
             packPermissions();
         }
+
         if (oldVersion < 2.0) {
             const QString WIZARD_COMPLETED_ONCE = "wizard.completed_once";
 
@@ -397,6 +406,7 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
 
             *wizardCompletedOnce = QVariant(true);
         }
+
         if (oldVersion < 2.1) {
             // convert old avatar scale settings into avatar height.
 
@@ -418,6 +428,21 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
             }
         }
 
+        if (oldVersion < 2.2) {
+            // migrate entity server rolling backup intervals to new location for automatic content archive intervals
+
+            const QString ENTITY_SERVER_BACKUPS_KEYPATH = "entity_server_settings.backups";
+            const QString AUTO_CONTENT_ARCHIVES_RULES_KEYPATH = AUTOMATIC_CONTENT_ARCHIVES_GROUP + ".backup_rules";
+
+            QVariant* previousBackupsVariant = _configMap.valueForKeyPath(ENTITY_SERVER_BACKUPS_KEYPATH);
+
+            if (previousBackupsVariant) {
+                auto migratedBackupsVariant = _configMap.valueForKeyPath(AUTO_CONTENT_ARCHIVES_RULES_KEYPATH, true);
+                *migratedBackupsVariant = *previousBackupsVariant;
+            }
+        }
+
+
         // write the current description version to our settings
         *versionVariant = _descriptionVersion;
 
@@ -428,17 +453,6 @@ void DomainServerSettingsManager::setupConfigMap(const QStringList& argumentList
     unpackPermissions();
 }
 
-QVariantMap& DomainServerSettingsManager::getDescriptorsMap() {
-    static const QString DESCRIPTORS{ "descriptors" };
-
-    auto& settingsMap = getSettingsMap();
-    if (!getSettingsMap().contains(DESCRIPTORS)) {
-        settingsMap.insert(DESCRIPTORS, QVariantMap());
-    }
-
-    return *static_cast<QVariantMap*>(getSettingsMap()[DESCRIPTORS].data());
-}
-
 void DomainServerSettingsManager::initializeGroupPermissions(NodePermissionsMap& permissionsRows,
                                                              QString groupName, NodePermissionsPointer perms) {
     // this is called when someone has used the domain-settings webpage to add a group.  They type the group's name
@@ -467,6 +481,9 @@ void DomainServerSettingsManager::initializeGroupPermissions(NodePermissionsMap&
 void DomainServerSettingsManager::packPermissionsForMap(QString mapName,
                                                         NodePermissionsMap& permissionsRows,
                                                         QString keyPath) {
+    // grab a write lock on the settings mutex since we're about to change the config map
+    QWriteLocker locker(&_settingsLock);
+
     // find (or create) the "security" section of the settings map
     QVariant* security = _configMap.valueForKeyPath("security", true);
     if (!security->canConvert(QMetaType::QVariantMap)) {
@@ -556,15 +573,20 @@ bool DomainServerSettingsManager::unpackPermissionsForKeypath(const QString& key
 
     mapPointer->clear();
 
-    QVariant* permissions = _configMap.valueForKeyPath(keyPath, true);
-    if (!permissions->canConvert(QMetaType::QVariantList)) {
+    QVariant permissions = valueOrDefaultValueForKeyPath(keyPath);
+
+    if (!permissions.isValid()) {
+        // we don't have a permissions object to unpack for this keypath, bail
+        return false;
+    }
+
+    if (!permissions.canConvert(QMetaType::QVariantList)) {
         qDebug() << "Failed to extract permissions for key path" << keyPath << "from settings.";
-        (*permissions) = QVariantList();
     }
 
     bool needPack = false;
 
-    QList<QVariant> permissionsList = permissions->toList();
+    QList<QVariant> permissionsList = permissions.toList();
     foreach (QVariant permsHash, permissionsList) {
         NodePermissionsPointer perms { new NodePermissions(permsHash.toMap()) };
         QString id = perms->getID();
@@ -591,6 +613,11 @@ bool DomainServerSettingsManager::unpackPermissionsForKeypath(const QString& key
 void DomainServerSettingsManager::unpackPermissions() {
     // transfer details from _configMap to _agentPermissions
 
+    // NOTE: Defaults for standard permissions (anonymous, friends, localhost, logged-in) used
+    // to be set here and then immediately persisted to the config JSON file.
+    // They have since been moved to describe-settings.json as the default value for AGENT_STANDARD_PERMISSIONS_KEYPATH.
+    // In order to change the default standard permissions you must change the default value in describe-settings.json.
+
     bool needPack = false;
 
     needPack |= unpackPermissionsForKeypath(AGENT_STANDARD_PERMISSIONS_KEYPATH, &_standardAgentPermissions);
@@ -650,57 +677,39 @@ void DomainServerSettingsManager::unpackPermissions() {
             }
     });
 
-    // if any of the standard names are missing, add them
-    foreach(const QString& standardName, NodePermissions::standardNames) {
-        NodePermissionsKey standardKey { standardName, 0 };
-        if (!_standardAgentPermissions.contains(standardKey)) {
-            // we don't have permissions for one of the standard groups, so we'll add them now
-            NodePermissionsPointer perms { new NodePermissions(standardKey) };
-
-            if (standardKey == NodePermissions::standardNameLocalhost) {
-                // the localhost user is granted all permissions by default
-                perms->setAll(true);
-            } else {
-                // anonymous, logged in, and friend users get connect permissions by default
-                perms->set(NodePermissions::Permission::canConnectToDomain);
-                perms->set(NodePermissions::Permission::canRezTemporaryCertifiedEntities);
-            }
-
-            // add the permissions to the standard map
-            _standardAgentPermissions[standardKey] = perms;
-
-            // this will require a packing of permissions
-            needPack = true;
-        }
-    }
-
     needPack |= ensurePermissionsForGroupRanks();
 
     if (needPack) {
         packPermissions();
     }
 
-    #ifdef WANT_DEBUG
+#ifdef WANT_DEBUG
     qDebug() << "--------------- permissions ---------------------";
-    QList<QHash<NodePermissionsKey, NodePermissionsPointer>> permissionsSets;
-    permissionsSets << _standardAgentPermissions.get() << _agentPermissions.get()
-                    << _groupPermissions.get() << _groupForbiddens.get()
-                    << _ipPermissions.get() << _macPermissions.get()
-                    << _machineFingerprintPermissions.get();
+    std::array<NodePermissionsMap*, 7> permissionsSets {{
+        &_standardAgentPermissions, &_agentPermissions,
+        &_groupPermissions, &_groupForbiddens,
+        &_ipPermissions, &_macPermissions,
+        &_machineFingerprintPermissions
+    }};
 
     foreach (auto permissionSet, permissionsSets) {
-        QHashIterator<NodePermissionsKey, NodePermissionsPointer> i(permissionSet);
-        while (i.hasNext()) {
-            i.next();
-            NodePermissionsPointer perms = i.value();
+        auto& permissionKeyMap = permissionSet->get();
+        auto it = permissionKeyMap.begin();
+
+        while (it != permissionKeyMap.end()) {
+
+            NodePermissionsPointer perms = it->second;
             if (perms->isGroup()) {
-                qDebug() << i.key() << perms->getGroupID() << perms;
+                qDebug() << it->first << perms->getGroupID() << perms;
             } else {
-                qDebug() << i.key() << perms;
+                qDebug() << it->first << perms;
             }
+
+            ++it;
         }
     }
-    #endif
+#endif
+
 }
 
 bool DomainServerSettingsManager::ensurePermissionsForGroupRanks() {
@@ -1048,12 +1057,22 @@ NodePermissions DomainServerSettingsManager::getForbiddensForGroup(const QUuid&
     return getForbiddensForGroup(groupKey.first, groupKey.second);
 }
 
+QVariant DomainServerSettingsManager::valueForKeyPath(const QString& keyPath) {
+    QReadLocker locker(&_settingsLock);
+    auto foundValue = _configMap.valueForKeyPath(keyPath);
+    return foundValue ? *foundValue : QVariant();
+}
+
 QVariant DomainServerSettingsManager::valueOrDefaultValueForKeyPath(const QString& keyPath) {
+    QReadLocker locker(&_settingsLock);
     const QVariant* foundValue = _configMap.valueForKeyPath(keyPath);
 
     if (foundValue) {
         return *foundValue;
     } else {
+        // we don't need the settings lock anymore since we're done reading from the config map
+        locker.unlock();
+
         int dotIndex = keyPath.indexOf('.');
 
         QString groupKey = keyPath.mid(0, dotIndex);
@@ -1092,9 +1111,6 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
             // we recurse one level deep below each group for the appropriate setting
             bool restartRequired = recurseJSONObjectAndOverwriteSettings(postedObject, endpointType);
 
-            // store whatever the current _settingsMap is to file
-            persistToFile();
-
             // return success to the caller
             QString jsonSuccess = "{\"status\": \"success\"}";
             connection->respond(HTTPConnection::StatusCode200, jsonSuccess.toUtf8(), "application/json");
@@ -1152,17 +1168,20 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
 
             QJsonObject rootObject;
 
-            bool forDomainSettings = (url.path() == SETTINGS_PATH_JSON);
-            bool forContentSettings = (url.path() == CONTENT_SETTINGS_PATH_JSON);;
+            DomainSettingsInclusion domainSettingsInclusion = (url.path() == SETTINGS_PATH_JSON)
+                ? IncludeDomainSettings : NoDomainSettings;
+            ContentSettingsInclusion contentSettingsInclusion = (url.path() == CONTENT_SETTINGS_PATH_JSON)
+                ? IncludeContentSettings : NoContentSettings;
 
-            rootObject[SETTINGS_RESPONSE_DESCRIPTION_KEY] = forDomainSettings
+            rootObject[SETTINGS_RESPONSE_DESCRIPTION_KEY] = (url.path() == SETTINGS_PATH_JSON)
                 ? _domainSettingsDescription : _contentSettingsDescription;
 
             // grab a domain settings object for all types, filtered for the right class of settings
             // and exclude default values
-            rootObject[SETTINGS_RESPONSE_VALUE_KEY] = settingsResponseObjectForType("", true,
-                                                                                    forDomainSettings, forContentSettings,
-                                                                                    true);
+            rootObject[SETTINGS_RESPONSE_VALUE_KEY] = settingsResponseObjectForType("", Authenticated,
+                                                                                    domainSettingsInclusion,
+                                                                                    contentSettingsInclusion,
+                                                                                    IncludeDefaultSettings);
 
             connection->respond(HTTPConnection::StatusCode200, QJsonDocument(rootObject).toJson(), "application/json");
 
@@ -1174,7 +1193,8 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
         } else if (url.path() == SETTINGS_BACKUP_PATH) {
             // grab the settings backup as an authenticated user
             // for the domain settings type only, excluding hidden and default values
-            auto currentDomainSettingsJSON = settingsResponseObjectForType("", true, true, false, false, true);
+            auto currentDomainSettingsJSON = settingsResponseObjectForType("", Authenticated, IncludeDomainSettings,
+                                                                           NoContentSettings, NoDefaultSettings, ForBackup);
 
             // setup headers that tell the client to download the file wth a special name
             Headers downloadHeaders;
@@ -1196,6 +1216,10 @@ bool DomainServerSettingsManager::handleAuthenticatedHTTPRequest(HTTPConnection
 }
 
 bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType) {
+
+    // grab a write lock since we're about to change the settings map
+    QWriteLocker locker(&_settingsLock);
+
     QJsonArray* filteredDescriptionArray = settingsType == DomainSettings
         ? &_domainSettingsDescription : &_contentSettingsDescription;
 
@@ -1277,6 +1301,9 @@ bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settings
                     }
                 } else {
                     // we have a value to restore, use update setting to set it
+                    // but clear the existing value first so that no merging between the restored settings
+                    // and existing settings occurs
+                    variantValue->clear();
 
                     // we might need to re-grab config group map in case it didn't exist when we looked for it before
                     // but was created by the call to valueForKeyPath before
@@ -1310,18 +1337,24 @@ bool DomainServerSettingsManager::restoreSettingsFromObject(QJsonObject settings
     } else {
         // restore completed, persist the new settings
         qDebug() << "Restore completed, persisting restored settings to file";
+
+        // let go of the write lock since we're done making changes to the config map
+        locker.unlock();
+
         persistToFile();
         return true;
     }
 }
 
-QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QString& typeValue, bool isAuthenticated,
-                                                                       bool includeDomainSettings,
-                                                                       bool includeContentSettings,
-                                                                       bool includeDefaults, bool isForBackup) {
+QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QString& typeValue,
+                                                                       SettingsRequestAuthentication authentication,
+                                                                       DomainSettingsInclusion domainSettingsInclusion,
+                                                                       ContentSettingsInclusion contentSettingsInclusion,
+                                                                       DefaultSettingsInclusion defaultSettingsInclusion,
+                                                                       SettingsBackupFlag settingsBackupFlag) {
     QJsonObject responseObject;
 
-    if (!typeValue.isEmpty() || isAuthenticated) {
+    if (!typeValue.isEmpty() || authentication == Authenticated) {
         // convert the string type value to a QJsonValue
         QJsonValue queryType = typeValue.isEmpty() ? QJsonValue() : QJsonValue(typeValue.toInt());
 
@@ -1329,9 +1362,10 @@ QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QSt
 
         // only enumerate the requested settings type (domain setting or content setting)
         QJsonArray* filteredDescriptionArray = &_descriptionArray;
-        if (includeDomainSettings && !includeContentSettings) {
+
+        if (domainSettingsInclusion == IncludeDomainSettings && contentSettingsInclusion != IncludeContentSettings) {
             filteredDescriptionArray = &_domainSettingsDescription;
-        } else if (includeContentSettings && !includeDomainSettings) {
+        } else if (contentSettingsInclusion == IncludeContentSettings && domainSettingsInclusion != IncludeDomainSettings) {
             filteredDescriptionArray = &_contentSettingsDescription;
         }
 
@@ -1354,35 +1388,35 @@ QJsonObject DomainServerSettingsManager::settingsResponseObjectForType(const QSt
                 bool includedInBackups = !settingObject.contains(DESCRIPTION_BACKUP_FLAG_KEY)
                     || settingObject[DESCRIPTION_BACKUP_FLAG_KEY].toBool();
 
-                if (!settingObject[VALUE_HIDDEN_FLAG_KEY].toBool() && (!isForBackup || includedInBackups)) {
+                if (!settingObject[VALUE_HIDDEN_FLAG_KEY].toBool() && (settingsBackupFlag != ForBackup || includedInBackups)) {
                     QJsonArray affectedTypesArray = settingObject[AFFECTED_TYPES_JSON_KEY].toArray();
                     if (affectedTypesArray.isEmpty()) {
                         affectedTypesArray = groupObject[AFFECTED_TYPES_JSON_KEY].toArray();
                     }
 
                     if (affectedTypesArray.contains(queryType) ||
-                        (queryType.isNull() && isAuthenticated)) {
+                        (queryType.isNull() && authentication == Authenticated)) {
                         QString settingName = settingObject[DESCRIPTION_NAME_KEY].toString();
 
                         // we need to check if the settings map has a value for this setting
                         QVariant variantValue;
 
                         if (!groupKey.isEmpty()) {
-                             QVariant settingsMapGroupValue = _configMap.value(groupKey);
+                             QVariant settingsMapGroupValue = valueForKeyPath(groupKey);
 
                             if (!settingsMapGroupValue.isNull()) {
                                 variantValue = settingsMapGroupValue.toMap().value(settingName);
                             }
                         } else {
-                            variantValue = _configMap.value(settingName);
+                            variantValue = valueForKeyPath(settingName);
                         }
 
                         // final check for inclusion
                         // either we include default values or we don't but this isn't a default value
-                        if (includeDefaults || !variantValue.isNull()) {
+                        if ((defaultSettingsInclusion == IncludeDefaultSettings) || variantValue.isValid()) {
                             QJsonValue result;
 
-                            if (variantValue.isNull()) {
+                            if (!variantValue.isValid()) {
                                 // no value for this setting, pass the default
                                 if (settingObject.contains(SETTING_DEFAULT_KEY)) {
                                     result = settingObject[SETTING_DEFAULT_KEY];
@@ -1521,6 +1555,10 @@ QJsonObject DomainServerSettingsManager::settingDescriptionFromGroup(const QJson
 
 bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject,
                                                                         SettingsType settingsType) {
+
+    // take a write lock since we're about to overwrite settings in the config map
+    QWriteLocker locker(&_settingsLock);
+
     static const QString SECURITY_ROOT_KEY = "security";
     static const QString AC_SUBNET_WHITELIST_KEY = "ac_subnet_whitelist";
     static const QString BROADCASTING_KEY = "broadcasting";
@@ -1618,6 +1656,12 @@ bool DomainServerSettingsManager::recurseJSONObjectAndOverwriteSettings(const QJ
         }
     }
 
+    // we're done making changes to the config map, let go of our read lock
+    locker.unlock();
+
+    // store whatever the current config map is to file
+    persistToFile();
+
     return needRestart;
 }
 
@@ -1644,6 +1688,9 @@ bool permissionVariantLessThan(const QVariant &v1, const QVariant &v2) {
 }
 
 void DomainServerSettingsManager::sortPermissions() {
+    // take a write lock since we're about to change the config map data
+    QWriteLocker locker(&_settingsLock);
+
     // sort the permission-names
     QVariant* standardPermissions = _configMap.valueForKeyPath(AGENT_STANDARD_PERMISSIONS_KEYPATH);
     if (standardPermissions && standardPermissions->canConvert(QMetaType::QVariantList)) {
@@ -1680,11 +1727,15 @@ void DomainServerSettingsManager::persistToFile() {
     QFile settingsFile(_configMap.getUserConfigFilename());
 
     if (settingsFile.open(QIODevice::WriteOnly)) {
+        // take a read lock so we can grab the config and write it to file
+        QReadLocker locker(&_settingsLock);
         settingsFile.write(QJsonDocument::fromVariant(_configMap.getConfig()).toJson());
     } else {
         qCritical("Could not write to JSON settings file. Unable to persist settings.");
 
         // failed to write, reload whatever the current config state is
+        // with a write lock since we're about to overwrite the config map
+        QWriteLocker locker(&_settingsLock);
         _configMap.loadConfig(_argumentList);
     }
 }

domain-server/src/DomainServerSettingsManager.h

@@ -27,9 +27,6 @@
 
 const QString SETTINGS_PATHS_KEY = "paths";
 
-const QString SETTINGS_PATH = "/settings";
-const QString SETTINGS_PATH_JSON = SETTINGS_PATH + ".json";
-const QString CONTENT_SETTINGS_PATH_JSON = "/content-settings.json";
 const QString AGENT_STANDARD_PERMISSIONS_KEYPATH = "security.standard_permissions";
 const QString AGENT_PERMISSIONS_KEYPATH = "security.permissions";
 const QString IP_PERMISSIONS_KEYPATH = "security.ip_permissions";
@@ -37,6 +34,7 @@ const QString MAC_PERMISSIONS_KEYPATH = "security.mac_permissions";
 const QString MACHINE_FINGERPRINT_PERMISSIONS_KEYPATH = "security.machine_fingerprint_permissions";
 const QString GROUP_PERMISSIONS_KEYPATH = "security.group_permissions";
 const QString GROUP_FORBIDDENS_KEYPATH = "security.group_forbiddens";
+const QString AUTOMATIC_CONTENT_ARCHIVES_GROUP = "automatic_content_archives";
 
 using GroupByUUIDKey = QPair<QUuid, QUuid>; // groupID, rankID
 
@@ -52,11 +50,12 @@ public:
     bool handleAuthenticatedHTTPRequest(HTTPConnection* connection, const QUrl& url);
 
     void setupConfigMap(const QStringList& argumentList);
+
+    // each of the three methods in this group takes a read lock of _settingsLock
+    // and cannot be called when the a write lock is held by the same thread
     QVariant valueOrDefaultValueForKeyPath(const QString& keyPath);
-
-    QVariantMap& getSettingsMap() { return _configMap.getConfig(); }
-
-    QVariantMap& getDescriptorsMap();
+    QVariant valueForKeyPath(const QString& keyPath);
+    bool containsKeyPath(const QString& keyPath) { return valueForKeyPath(keyPath).isValid(); }
 
     // these give access to anonymous/localhost/logged-in settings from the domain-server settings page
     bool haveStandardPermissionsForName(const QString& name) const { return _standardAgentPermissions.contains(name, 0); }
@@ -111,6 +110,24 @@ public:
 
     void debugDumpGroupsState();
 
+    enum SettingsRequestAuthentication { NotAuthenticated, Authenticated };
+    enum DomainSettingsInclusion { NoDomainSettings, IncludeDomainSettings };
+    enum ContentSettingsInclusion { NoContentSettings, IncludeContentSettings };
+    enum DefaultSettingsInclusion { NoDefaultSettings, IncludeDefaultSettings };
+    enum SettingsBackupFlag { NotForBackup, ForBackup };
+
+    /// thread safe method to retrieve a JSON representation of settings
+    QJsonObject settingsResponseObjectForType(const QString& typeValue,
+                                              SettingsRequestAuthentication authentication = NotAuthenticated,
+                                              DomainSettingsInclusion domainSettingsInclusion = IncludeDomainSettings,
+                                              ContentSettingsInclusion contentSettingsInclusion = IncludeContentSettings,
+                                              DefaultSettingsInclusion defaultSettingsInclusion = IncludeDefaultSettings,
+                                              SettingsBackupFlag settingsBackupFlag = NotForBackup);
+    /// thread safe method to restore settings from a JSON object
+    Q_INVOKABLE bool restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType);
+
+    bool recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject, SettingsType settingsType);
+
 signals:
     void updateNodePermissions();
     void settingsUpdated();
@@ -130,21 +147,17 @@ private:
     QStringList _argumentList;
 
     QJsonArray filteredDescriptionArray(bool isContentSettings);
-    QJsonObject settingsResponseObjectForType(const QString& typeValue, bool isAuthenticated = false,
-                                              bool includeDomainSettings = true, bool includeContentSettings = true,
-                                              bool includeDefaults = true, bool isForBackup = false);
-    bool recurseJSONObjectAndOverwriteSettings(const QJsonObject& postedObject, SettingsType settingsType);
-
     void updateSetting(const QString& key, const QJsonValue& newValue, QVariantMap& settingMap,
                        const QJsonObject& settingDescription);
     QJsonObject settingDescriptionFromGroup(const QJsonObject& groupObject, const QString& settingName);
     void sortPermissions();
+
+    // you cannot be holding the _settingsLock when persisting to file from the same thread
+    // since it may take either a read lock or write lock and recursive locking doesn't allow a change in type
     void persistToFile();
 
     void splitSettingsDescription();
 
-    bool restoreSettingsFromObject(QJsonObject settingsToRestore, SettingsType settingsType);
-
     double _descriptionVersion;
 
     QJsonArray _descriptionArray;
@@ -152,10 +165,10 @@ private:
     QJsonArray _contentSettingsDescription;
     QJsonObject _settingsMenuGroups;
 
+    // any method that calls valueForKeyPath on this _configMap must get a write lock it keeps until it
+    // is done with the returned QVariant*
     HifiConfigVariantMap _configMap;
 
-    friend class DomainServer;
-
     // these cause calls to metaverse's group api
     void apiGetGroupID(const QString& groupName);
     void apiGetGroupRanks(const QUuid& groupID);
@@ -189,6 +202,9 @@ private:
 
     // keep track of answers to api queries about which users are in which groups
     QHash<QString, QHash<QUuid, QUuid>> _groupMembership; // QHash<user-name, QHash<group-id, rank-id>>
+
+    /// guard read/write access from multiple threads to settings 
+    QReadWriteLock _settingsLock { QReadWriteLock::Recursive };
 };
 
 #endif // hifi_DomainServerSettingsManager_h

domain-server/src/EntitiesBackupHandler.cpp

@@ -0,0 +1,83 @@
+//
+//  EntitiesBackupHandler.cpp
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/14/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#include "EntitiesBackupHandler.h"
+
+#include <QDebug>
+
+#include <quazip5/quazip.h>
+#include <quazip5/quazipfile.h>
+
+#include <OctreeDataUtils.h>
+
+EntitiesBackupHandler::EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath) :
+    _entitiesFilePath(entitiesFilePath),
+    _entitiesReplacementFilePath(entitiesReplacementFilePath)
+{
+}
+
+static const QString ENTITIES_BACKUP_FILENAME = "models.json.gz";
+
+void EntitiesBackupHandler::createBackup(const QString& backupName, QuaZip& zip) {
+    QFile entitiesFile { _entitiesFilePath };
+
+    if (entitiesFile.open(QIODevice::ReadOnly)) {
+        QuaZipFile zipFile { &zip };
+        if (!zipFile.open(QIODevice::WriteOnly, QuaZipNewInfo(ENTITIES_BACKUP_FILENAME, _entitiesFilePath))) {
+            qCritical().nospace() << "Failed to open " << ENTITIES_BACKUP_FILENAME << " for writing in zip";
+            return;
+        }
+        auto entityData = entitiesFile.readAll();
+        if (zipFile.write(entityData) != entityData.size()) {
+            qCritical() << "Failed to write entities file to backup";
+            zipFile.close();
+            return;
+        }
+        zipFile.close();
+        if (zipFile.getZipError() != UNZ_OK) {
+            qCritical().nospace() << "Failed to zip " << ENTITIES_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        }
+    }
+}
+
+void EntitiesBackupHandler::recoverBackup(const QString& backupName, QuaZip& zip) {
+    if (!zip.setCurrentFile(ENTITIES_BACKUP_FILENAME)) {
+        qWarning() << "Failed to find" << ENTITIES_BACKUP_FILENAME << "while recovering backup";
+        return;
+    }
+    QuaZipFile zipFile { &zip };
+    if (!zipFile.open(QIODevice::ReadOnly)) {
+        qCritical() << "Failed to open" << ENTITIES_BACKUP_FILENAME << "in backup";
+        return;
+    }
+    auto rawData = zipFile.readAll();
+
+    zipFile.close();
+
+    OctreeUtils::RawEntityData data;
+    if (!data.readOctreeDataInfoFromData(rawData)) {
+        qCritical() << "Unable to parse octree data during backup recovery";
+        return;
+    }
+
+    data.resetIdAndVersion();
+
+    if (zipFile.getZipError() != UNZ_OK) {
+        qCritical().nospace() << "Failed to unzip " << ENTITIES_BACKUP_FILENAME << ": " << zipFile.getZipError();
+        return;
+    }
+
+    QFile entitiesFile { _entitiesReplacementFilePath };
+
+    if (entitiesFile.open(QIODevice::WriteOnly)) {
+        entitiesFile.write(data.toGzippedByteArray());
+    }
+}

domain-server/src/EntitiesBackupHandler.h

@@ -0,0 +1,47 @@
+//
+//  EntitiesBackupHandler.h
+//  domain-server/src
+//
+//  Created by Clement Brisset on 2/14/18.
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+#ifndef hifi_EntitiesBackupHandler_h
+#define hifi_EntitiesBackupHandler_h
+
+#include "BackupHandler.h"
+
+class EntitiesBackupHandler : public BackupHandlerInterface {
+public:
+    EntitiesBackupHandler(QString entitiesFilePath, QString entitiesReplacementFilePath);
+
+    std::pair<bool, float> isAvailable(const QString& backupName) override { return { true, 1.0f }; }
+    std::pair<bool, float> getRecoveryStatus() override { return { false, 1.0f }; }
+
+    void loadBackup(const QString& backupName, QuaZip& zip) override {}
+
+    void loadingComplete() override {}
+
+    // Create a skeleton backup
+    void createBackup(const QString& backupName, QuaZip& zip) override;
+
+    // Recover from a full backup
+    void recoverBackup(const QString& backupName, QuaZip& zip) override;
+
+    // Delete a skeleton backup
+    void deleteBackup(const QString& backupName) override {}
+
+    // Create a full backup
+    void consolidateBackup(const QString& backupName, QuaZip& zip) override {}
+
+    bool isCorruptedBackup(const QString& backupName) override { return false; }
+
+private:
+    QString _entitiesFilePath;
+    QString _entitiesReplacementFilePath;
+};
+
+#endif /* hifi_EntitiesBackupHandler_h */

domain-server/src/main.cpp

@@ -22,22 +22,10 @@
 #include "DomainServer.h"
 
 int main(int argc, char* argv[]) {
-    disableQtBearerPoll(); // Fixes wifi ping spikes
-
-    QCoreApplication::setApplicationName(BuildInfo::DOMAIN_SERVER_NAME);
-    QCoreApplication::setOrganizationName(BuildInfo::MODIFIED_ORGANIZATION);
-    QCoreApplication::setOrganizationDomain(BuildInfo::ORGANIZATION_DOMAIN);
-    QCoreApplication::setApplicationVersion(BuildInfo::VERSION);
+    setupHifiApplication(BuildInfo::DOMAIN_SERVER_NAME);
 
     Setting::init();
 
-#ifndef WIN32
-    setvbuf(stdout, NULL, _IOLBF, 0);
-#endif
-
-    qInstallMessageHandler(LogHandler::verboseMessageHandler);
-    qInfo() << "Starting.";
-
     int currentExitCode = 0;
 
     // use a do-while to handle domain-server restart

ice-server/src/main.cpp

@@ -11,18 +11,13 @@
 
 #include <QtCore/QCoreApplication>
 
-#include <LogHandler.h>
+#include <SharedUtil.h>
 
 #include "IceServer.h"
 
 int main(int argc, char* argv[]) {
-#ifndef WIN32
-    setvbuf(stdout, NULL, _IOLBF, 0);
-#endif
-
-    qInstallMessageHandler(LogHandler::verboseMessageHandler);
-    qInfo() << "Starting.";
+    setupHifiApplication("Ice Server");
     
     IceServer iceServer(argc, argv);
     return iceServer.exec();
-}
+}

interface/CMakeLists.txt

@@ -204,7 +204,7 @@ endif()
 
 # link required hifi libraries
 link_hifi_libraries(
-  shared workload task octree ktx gpu gl procedural graphics render
+  shared workload task octree ktx gpu gl procedural graphics graphics-scripting render
   pointers
   recording fbx networking model-networking entities avatars trackers
   audio audio-client animation script-engine physics

interface/resources/qml/controls-uit/Separator.qml

@@ -14,8 +14,8 @@ import "../styles-uit"
 Item {
     property int colorScheme: 0;
     
-    readonly property var topColor: [ hifi.colors.baseGrayShadow, hifi.colors.faintGray ];
-    readonly property var bottomColor: [ hifi.colors.baseGrayHighlight, hifi.colors.faintGray ];
+    readonly property var topColor: [ hifi.colors.baseGrayShadow, hifi.colors.faintGray, "#89858C" ];
+    readonly property var bottomColor: [ hifi.colors.baseGrayHighlight, hifi.colors.faintGray, "#89858C" ];
 
     // Size
     height: colorScheme === 0 ? 2 : 1;

interface/resources/qml/controls-uit/Table.qml

@@ -22,6 +22,10 @@ TableView {
     readonly property bool isLightColorScheme: colorScheme == hifi.colorSchemes.light
     property bool expandSelectedRow: false
     property bool centerHeaderText: false
+    readonly property real headerSpacing: 3 //spacing between sort indicator and table header title
+    property var titlePaintedPos: [] // storing extra data position behind painted
+                                     // title text and sort indicatorin table's header
+    signal titlePaintedPosSignal(int column) //signal that extradata position gets changed
 
     model: ListModel { }
 
@@ -69,36 +73,39 @@ TableView {
         height: hifi.dimensions.tableHeaderHeight
         color: isLightColorScheme ? hifi.colors.tableBackgroundLight : hifi.colors.tableBackgroundDark
 
+
         RalewayRegular {
             id: titleText
+            x: centerHeaderText ? (parent.width - paintedWidth -
+                                  ((sortIndicatorVisible &&
+                                    sortIndicatorColumn === styleData.column) ?
+                                       (titleSort.paintedWidth / 5 + tableView.headerSpacing) : 0)) / 2 :
+                                  hifi.dimensions.tablePadding
             text: styleData.value
             size: hifi.fontSizes.tableHeading
             font.capitalization: Font.AllUppercase
             color: hifi.colors.baseGrayHighlight
             horizontalAlignment: (centerHeaderText ? Text.AlignHCenter : Text.AlignLeft)
-            anchors {
-                left: parent.left
-                leftMargin: hifi.dimensions.tablePadding
-                right: parent.right
-                rightMargin: hifi.dimensions.tablePadding
-                verticalCenter: parent.verticalCenter
-            }
+            anchors.verticalCenter: parent.verticalCenter
         }
 
+        //actual image of sort indicator in glyph font only 20% of real font size
+        //i.e. if the charachter size set to 60 pixels, actual image is 12 pixels
         HiFiGlyphs {
             id: titleSort
             text:  sortIndicatorOrder == Qt.AscendingOrder ? hifi.glyphs.caratUp : hifi.glyphs.caratDn
             color: hifi.colors.darkGray
             opacity: 0.6;
             size: hifi.fontSizes.tableHeadingIcon
-            anchors {
-                left: titleText.right
-                leftMargin: -hifi.fontSizes.tableHeadingIcon / 3 - (centerHeaderText ? 15 : 10)
-                right: parent.right
-                rightMargin: hifi.dimensions.tablePadding
-                verticalCenter: titleText.verticalCenter
-            }
+            anchors.verticalCenter: titleText.verticalCenter
+            anchors.left: titleText.right
+            anchors.leftMargin: -(hifi.fontSizes.tableHeadingIcon / 2.5) + tableView.headerSpacing
             visible: sortIndicatorVisible && sortIndicatorColumn === styleData.column
+            onXChanged: {
+                titlePaintedPos[styleData.column] = titleText.x + titleText.paintedWidth +
+                        paintedWidth / 5 + tableView.headerSpacing*2
+                titlePaintedPosSignal(styleData.column)
+            }
         }
 
         Rectangle {
@@ -152,7 +159,7 @@ TableView {
         color: styleData.selected
                ? hifi.colors.primaryHighlight
                : tableView.isLightColorScheme
-                   ? (styleData.alternate ? hifi.colors.tableRowLightEven : hifi.colors.tableRowLightOdd)
-                   : (styleData.alternate ? hifi.colors.tableRowDarkEven : hifi.colors.tableRowDarkOdd)
+                 ? (styleData.alternate ? hifi.colors.tableRowLightEven : hifi.colors.tableRowLightOdd)
+                 : (styleData.alternate ? hifi.colors.tableRowDarkEven : hifi.colors.tableRowDarkOdd)
     }
 }

interface/resources/qml/controls-uit/TextField.qml

@@ -34,11 +34,10 @@ TextField {
 
     placeholderText: textField.placeholderText
 
-    FontLoader { id: firaSansSemiBold; source: "../../fonts/FiraSans-SemiBold.ttf"; }
+    FontLoader { id: firaSansRegular; source: "../../fonts/FiraSans-Regular.ttf"; }
     FontLoader { id: hifiGlyphs; source: "../../fonts/hifi-glyphs.ttf"; }
-    font.family: firaSansSemiBold.name
+    font.family: firaSansRegular.name
     font.pixelSize: hifi.fontSizes.textFieldInput
-    font.italic: textField.text == ""
     height: implicitHeight + 3  // Make surrounding box higher so that highlight is vertically centered.
     property alias textFieldLabel: textFieldLabel
 

interface/resources/qml/hifi/AssetServer.qml

@@ -206,7 +206,7 @@ Windows.ScrollingWindow {
             SHAPE_TYPES[SHAPE_TYPE_BOX] = "Box";
             SHAPE_TYPES[SHAPE_TYPE_SPHERE] = "Sphere";
         
-            var SHAPE_TYPE_DEFAULT = SHAPE_TYPE_STATIC_MESH;
+            var SHAPE_TYPE_DEFAULT = SHAPE_TYPE_SIMPLE_COMPOUND;
             var DYNAMIC_DEFAULT = false;
             var prompt = desktop.customInputDialog({
                 textInput: {

interface/resources/qml/hifi/Pal.qml

@@ -30,7 +30,7 @@ Rectangle {
     property int myCardWidth: width - upperRightInfoContainer.width;
     property int myCardHeight: 100;
     property int rowHeight: 60;
-    property int actionButtonWidth: 55;
+    property int actionButtonWidth: 65;
     property int locationColumnWidth: 170;
     property int nearbyNameCardWidth: nearbyTable.width - (iAmAdmin ? (actionButtonWidth * 4) : (actionButtonWidth * 2)) - 4 - hifi.dimensions.scrollbarBackgroundWidth;
     property int connectionsNameCardWidth: connectionsTable.width - locationColumnWidth - actionButtonWidth - 4 - hifi.dimensions.scrollbarBackgroundWidth;
@@ -415,6 +415,7 @@ Rectangle {
                 movable: false;
                 resizable: false;
             }
+
             TableViewColumn {
                 role: "ignore";
                 title: "IGNORE";
@@ -599,13 +600,23 @@ Rectangle {
         }
         // This Rectangle refers to the [?] popup button next to "NAMES"
         Rectangle {
+            id: questionRect
             color: hifi.colors.tableBackgroundLight;
             width: 20;
             height: hifi.dimensions.tableHeaderHeight - 2;
             anchors.left: nearbyTable.left;
             anchors.top: nearbyTable.top;
             anchors.topMargin: 1;
-            anchors.leftMargin: actionButtonWidth + nearbyNameCardWidth/2 + displayNameHeaderMetrics.width/2 + 6;
+
+            Connections {
+                target: nearbyTable
+                onTitlePaintedPosSignal: {
+                    if (column === 1) { // name column
+                        questionRect.anchors.leftMargin = actionButtonWidth + nearbyTable.titlePaintedPos[column]
+                    }
+                }
+            }
+
             RalewayRegular {
                 id: helpText;
                 text: "[?]";

interface/resources/qml/hifi/commerce/checkout/Checkout.qml

@@ -28,7 +28,7 @@ Rectangle {
     id: root;
     objectName: "checkout"
     property string activeView: "initialize";
-    property bool purchasesReceived: false;
+    property bool ownershipStatusReceived: false;
     property bool balanceReceived: false;
     property string itemName;
     property string itemId;
@@ -37,12 +37,18 @@ Rectangle {
     property double balanceAfterPurchase;
     property bool alreadyOwned: false;
     property int itemPrice: -1;
-    property bool itemIsJson: true;
+    property bool isCertified;
+    property string itemType;
+    property var itemTypesArray: ["entity", "wearable", "contentSet", "app", "avatar"];
+    property var itemTypesText: ["entity", "wearable", "content set", "app", "avatar"];
+    property var buttonTextNormal: ["REZ", "WEAR", "REPLACE CONTENT SET", "INSTALL", "WEAR"];
+    property var buttonTextClicked: ["REZZED!", "WORN!", "CONTENT SET REPLACED!", "INSTALLED!", "AVATAR CHANGED!"]
+    property var buttonGlyph: [hifi.glyphs.wand, hifi.glyphs.hat, hifi.glyphs.globe, hifi.glyphs.install, hifi.glyphs.avatar];
     property bool shouldBuyWithControlledFailure: false;
     property bool debugCheckoutSuccess: false;
     property bool canRezCertifiedItems: Entities.canRezCertified() || Entities.canRezTmpCertified();
-    property bool isWearable;
     property string referrer;
+    property bool isInstalled;
     // Style
     color: hifi.colors.white;
     Connections {
@@ -53,17 +59,17 @@ Rectangle {
                 if (root.activeView !== "needsLogIn") {
                     root.activeView = "needsLogIn";
                 }
-            } else if (walletStatus === 1) {
+            } else if ((walletStatus === 1) || (walletStatus === 2) || (walletStatus === 3)) {
                 if (root.activeView !== "notSetUp") {
                     root.activeView = "notSetUp";
                     notSetUpTimer.start();
                 }
-            } else if (walletStatus === 2) {
+            } else if (walletStatus === 4) {
                 if (root.activeView !== "passphraseModal") {
                     root.activeView = "passphraseModal";
                     UserActivityLogger.commercePassphraseEntry("marketplace checkout");
                 }
-            } else if (walletStatus === 3) {
+            } else if (walletStatus === 5) {
                 authSuccessStep();
             } else {
                 console.log("ERROR in Checkout.qml: Unknown wallet status: " + walletStatus);
@@ -85,7 +91,9 @@ Rectangle {
                 UserActivityLogger.commercePurchaseFailure(root.itemId, root.itemAuthor, root.itemPrice, !root.alreadyOwned, result.message);
             } else {
                 root.itemHref = result.data.download_url;
-                root.isWearable = result.data.categories.indexOf("Wearables") > -1;
+                if (result.data.categories.indexOf("Wearables") > -1) {
+                    root.itemType = "wearable";
+                }
                 root.activeView = "checkoutSuccess";
                 UserActivityLogger.commercePurchaseSuccess(root.itemId, root.itemAuthor, root.itemPrice, !root.alreadyOwned);
             }
@@ -97,32 +105,60 @@ Rectangle {
             } else {
                 root.balanceReceived = true;
                 root.balanceAfterPurchase = result.data.balance - root.itemPrice;
-                root.setBuyText();
+                root.refreshBuyUI();
             }
         }
 
-        onInventoryResult: {
+        onAlreadyOwnedResult: {
             if (result.status !== 'success') {
-                console.log("Failed to get purchases", result.data.message);
+                console.log("Failed to get Already Owned status", result.data.message);
             } else {
-                root.purchasesReceived = true;
-                if (purchasesContains(result.data.assets, itemId)) {
-                    root.alreadyOwned = true;
+                root.ownershipStatusReceived = true;
+                if (result.data.marketplace_item_id === root.itemId) {
+                    root.alreadyOwned = result.data.already_owned;
                 } else {
+                    console.log("WARNING - Received 'Already Owned' status about different Marketplace ID!");
                     root.alreadyOwned = false;
                 }
-                root.setBuyText();
+                root.refreshBuyUI();
+            }
+        }
+
+        onAppInstalled: {
+            if (appHref === root.itemHref) {
+                root.isInstalled = true;
             }
         }
     }
 
     onItemIdChanged: {
-        Commerce.inventory();
+        root.ownershipStatusReceived = false;
+        Commerce.alreadyOwned(root.itemId);
         itemPreviewImage.source = "https://hifi-metaverse.s3-us-west-1.amazonaws.com/marketplace/previews/" + itemId + "/thumbnail/hifi-mp-" + itemId + ".jpg";
     }
 
     onItemHrefChanged: {
-        itemIsJson = root.itemHref.endsWith('.json');
+        if (root.itemHref.indexOf(".fst") > -1) {
+            root.itemType = "avatar";
+        } else if (root.itemHref.indexOf('.json.gz') > -1) {
+            root.itemType = "contentSet";
+        } else if (root.itemHref.indexOf('.app.json') > -1) {
+            root.itemType = "app";
+        } else if (root.itemHref.indexOf('.json') > -1) {
+            root.itemType = "entity"; // "wearable" type handled later
+        } else {
+            console.log("WARNING - Item type is UNKNOWN!");
+            root.itemType = "entity";
+        }
+    }
+
+    onItemTypeChanged: {
+        if (root.itemType === "entity" || root.itemType === "wearable" ||
+            root.itemType === "contentSet" || root.itemType === "avatar" || root.itemType === "app") {
+            root.isCertified = true;
+        } else {
+            root.isCertified = false;
+        }
     }
 
     onItemPriceChanged: {
@@ -203,7 +239,7 @@ Rectangle {
         color: hifi.colors.white;
 
         Component.onCompleted: {
-            purchasesReceived = false;
+            ownershipStatusReceived = false;
             balanceReceived = false;
             Commerce.getWalletStatus();
         }
@@ -278,6 +314,32 @@ Rectangle {
         anchors.left: parent.left;
         anchors.right: parent.right;
 
+        Rectangle {
+            id: loading;
+            z: 997;
+            visible: !root.ownershipStatusReceived || !root.balanceReceived;
+            anchors.fill: parent;
+            color: hifi.colors.white;
+
+            // This object is always used in a popup.
+            // This MouseArea is used to prevent a user from being
+            //     able to click on a button/mouseArea underneath the popup/section.
+            MouseArea {
+                anchors.fill: parent;
+                hoverEnabled: true;
+                propagateComposedEvents: false;
+            }
+                
+            AnimatedImage {
+                id: loadingImage;
+                source: "../common/images/loader-blue.gif"
+                width: 74;
+                height: width;
+                anchors.verticalCenter: parent.verticalCenter;
+                anchors.horizontalCenter: parent.horizontalCenter;
+            }
+        }
+
         RalewayRegular {
             id: confirmPurchaseText;
             anchors.top: parent.top;
@@ -286,8 +348,8 @@ Rectangle {
             anchors.leftMargin: 16;
             width: paintedWidth;
             height: paintedHeight;
-            text: "Confirm Purchase:";
-            color: hifi.colors.baseGray;
+            text: "Review Purchase:";
+            color: hifi.colors.black;
             size: 28;
         }
         
@@ -400,7 +462,7 @@ Rectangle {
             width: root.width;
             // Anchors
             anchors.top: separator2.bottom;
-            anchors.topMargin: 16;
+            anchors.topMargin: 0;
             anchors.left: parent.left;
             anchors.leftMargin: 16;
             anchors.right: parent.right;
@@ -411,8 +473,8 @@ Rectangle {
             Rectangle {
                 id: buyTextContainer;
                 visible: buyText.text !== "";
-                anchors.top: cancelPurchaseButton.bottom;
-                anchors.topMargin: 16;
+                anchors.top: parent.top;
+                anchors.topMargin: 10;
                 anchors.left: parent.left;
                 anchors.right: parent.right;
                 height: buyText.height + 30;
@@ -454,32 +516,63 @@ Rectangle {
                     // Alignment
                     horizontalAlignment: Text.AlignLeft;
                     verticalAlignment: Text.AlignVCenter;
+                }
+            }
 
-                    onLinkActivated: {
-                        sendToScript({method: 'checkout_goToPurchases', filterText: root.itemName});
-                    }
+            // "View in My Purchases" button
+            HifiControlsUit.Button {
+                id: viewInMyPurchasesButton;
+                visible: false;
+                color: hifi.buttons.blue;
+                colorScheme: hifi.colorSchemes.light;
+                anchors.top: buyTextContainer.visible ? buyTextContainer.bottom : checkoutActionButtonsContainer.top;
+                anchors.topMargin: 10;
+                height: 50;
+                anchors.left: parent.left;
+                anchors.right: parent.right;
+                text: "VIEW THIS ITEM IN MY PURCHASES";
+                onClicked: {
+                    sendToScript({method: 'checkout_goToPurchases', filterText: root.itemName});
                 }
             }
 
             // "Buy" button
             HifiControlsUit.Button {
                 id: buyButton;
-                enabled: (root.balanceAfterPurchase >= 0 && purchasesReceived && balanceReceived) || !itemIsJson;
-                color: hifi.buttons.blue;
+                visible: !((root.itemType === "avatar" || root.itemType === "app") && viewInMyPurchasesButton.visible)
+                enabled: (root.balanceAfterPurchase >= 0 && ownershipStatusReceived && balanceReceived) || (!root.isCertified);
+                color: viewInMyPurchasesButton.visible ? hifi.buttons.white : hifi.buttons.blue;
                 colorScheme: hifi.colorSchemes.light;
-                anchors.top: checkoutActionButtonsContainer.top;
-                anchors.topMargin: 16;
-                height: 40;
+                anchors.top: viewInMyPurchasesButton.visible ? viewInMyPurchasesButton.bottom :
+                    (buyTextContainer.visible ? buyTextContainer.bottom : checkoutActionButtonsContainer.top);
+                anchors.topMargin: 10;
+                height: 50;
                 anchors.left: parent.left;
                 anchors.right: parent.right;
-                text: (itemIsJson ? ((purchasesReceived && balanceReceived) ? "Confirm Purchase" : "--") : "Get Item");
+                text: ((root.isCertified) ? ((ownershipStatusReceived && balanceReceived) ?
+                    (viewInMyPurchasesButton.visible ? "Buy It Again" : "Confirm Purchase") : "--") : "Get Item");
                 onClicked: {
-                    if (itemIsJson) {
-                        buyButton.enabled = false;
+                    if (root.isCertified) {
                         if (!root.shouldBuyWithControlledFailure) {
-                            Commerce.buy(itemId, itemPrice);
+                            if (root.itemType === "contentSet" && !Entities.canReplaceContent()) {
+                                lightboxPopup.titleText = "Purchase Content Set";
+                                lightboxPopup.bodyText = "You will not be able to replace this domain's content with <b>" + root.itemName +
+                                    " </b>until the server owner gives you 'Replace Content' permissions.<br><br>Are you sure you want to purchase this content set?";
+                                lightboxPopup.button1text = "CANCEL";
+                                lightboxPopup.button1method = "root.visible = false;"
+                                lightboxPopup.button2text = "CONFIRM";
+                                lightboxPopup.button2method = "Commerce.buy('" + root.itemId + "', " + root.itemPrice + ");" +
+                                    "root.visible = false; buyButton.enabled = false; loading.visible = true;";
+                                lightboxPopup.visible = true;
+                            } else {
+                                buyButton.enabled = false;
+                                loading.visible = true;
+                                Commerce.buy(root.itemId, root.itemPrice);
+                            }
                         } else {
-                            Commerce.buy(itemId, itemPrice, true);
+                            buyButton.enabled = false;
+                            loading.visible = true;
+                            Commerce.buy(root.itemId, root.itemPrice, true);
                         }
                     } else {
                         if (urlHandler.canHandleUrl(itemHref)) {
@@ -494,9 +587,9 @@ Rectangle {
                 id: cancelPurchaseButton;
                 color: hifi.buttons.noneBorderlessGray;
                 colorScheme: hifi.colorSchemes.light;
-                anchors.top: buyButton.bottom;
-                anchors.topMargin: 16;
-                height: 40;
+                anchors.top: buyButton.visible ? buyButton.bottom : viewInMyPurchasesButton.bottom;
+                anchors.topMargin: 10;
+                height: 50;
                 anchors.left: parent.left;
                 anchors.right: parent.right;
                 text: "Cancel"
@@ -522,31 +615,32 @@ Rectangle {
         anchors.top: titleBarContainer.bottom;
         anchors.bottom: root.bottom;
         anchors.left: parent.left;
-        anchors.leftMargin: 16;
+        anchors.leftMargin: 20;
         anchors.right: parent.right;
-        anchors.rightMargin: 16;
+        anchors.rightMargin: 20;
 
         RalewayRegular {
             id: completeText;
             anchors.top: parent.top;
-            anchors.topMargin: 30;
+            anchors.topMargin: 18;
             anchors.left: parent.left;
             width: paintedWidth;
             height: paintedHeight;
             text: "Thank you for your order!";
             color: hifi.colors.baseGray;
-            size: 28;
+            size: 36;
         }
 
         RalewaySemiBold {
             id: completeText2;
-            text: "The item " + '<font color="' + hifi.colors.blueAccent + '"><a href="#">' + root.itemName + '</a></font>' +
-            " has been added to your Purchases and a receipt will appear in your Wallet's transaction history.";
+            text: "The " + (root.itemTypesText)[itemTypesArray.indexOf(root.itemType)] +
+                ' <font color="' + hifi.colors.blueAccent + '"><a href="#">' + root.itemName + '</a></font>' +
+                " has been added to your Purchases and a receipt will appear in your Wallet's transaction history.";
             // Text size
-            size: 20;
+            size: 18;
             // Anchors
             anchors.top: completeText.bottom;
-            anchors.topMargin: 10;
+            anchors.topMargin: 15;
             height: paintedHeight;
             anchors.left: parent.left;
             anchors.right: parent.right;
@@ -576,7 +670,7 @@ Rectangle {
 
             RalewayBold {
                 anchors.fill: parent;
-                text: "REZZED";
+                text: (root.buttonTextClicked)[itemTypesArray.indexOf(root.itemType)];
                 size: 18;
                 color: hifi.colors.white;
                 verticalAlignment: Text.AlignVCenter;
@@ -592,26 +686,58 @@ Rectangle {
         // "Rez" button
         HifiControlsUit.Button {
             id: rezNowButton;
-            enabled: root.canRezCertifiedItems || root.isWearable;
-            buttonGlyph: hifi.glyphs.lightning;
+            enabled: (root.itemType === "entity" && root.canRezCertifiedItems) ||
+                (root.itemType === "contentSet" && Entities.canReplaceContent()) ||
+                root.itemType === "wearable" || root.itemType === "avatar" || root.itemType === "app";
+            buttonGlyph: (root.buttonGlyph)[itemTypesArray.indexOf(root.itemType)];
             color: hifi.buttons.red;
             colorScheme: hifi.colorSchemes.light;
             anchors.top: completeText2.bottom;
-            anchors.topMargin: 30;
+            anchors.topMargin: 27;
             height: 50;
             anchors.left: parent.left;
             anchors.right: parent.right;
-            text: root.isWearable ? "Wear It" : "Rez It"
+            text: root.itemType === "app" && root.isInstalled ? "OPEN APP" : (root.buttonTextNormal)[itemTypesArray.indexOf(root.itemType)];
             onClicked: {
-                sendToScript({method: 'checkout_rezClicked', itemHref: root.itemHref, isWearable: root.isWearable});
-                rezzedNotifContainer.visible = true;
-                rezzedNotifContainerTimer.start();
-                UserActivityLogger.commerceEntityRezzed(root.itemId, "checkout", root.isWearable ? "rez" : "wear");
+                if (root.itemType === "contentSet") {
+                    lightboxPopup.titleText = "Replace Content";
+                    lightboxPopup.bodyText = "Rezzing this content set will replace the existing environment and all of the items in this domain. " +
+                        "If you want to save the state of the content in this domain, create a backup before proceeding.<br><br>" +
+                        "For more information about backing up and restoring content, " +
+                        "<a href='https://docs.highfidelity.com/create-and-explore/start-working-in-your-sandbox/restoring-sandbox-content'>" +
+                        "click here to open info on your desktop browser.";
+                    lightboxPopup.button1text = "CANCEL";
+                    lightboxPopup.button1method = "root.visible = false;"
+                    lightboxPopup.button2text = "CONFIRM";
+                    lightboxPopup.button2method = "Commerce.replaceContentSet('" + root.itemHref + "');" + 
+                    "root.visible = false;rezzedNotifContainer.visible = true; rezzedNotifContainerTimer.start();" + 
+                    "UserActivityLogger.commerceEntityRezzed('" + root.itemId + "', 'checkout', '" + root.itemType + "');";
+                    lightboxPopup.visible = true;
+                } else if (root.itemType === "avatar") {
+                    lightboxPopup.titleText = "Change Avatar";
+                    lightboxPopup.bodyText = "This will change your current avatar to " + root.itemName + " while retaining your wearables.";
+                    lightboxPopup.button1text = "CANCEL";
+                    lightboxPopup.button1method = "root.visible = false;"
+                    lightboxPopup.button2text = "CONFIRM";
+                    lightboxPopup.button2method = "MyAvatar.useFullAvatarURL('" + root.itemHref + "'); root.visible = false;";
+                    lightboxPopup.visible = true;
+                } else if (root.itemType === "app") {
+                    if (root.isInstalled) {
+                        Commerce.openApp(root.itemHref);
+                    } else {
+                        Commerce.installApp(root.itemHref);
+                    }
+                } else {
+                    sendToScript({method: 'checkout_rezClicked', itemHref: root.itemHref, itemType: root.itemType});
+                    rezzedNotifContainer.visible = true;
+                    rezzedNotifContainerTimer.start();
+                    UserActivityLogger.commerceEntityRezzed(root.itemId, "checkout", root.itemType);
+                }
             }
         }
         RalewaySemiBold {
             id: noPermissionText;
-            visible: !root.canRezCertifiedItems && !root.isWearable;
+            visible: !root.canRezCertifiedItems && root.itemType === "entity";
             text: '<font color="' + hifi.colors.redAccent + '"><a href="#">You do not have Certified Rez permissions in this domain.</a></font>'
             // Text size
             size: 16;
@@ -640,7 +766,7 @@ Rectangle {
         }
         RalewaySemiBold {
             id: explainRezText;
-            visible: !root.isWearable;
+            visible: root.itemType === "entity";
             text: '<font color="' + hifi.colors.redAccent + '"><a href="#">What does "Rez" mean?</a></font>'
             // Text size
             size: 16;
@@ -663,9 +789,9 @@ Rectangle {
 
         RalewaySemiBold {
             id: myPurchasesLink;
-            text: '<font color="' + hifi.colors.blueAccent + '"><a href="#">View this item in My Purchases</a></font>';
+            text: '<font color="' + hifi.colors.primaryHighlight + '"><a href="#">View this item in My Purchases</a></font>';
             // Text size
-            size: 20;
+            size: 18;
             // Anchors
             anchors.top: explainRezText.visible ? explainRezText.bottom : (noPermissionText.visible ? noPermissionText.bottom : rezNowButton.bottom);
             anchors.topMargin: 40;
@@ -685,12 +811,12 @@ Rectangle {
 
         RalewaySemiBold {
             id: walletLink;
-            text: '<font color="' + hifi.colors.blueAccent + '"><a href="#">View receipt in Wallet</a></font>';
+            text: '<font color="' + hifi.colors.primaryHighlight + '"><a href="#">View receipt in Wallet</a></font>';
             // Text size
-            size: 20;
+            size: 18;
             // Anchors
             anchors.top: myPurchasesLink.bottom;
-            anchors.topMargin: 20;
+            anchors.topMargin: 16;
             height: paintedHeight;
             anchors.left: parent.left;
             anchors.right: parent.right;
@@ -708,12 +834,12 @@ Rectangle {
         RalewayRegular {
             id: pendingText;
             text: 'Your item is marked "pending" while your purchase is being confirmed. ' +
-            '<font color="' + hifi.colors.blueAccent + '"><a href="#">Learn More</a></font>';
+            '<b><font color="' + hifi.colors.primaryHighlight + '"><a href="#">Learn More</a></font></b>';
             // Text size
-            size: 20;
+            size: 18;
             // Anchors
             anchors.top: walletLink.bottom;
-            anchors.topMargin: 60;
+            anchors.topMargin: 32;
             height: paintedHeight;
             anchors.left: parent.left;
             anchors.right: parent.right;
@@ -739,11 +865,10 @@ Rectangle {
             color: hifi.buttons.noneBorderlessGray;
             colorScheme: hifi.colorSchemes.light;
             anchors.bottom: parent.bottom;
-            anchors.bottomMargin: 20;
+            anchors.bottomMargin: 54;
             anchors.right: parent.right;
-            anchors.rightMargin: 14;
-            width: parent.width/2 - anchors.rightMargin;
-            height: 60;
+            width: 193;
+            height: 44;
             text: "Continue Shopping";
             onClicked: {
                 sendToScript({method: 'checkout_continueShopping', itemId: itemId});
@@ -851,7 +976,7 @@ Rectangle {
                 buyButton.color = hifi.buttons.red;
                 root.shouldBuyWithControlledFailure = true;
             } else {
-                buyButton.text = (itemIsJson ? ((purchasesReceived && balanceReceived) ? (root.alreadyOwned ? "Buy Another" : "Buy"): "--") : "Get Item");
+                buyButton.text = (root.isCertified ? ((ownershipStatusReceived && balanceReceived) ? (root.alreadyOwned ? "Buy Another" : "Buy"): "--") : "Get Item");
                 buyButton.color = hifi.buttons.blue;
                 root.shouldBuyWithControlledFailure = false;
             }
@@ -883,7 +1008,7 @@ Rectangle {
                 itemHref = message.params.itemHref;
                 referrer = message.params.referrer;
                 itemAuthor = message.params.itemAuthor;
-                setBuyText();
+                refreshBuyUI();
             break;
             default:
                 console.log('Unrecognized message from marketplaces.js:', JSON.stringify(message));
@@ -891,22 +1016,13 @@ Rectangle {
     }
     signal sendToScript(var message);
 
-    function purchasesContains(purchasesJson, id) {
-        for (var idx = 0; idx < purchasesJson.length; idx++) {
-            if(purchasesJson[idx].id === id) {
-                return true;
-            }
-        }
-        return false;
-    }
-
-    function setBuyText() {
-        if (root.itemIsJson) {
-            if (root.purchasesReceived && root.balanceReceived) {
+    function refreshBuyUI() {
+        if (root.isCertified) {
+            if (root.ownershipStatusReceived && root.balanceReceived) {
                 if (root.balanceAfterPurchase < 0) {
                     if (root.alreadyOwned) {
-                        buyText.text = "<b>Your Wallet does not have sufficient funds to purchase this item again.<br>" +
-                        '<font color="' + hifi.colors.blueAccent + '"><a href="#">View the copy you own in My Purchases</a></font></b>';
+                        buyText.text = "<b>Your Wallet does not have sufficient funds to purchase this item again.</b>";
+                        viewInMyPurchasesButton.visible = true;
                     } else {
                         buyText.text = "<b>Your Wallet does not have sufficient funds to purchase this item.</b>";
                     }
@@ -916,15 +1032,19 @@ Rectangle {
                     buyGlyph.size = 54;
                 } else {
                     if (root.alreadyOwned) {
-                        buyText.text = '<b>You already own this item.<br>Purchasing it will buy another copy.<br><font color="'
-                        + hifi.colors.blueAccent + '"><a href="#">View this item in My Purchases</a></font></b>';
-                        buyTextContainer.color = "#FFD6AD";
-                        buyTextContainer.border.color = "#FAC07D";
-                        buyGlyph.text = hifi.glyphs.alert;
-                        buyGlyph.size = 46;
+                        viewInMyPurchasesButton.visible = true;
                     } else {
                         buyText.text = "";
                     }
+
+                    if (root.itemType === "contentSet" && !Entities.canReplaceContent()) {
+                        buyText.text = "The domain owner must enable 'Replace Content' permissions for you in this " +
+                            "<b>domain's server settings</b> before you can replace this domain's content with <b>" + root.itemName + "</b>";
+                        buyTextContainer.color = "#FFC3CD";
+                        buyTextContainer.border.color = "#F3808F";
+                        buyGlyph.text = hifi.glyphs.alert;
+                        buyGlyph.size = 54;
+                    }
                 }
             } else {
                 buyText.text = "";
@@ -945,8 +1065,8 @@ Rectangle {
             root.activeView = "checkoutSuccess";
         }
         root.balanceReceived = false;
-        root.purchasesReceived = false;
-        Commerce.inventory();
+        root.ownershipStatusReceived = false;
+        Commerce.alreadyOwned(root.itemId);
         Commerce.balance();
     }
 

interface/resources/qml/hifi/commerce/common/CommerceLightbox.qml

@@ -25,10 +25,13 @@ Rectangle {
     property string titleText;
     property string bodyImageSource;
     property string bodyText;
+    property string button1color: hifi.buttons.noneBorderlessGray;
     property string button1text;
     property string button1method;
+    property string button2color: hifi.buttons.noneBorderless;
     property string button2text;
     property string button2method;
+    property string buttonLayout: "leftright";
 
     readonly property string securityPicBodyText: "When you see your Security Pic, your actions and data are securely making use of your " +
         "Wallet's private keys.<br><br>You can change your Security Pic in your Wallet.";
@@ -39,6 +42,12 @@ Rectangle {
     color: Qt.rgba(0, 0, 0, 0.5);
     z: 999;
 
+    onVisibleChanged: {
+        if (!visible) {
+            resetLightbox();
+        }
+    }
+
     // This object is always used in a popup.
     // This MouseArea is used to prevent a user from being
     //     able to click on a button/mouseArea underneath the popup.
@@ -100,6 +109,10 @@ Rectangle {
             size: 20;
             verticalAlignment: Text.AlignTop;
             wrapMode: Text.WordWrap;
+
+            onLinkActivated: {
+                sendToParent({ method: 'commerceLightboxLinkClicked', linkUrl: link });
+            }
         }
 
         Item {
@@ -108,18 +121,21 @@ Rectangle {
             anchors.topMargin: 30;
             anchors.left: parent.left;
             anchors.right: parent.right;
-            height: 70;
+            height: root.buttonLayout === "leftright" ? 70 : 150;
 
             // Button 1
             HifiControlsUit.Button {
-                color: hifi.buttons.noneBorderlessGray;
+                id: button1;
+                color: root.button1color;
                 colorScheme: hifi.colorSchemes.light;
-                anchors.top: parent.top;
-                anchors.bottom: parent.bottom;
-                anchors.bottomMargin: 20;
+                anchors.top: root.buttonLayout === "leftright" ? parent.top : parent.top;
                 anchors.left: parent.left;
                 anchors.leftMargin: 10;
-                width: root.button2text ? parent.width/2 - anchors.leftMargin*2 : parent.width - anchors.leftMargin * 2;
+                anchors.right: root.buttonLayout === "leftright" ? undefined : parent.right;
+                anchors.rightMargin: root.buttonLayout === "leftright" ? undefined : 10;
+                width: root.buttonLayout === "leftright" ? (root.button2text ? parent.width/2 - anchors.leftMargin*2 : parent.width - anchors.leftMargin * 2) :
+                    (undefined);
+                height: 50;
                 text: root.button1text;
                 onClicked: {
                     eval(button1method);
@@ -128,15 +144,18 @@ Rectangle {
 
             // Button 2
             HifiControlsUit.Button {
+                id: button2;
                 visible: root.button2text;
-                color: hifi.buttons.noneBorderless;
+                color: root.button2color;
                 colorScheme: hifi.colorSchemes.light;
-                anchors.top: parent.top;
-                anchors.bottom: parent.bottom;
-                anchors.bottomMargin: 20;
+                anchors.top: root.buttonLayout === "leftright" ? parent.top : button1.bottom;
+                anchors.topMargin: root.buttonLayout === "leftright" ? undefined : 20;
+                anchors.left: root.buttonLayout === "leftright" ? undefined : parent.left;
+                anchors.leftMargin: root.buttonLayout === "leftright" ? undefined : 10;
                 anchors.right: parent.right;
                 anchors.rightMargin: 10;
-                width: parent.width/2 - anchors.rightMargin*2;
+                width: root.buttonLayout === "leftright" ? parent.width/2 - anchors.rightMargin*2 : undefined;
+                height: 50;
                 text: root.button2text;
                 onClicked: {
                     eval(button2method);
@@ -149,6 +168,19 @@ Rectangle {
     // FUNCTION DEFINITIONS START
     //
     signal sendToParent(var msg);
+
+    function resetLightbox() {
+        root.titleText = "";
+        root.bodyImageSource = "";
+        root.bodyText = "";
+        root.button1color = hifi.buttons.noneBorderlessGray;
+        root.button1text = "";
+        root.button1method = "";
+        root.button2color = hifi.buttons.noneBorderless;
+        root.button2text = "";
+        root.button2method = "";
+        root.buttonLayout = "leftright";
+    }
     //
     // FUNCTION DEFINITIONS END
     //

interface/resources/qml/hifi/commerce/common/EmulatedMarketplaceHeader.qml

@@ -37,9 +37,9 @@ Item {
         onWalletStatusResult: {
             if (walletStatus === 0) {
                 sendToParent({method: "needsLogIn"});
-            } else if (walletStatus === 3) {
+            } else if (walletStatus === 5) {
                 Commerce.getSecurityImage();
-            } else if (walletStatus > 3) {
+            } else if (walletStatus > 5) {
                 console.log("ERROR in EmulatedMarketplaceHeader.qml: Unknown wallet status: " + walletStatus);
             }
         }

interface/resources/qml/hifi/commerce/common/FirstUseTutorial.qml

@@ -90,11 +90,11 @@ Rectangle {
             id: introText2;
             text: "My Purchases";
             // Text size
-            size: 28;
+            size: 22;
             // Anchors
             anchors.top: introText1.bottom;
             anchors.left: parent.left;
-            anchors.leftMargin: 12;
+            anchors.leftMargin: 24;
             anchors.right: parent.right;
             height: paintedHeight;
             // Style

interface/resources/qml/hifi/commerce/inspectionCertificate/InspectionCertificate.qml

@@ -208,6 +208,7 @@ Rectangle {
         //     able to click on a button/mouseArea underneath the popup/section.
         MouseArea {
             anchors.fill: parent;
+            hoverEnabled: true;
             propagateComposedEvents: false;
         }
                 

interface/resources/qml/hifi/commerce/purchases/PurchasedItem.qml

@@ -20,6 +20,7 @@ import "../../../styles-uit"
 import "../../../controls-uit" as HifiControlsUit
 import "../../../controls" as HifiControls
 import "../wallet" as HifiWallet
+import TabletScriptingInterface 1.0
 
 // references XXX from root context
 
@@ -29,7 +30,6 @@ Item {
     id: root;
     property string purchaseStatus;
     property bool purchaseStatusChanged;
-    property bool canRezCertifiedItems: false;
     property string itemName;
     property string itemId;
     property string itemPreviewImageUrl;
@@ -39,7 +39,15 @@ Item {
     property int itemEdition;
     property int numberSold;
     property int limitedRun;
-    property bool isWearable;
+    property string itemType;
+    property var itemTypesArray: ["entity", "wearable", "contentSet", "app", "avatar"];
+    property var buttonTextNormal: ["REZ", "WEAR", "REPLACE", "INSTALL", "WEAR"];
+    property var buttonTextClicked: ["REZZED", "WORN", "REPLACED", "INSTALLED", "WORN"]
+    property var buttonGlyph: [hifi.glyphs.wand, hifi.glyphs.hat, hifi.glyphs.globe, hifi.glyphs.install, hifi.glyphs.avatar];
+    property bool showConfirmation: false;
+    property bool hasPermissionToRezThis;
+    property bool permissionExplanationCardVisible;
+    property bool isInstalled;
 
     property string originalStatusText;
     property string originalStatusColor;
@@ -47,6 +55,47 @@ Item {
     height: 110;
     width: parent.width;
 
+    Connections {
+        target: Commerce;
+        
+        onContentSetChanged: {
+            if (contentSetHref === root.itemHref) {
+                showConfirmation = true;
+            }
+        }
+
+        onAppInstalled: {
+            if (appHref === root.itemHref) {
+                root.isInstalled = true;
+            }
+        }
+
+        onAppUninstalled: {
+            if (appHref === root.itemHref) {
+                root.isInstalled = false;
+            }
+        }
+    }
+
+    Connections {
+        target: MyAvatar;
+
+        onSkeletonModelURLChanged: {
+            if (skeletonModelURL === root.itemHref) {
+                showConfirmation = true;
+            }
+        }
+    }
+
+    onItemTypeChanged: {
+        if ((itemType === "entity" && (!Entities.canRezCertified() && !Entities.canRezTmpCertified())) ||
+            (itemType === "contentSet" && !Entities.canReplaceContent())) {
+            root.hasPermissionToRezThis = false;
+        } else {
+            root.hasPermissionToRezThis = true;
+        }
+    }
+
     onPurchaseStatusChangedChanged: {
         if (root.purchaseStatusChanged === true && root.purchaseStatus === "confirmed") {
             root.originalStatusText = statusText.text;
@@ -57,6 +106,15 @@ Item {
         }
     }
 
+    onShowConfirmationChanged: {
+        if (root.showConfirmation) {
+            rezzedNotifContainer.visible = true;
+            rezzedNotifContainerTimer.start();
+            UserActivityLogger.commerceEntityRezzed(root.itemId, "purchases", root.itemType);
+            root.showConfirmation = false;
+        }
+    }
+
     Timer {
         id: confirmedTimer;
         interval: 3000;
@@ -73,10 +131,10 @@ Item {
         color: hifi.colors.white;
         // Size
         anchors.left: parent.left;
-        anchors.leftMargin: 8;
+        anchors.leftMargin: 16;
         anchors.right: parent.right;
-        anchors.rightMargin: 8;
-        anchors.top: parent.top;
+        anchors.rightMargin: 16;
+        anchors.verticalCenter: parent.verticalCenter;
         height: root.height - 10;
 
         Image {
@@ -96,15 +154,20 @@ Item {
             }
         }
 
-    
+        TextMetrics {
+            id:     itemNameTextMetrics;
+            font:   itemName.font;
+            text:   itemName.text;
+        }
         RalewaySemiBold {
             id: itemName;
             anchors.top: itemPreviewImage.top;
             anchors.topMargin: 4;
             anchors.left: itemPreviewImage.right;
             anchors.leftMargin: 8;
-            anchors.right: buttonContainer.left;
-            anchors.rightMargin: 8;
+            width: !noPermissionGlyph.visible ? (buttonContainer.x - itemPreviewImage.x - itemPreviewImage.width - anchors.leftMargin) :
+                Math.min(itemNameTextMetrics.tightBoundingRect.width + 2,
+                buttonContainer.x - itemPreviewImage.x - itemPreviewImage.width - anchors.leftMargin - noPermissionGlyph.width + 2);
             height: paintedHeight;
             // Text size
             size: 24;
@@ -130,6 +193,93 @@ Item {
                 }
             }
         }
+        HiFiGlyphs {
+            id: noPermissionGlyph;
+            visible: !root.hasPermissionToRezThis;
+            anchors.verticalCenter: itemName.verticalCenter;
+            anchors.left: itemName.right;
+            anchors.leftMargin: itemName.truncated ? -10 : -2;
+            text: hifi.glyphs.info;
+            // Size
+            size: 40;
+            width: 32;
+            // Style
+            color: hifi.colors.redAccent;
+            
+            MouseArea {
+                anchors.fill: parent;
+                hoverEnabled: true;
+
+                onEntered: {
+                    noPermissionGlyph.color = hifi.colors.redHighlight;
+                }
+                onExited: {
+                    noPermissionGlyph.color = hifi.colors.redAccent;
+                }
+                onClicked: {
+                    root.sendToPurchases({ method: 'openPermissionExplanationCard' });
+                }
+            }
+        }
+        Rectangle {
+            id: permissionExplanationCard;
+            z: 995;
+            visible: root.permissionExplanationCardVisible;
+            anchors.fill: parent;
+            color: hifi.colors.white;
+
+            RalewayRegular {
+                id: permissionExplanationText;
+                text: {
+                    if (root.itemType === "contentSet") {
+                        "You do not have 'Replace Content' permissions in this domain. <a href='#replaceContentPermission'>Learn more</a>";
+                    } else if (root.itemType === "entity") {
+                        "You do not have 'Rez Certified' permissions in this domain. <a href='#rezCertifiedPermission'>Learn more</a>";
+                    } else {
+                        ""
+                    }
+                }
+                size: 16;
+                anchors.left: parent.left;
+                anchors.leftMargin: 30;
+                anchors.top: parent.top;
+                anchors.bottom: parent.bottom;
+                anchors.right: permissionExplanationGlyph.left;
+                color: hifi.colors.baseGray;
+                wrapMode: Text.WordWrap;
+                verticalAlignment: Text.AlignVCenter;
+
+                onLinkActivated: {
+                    sendToPurchases({method: 'showPermissionsExplanation', itemType: root.itemType});
+                }
+            }
+            // "Close" button
+            HiFiGlyphs {
+                id: permissionExplanationGlyph;
+                text: hifi.glyphs.close;
+                color: hifi.colors.baseGray;
+                size: 26;
+                anchors.top: parent.top;
+                anchors.bottom: parent.bottom;
+                anchors.right: parent.right;
+                width: 77;
+                horizontalAlignment: Text.AlignHCenter;
+                verticalAlignment: Text.AlignVCenter;
+                MouseArea {
+                    anchors.fill: parent;
+                    hoverEnabled: true;
+                    onEntered: {
+                        parent.text = hifi.glyphs.closeInverted;
+                    }
+                    onExited: {
+                        parent.text = hifi.glyphs.close;
+                    }
+                    onClicked: {
+                        root.sendToPurchases({ method: 'openPermissionExplanationCard', closeAll: true });
+                    }
+                }
+            }
+        }
 
         Item {
             id: certificateContainer;
@@ -151,19 +301,19 @@ Item {
                 anchors.bottom: parent.bottom;
                 width: 32;
                 // Style
-                color: hifi.colors.lightGray;
+                color: hifi.colors.black;
             }
 
             RalewayRegular {
                 id: viewCertificateText;
                 text: "VIEW CERTIFICATE";
-                size: 14;
+                size: 13;
                 anchors.left: certificateIcon.right;
                 anchors.leftMargin: 4;
                 anchors.top: parent.top;
                 anchors.bottom: parent.bottom;
                 anchors.right: parent.right;
-                color: hifi.colors.lightGray;
+                color: hifi.colors.black;
             }
 
             MouseArea {
@@ -173,13 +323,13 @@ Item {
                     sendToPurchases({method: 'purchases_itemCertificateClicked', itemCertificateId: root.certificateId});
                 }
                 onEntered: {
-                    certificateIcon.color = hifi.colors.black;
-                    viewCertificateText.color = hifi.colors.black;
-                }
-                onExited: {
                     certificateIcon.color = hifi.colors.lightGray;
                     viewCertificateText.color = hifi.colors.lightGray;
                 }
+                onExited: {
+                    certificateIcon.color = hifi.colors.black;
+                    viewCertificateText.color = hifi.colors.black;
+                }
             }
         }
 
@@ -193,14 +343,14 @@ Item {
             anchors.right: buttonContainer.left;
             anchors.rightMargin: 2;
 
-            FiraSansRegular {
+            RalewayRegular {
                 anchors.left: parent.left;
                 anchors.top: parent.top;
                 anchors.bottom: parent.bottom;
                 width: paintedWidth;
                 text: "#" + root.itemEdition;
-                size: 15;
-                color: "#cc6a6a6a";
+                size: 13;
+                color: hifi.colors.black;
                 verticalAlignment: Text.AlignTop;
             }
         }
@@ -311,7 +461,7 @@ Item {
             id: rezzedNotifContainer;
             z: 998;
             visible: false;
-            color: hifi.colors.blueHighlight;
+            color: "#1FC6A6";
             anchors.fill: buttonContainer;
             MouseArea {
                 anchors.fill: parent;
@@ -321,8 +471,8 @@ Item {
 
             RalewayBold {
                 anchors.fill: parent;
-                text: "REZZED";
-                size: 18;
+                text: (root.buttonTextClicked)[itemTypesArray.indexOf(root.itemType)];
+                size: 15;
                 color: hifi.colors.white;
                 verticalAlignment: Text.AlignVCenter;
                 horizontalAlignment: Text.AlignHCenter;
@@ -335,27 +485,89 @@ Item {
                 }
         }
 
+        Rectangle {
+            id: appButtonContainer;
+            color: hifi.colors.white;
+            z: 994;
+            visible: root.isInstalled;
+            anchors.fill: buttonContainer;
+
+            HifiControlsUit.Button {
+                id: openAppButton;
+                color: hifi.buttons.blue;
+                colorScheme: hifi.colorSchemes.light;
+                anchors.top: parent.top;
+                anchors.right: parent.right;
+                anchors.left: parent.left;
+                width: 92;
+                height: 44;
+                text: "OPEN"
+                onClicked: {
+                    Commerce.openApp(root.itemHref);
+                }
+            }
+
+            HifiControlsUit.Button {
+                id: uninstallAppButton;
+                color: hifi.buttons.noneBorderless;
+                colorScheme: hifi.colorSchemes.light;
+                anchors.bottom: parent.bottom;
+                anchors.right: parent.right;
+                anchors.left: parent.left;
+                height: 44;
+                text: "UNINSTALL"
+                onClicked: {
+                    Commerce.uninstallApp(root.itemHref);
+                }
+            }
+        }
+
         Button {
             id: buttonContainer;
-            property int color: hifi.buttons.red;
+            property int color: hifi.buttons.blue;
             property int colorScheme: hifi.colorSchemes.light;
 
             anchors.top: parent.top;
+            anchors.topMargin: 4;
             anchors.bottom: parent.bottom;
+            anchors.bottomMargin: 4;
             anchors.right: parent.right;
+            anchors.rightMargin: 4;
             width: height;
-            enabled: (root.canRezCertifiedItems || root.isWearable) && root.purchaseStatus !== "invalidated";
+            enabled: root.hasPermissionToRezThis &&
+                root.purchaseStatus !== "invalidated" &&
+                MyAvatar.skeletonModelURL !== root.itemHref;
+
+            onHoveredChanged: {
+                if (hovered) {
+                    Tablet.playSound(TabletEnums.ButtonHover);
+                }
+            }
+    
+            onFocusChanged: {
+                if (focus) {
+                    Tablet.playSound(TabletEnums.ButtonHover);
+                }
+            }
             
             onClicked: {
-                sendToPurchases({method: 'purchases_rezClicked', itemHref: root.itemHref, isWearable: root.isWearable});
-                rezzedNotifContainer.visible = true;
-                rezzedNotifContainerTimer.start();
-                UserActivityLogger.commerceEntityRezzed(root.itemId, "purchases", root.isWearable ? "rez" : "wear");
+                Tablet.playSound(TabletEnums.ButtonClick);
+                if (root.itemType === "contentSet") {
+                    sendToPurchases({method: 'showReplaceContentLightbox', itemHref: root.itemHref});
+                } else if (root.itemType === "avatar") {
+                    sendToPurchases({method: 'showChangeAvatarLightbox', itemName: root.itemName, itemHref: root.itemHref});
+                } else if (root.itemType === "app") {
+                    // "Run" and "Uninstall" buttons are separate.
+                    Commerce.installApp(root.itemHref);
+                } else {
+                    sendToPurchases({method: 'purchases_rezClicked', itemHref: root.itemHref, itemType: root.itemType});
+                    root.showConfirmation = true;
+                }
             }
 
             style: ButtonStyle {
-
                 background: Rectangle {
+                    radius: 4;
                     gradient: Gradient {
                         GradientStop {
                             position: 0.2
@@ -390,13 +602,13 @@ Item {
 
                 label: Item {
                     HiFiGlyphs {
-                        id: lightningIcon;
-                        text: hifi.glyphs.lightning;
+                        id: rezIcon;
+                        text: (root.buttonGlyph)[itemTypesArray.indexOf(root.itemType)];
                         // Size
-                        size: 32;
+                        size: 60;
                         // Anchors
                         anchors.top: parent.top;
-                        anchors.topMargin: 12;
+                        anchors.topMargin: 0;
                         anchors.left: parent.left;
                         anchors.right: parent.right;
                         horizontalAlignment: Text.AlignHCenter;
@@ -405,18 +617,19 @@ Item {
                                        : hifi.buttons.disabledTextColor[control.colorScheme]
                     }
                     RalewayBold {
-                        anchors.top: lightningIcon.bottom;
-                        anchors.topMargin: -20;
+                        id: rezIconLabel;
+                        anchors.top: rezIcon.bottom;
+                        anchors.topMargin: -4;
                         anchors.right: parent.right;
                         anchors.left: parent.left;
                         anchors.bottom: parent.bottom;
                         font.capitalization: Font.AllUppercase
                         color: enabled ? hifi.buttons.textColor[control.color]
                                        : hifi.buttons.disabledTextColor[control.colorScheme]
-                        size: 16;
+                        size: 15;
                         verticalAlignment: Text.AlignVCenter
                         horizontalAlignment: Text.AlignHCenter
-                        text: root.isWearable ? "Wear It" : "Rez It"
+                        text: MyAvatar.skeletonModelURL === root.itemHref ? "CURRENT" : (root.buttonTextNormal)[itemTypesArray.indexOf(root.itemType)];
                     }
                 }
             }
@@ -425,11 +638,11 @@ Item {
 
     DropShadow {
         anchors.fill: mainContainer;
-        horizontalOffset: 3;
-        verticalOffset: 3;
-        radius: 8.0;
-        samples: 17;
-        color: "#80000000";
+        horizontalOffset: 0;
+        verticalOffset: 4;
+        radius: 4.0;
+        samples: 9
+        color: Qt.rgba(0, 0, 0, 0.25);
         source: mainContainer;
     }
 

interface/resources/qml/hifi/commerce/purchases/Purchases.qml

@@ -32,11 +32,11 @@ Rectangle {
     property bool securityImageResultReceived: false;
     property bool purchasesReceived: false;
     property bool punctuationMode: false;
-    property bool canRezCertifiedItems: Entities.canRezCertified() || Entities.canRezTmpCertified();
     property bool pendingInventoryReply: true;
     property bool isShowingMyItems: false;
     property bool isDebuggingFirstUseTutorial: false;
     property int pendingItemCount: 0;
+    property string installedApps;
     // Style
     color: hifi.colors.white;
     Connections {
@@ -47,21 +47,22 @@ Rectangle {
                 if (root.activeView !== "needsLogIn") {
                     root.activeView = "needsLogIn";
                 }
-            } else if (walletStatus === 1) {
+            } else if ((walletStatus === 1) || (walletStatus === 2) || (walletStatus === 3)) {
                 if (root.activeView !== "notSetUp") {
                     root.activeView = "notSetUp";
                     notSetUpTimer.start();
                 }
-            } else if (walletStatus === 2) {
+            } else if (walletStatus === 4) {
                 if (root.activeView !== "passphraseModal") {
                     root.activeView = "passphraseModal";
                     UserActivityLogger.commercePassphraseEntry("marketplace purchases");
                 }
-            } else if (walletStatus === 3) {
+            } else if (walletStatus === 5) {
                 if ((Settings.getValue("isFirstUseOfPurchases", true) || root.isDebuggingFirstUseTutorial) && root.activeView !== "firstUseTutorial") {
                     root.activeView = "firstUseTutorial";
                 } else if (!Settings.getValue("isFirstUseOfPurchases", true) && root.activeView === "initialize") {
                     root.activeView = "purchasesMain";
+                    root.installedApps = Commerce.getInstalledApps();
                     Commerce.inventory();
                 }
             } else {
@@ -148,7 +149,11 @@ Rectangle {
 
         Connections {
             onSendToParent: {
-                sendToScript(msg);
+                if (msg.method === 'commerceLightboxLinkClicked') {
+                    Qt.openUrlExternally(msg.linkUrl);
+                } else {
+                    sendToScript(msg);
+                }
             }
         }
     }
@@ -266,6 +271,7 @@ Rectangle {
                     case 'tutorial_finished':
                         Settings.setValue("isFirstUseOfPurchases", false);
                         root.activeView = "purchasesMain";
+                        root.installedApps = Commerce.getInstalledApps();
                         Commerce.inventory();
                     break;
                 }
@@ -297,7 +303,7 @@ Rectangle {
             anchors.left: parent.left;
             anchors.leftMargin: 8;
             anchors.right: parent.right;
-            anchors.rightMargin: 12;
+            anchors.rightMargin: 16;
             anchors.top: parent.top;
             anchors.topMargin: 4;
 
@@ -308,11 +314,11 @@ Rectangle {
                 anchors.bottom: parent.bottom;
                 anchors.bottomMargin: 10;
                 anchors.left: parent.left;
-                anchors.leftMargin: 4;
+                anchors.leftMargin: 16;
                 width: paintedWidth;
                 text: isShowingMyItems ? "My Items" : "My Purchases";
-                color: hifi.colors.baseGray;
-                size: 28;
+                color: hifi.colors.black;
+                size: 22;
             }
 
             HifiControlsUit.TextField {
@@ -323,8 +329,8 @@ Rectangle {
                 hasRoundedBorder: true;
                 anchors.left: myText.right;
                 anchors.leftMargin: 16;
-                anchors.top: parent.top;
-                anchors.bottom: parent.bottom;
+                height: 39;
+                anchors.verticalCenter: parent.verticalCenter;
                 anchors.right: parent.right;
                 placeholderText: "filter items";
 
@@ -345,7 +351,7 @@ Rectangle {
 
         HifiControlsUit.Separator {
             id: separator;
-            colorScheme: 1;
+            colorScheme: 2;
             anchors.left: parent.left;
             anchors.right: parent.right;
             anchors.top: filterBarContainer.bottom;
@@ -365,69 +371,6 @@ Rectangle {
             id: filteredPurchasesModel;
         }
 
-        Rectangle {
-            id: cantRezCertified;
-            visible: !root.canRezCertifiedItems;
-            color: "#FFC3CD";
-            radius: 4;
-            border.color: hifi.colors.redAccent;
-            border.width: 1;
-            anchors.top: separator.bottom;
-            anchors.topMargin: 12;
-            anchors.left: parent.left;
-            anchors.leftMargin: 16;
-            anchors.right: parent.right;
-            anchors.rightMargin: 16;
-            height: 80;
-
-            HiFiGlyphs {
-                id: lightningIcon;
-                text: hifi.glyphs.lightning;
-                // Size
-                size: 36;
-                // Anchors
-                anchors.top: parent.top;
-                anchors.topMargin: 18;
-                anchors.left: parent.left;
-                anchors.leftMargin: 12;
-                horizontalAlignment: Text.AlignHCenter;
-                // Style
-                color: hifi.colors.lightGray;
-            }
-
-            RalewayRegular {
-                text: "You don't have permission to rez certified items in this domain. " +
-                '<b><font color="' + hifi.colors.blueAccent + '"><a href="#">Learn More</a></font></b>';
-                // Text size
-                size: 18;
-                // Anchors
-                anchors.top: parent.top;
-                anchors.topMargin: 4;
-                anchors.left: lightningIcon.right;
-                anchors.leftMargin: 8;
-                anchors.right: parent.right;
-                anchors.rightMargin: 8;
-                anchors.bottom: parent.bottom;
-                anchors.bottomMargin: 4;
-                // Style
-                color: hifi.colors.baseGray;
-                wrapMode: Text.WordWrap;
-                // Alignment
-                verticalAlignment: Text.AlignVCenter;
-                
-                onLinkActivated: {
-                    lightboxPopup.titleText = "Rez Permission Required";
-                    lightboxPopup.bodyText = "You don't have permission to rez certified items in this domain.<br><br>" +
-                        "Use the <b>GOTO app</b> to visit another domain or <b>go to your own sandbox.</b>";
-                    lightboxPopup.button1text = "CLOSE";
-                    lightboxPopup.button1method = "root.visible = false;"
-                    lightboxPopup.button2text = "OPEN GOTO";
-                    lightboxPopup.button2method = "sendToParent({method: 'purchases_openGoTo'});";
-                    lightboxPopup.visible = true;
-                }
-            }
-        }
-
         ListView {
             id: purchasesContentsList;
             visible: (root.isShowingMyItems && filteredPurchasesModel.count !== 0) || (!root.isShowingMyItems && filteredPurchasesModel.count !== 0);
@@ -436,13 +379,12 @@ Rectangle {
             snapMode: ListView.SnapToItem;
             highlightRangeMode: ListView.StrictlyEnforceRange;
             // Anchors
-            anchors.top: root.canRezCertifiedItems ? separator.bottom : cantRezCertified.bottom;
+            anchors.top: separator.bottom;
             anchors.topMargin: 12;
             anchors.left: parent.left;
             anchors.bottom: parent.bottom;
             width: parent.width;
             delegate: PurchasedItem {
-                canRezCertifiedItems: root.canRezCertifiedItems;
                 itemName: title;
                 itemId: id;
                 itemPreviewImageUrl: preview;
@@ -454,16 +396,32 @@ Rectangle {
                 numberSold: model.number_sold;
                 limitedRun: model.limited_run;
                 displayedItemCount: model.displayedItemCount;
-                isWearable: model.categories.indexOf("Wearables") > -1;
-                anchors.topMargin: 12;
-                anchors.bottomMargin: 12;
+                permissionExplanationCardVisible: model.permissionExplanationCardVisible;
+                isInstalled: model.isInstalled;
+                itemType: {
+                    if (model.root_file_url.indexOf(".fst") > -1) {
+                        "avatar";
+                    } else if (model.categories.indexOf("Wearables") > -1) {
+                        "wearable";
+                    } else if (model.root_file_url.endsWith('.json.gz')) {
+                        "contentSet";
+                    } else if (model.root_file_url.endsWith('.app.json')) {
+                        "app";
+                    } else if (model.root_file_url.endsWith('.json')) {
+                        "entity";
+                    } else {
+                        "unknown";
+                    }
+                }
+                anchors.topMargin: 10;
+                anchors.bottomMargin: 10;
 
                 Connections {
                     onSendToPurchases: {
                         if (msg.method === 'purchases_itemInfoClicked') {
                             sendToScript({method: 'purchases_itemInfoClicked', itemId: itemId});
                         } else if (msg.method === "purchases_rezClicked") {
-                            sendToScript({method: 'purchases_rezClicked', itemHref: itemHref, isWearable: isWearable});
+                            sendToScript({method: 'purchases_rezClicked', itemHref: itemHref, itemType: itemType});
                         } else if (msg.method === 'purchases_itemCertificateClicked') {
                             inspectionCertificate.visible = true;
                             inspectionCertificate.isLightbox = true;
@@ -482,8 +440,51 @@ Rectangle {
                             lightboxPopup.button1text = "CLOSE";
                             lightboxPopup.button1method = "root.visible = false;"
                             lightboxPopup.visible = true;
+                        } else if (msg.method === "showReplaceContentLightbox") {
+                            lightboxPopup.titleText = "Replace Content";
+                            lightboxPopup.bodyText = "Rezzing this content set will replace the existing environment and all of the items in this domain. " +
+                                "If you want to save the state of the content in this domain, create a backup before proceeding.<br><br>" +
+                                "For more information about backing up and restoring content, " +
+                                "<a href='https://docs.highfidelity.com/create-and-explore/start-working-in-your-sandbox/restoring-sandbox-content'>" +
+                                "click here to open info on your desktop browser.";
+                            lightboxPopup.button1text = "CANCEL";
+                            lightboxPopup.button1method = "root.visible = false;"
+                            lightboxPopup.button2text = "CONFIRM";
+                            lightboxPopup.button2method = "Commerce.replaceContentSet('" + msg.itemHref + "'); root.visible = false;";
+                            lightboxPopup.visible = true;
+                        } else if (msg.method === "showChangeAvatarLightbox") {
+                            lightboxPopup.titleText = "Change Avatar";
+                            lightboxPopup.bodyText = "This will change your current avatar to " + msg.itemName + " while retaining your wearables.";
+                            lightboxPopup.button1text = "CANCEL";
+                            lightboxPopup.button1method = "root.visible = false;"
+                            lightboxPopup.button2text = "CONFIRM";
+                            lightboxPopup.button2method = "MyAvatar.useFullAvatarURL('" + msg.itemHref + "'); root.visible = false;";
+                            lightboxPopup.visible = true;
+                        } else if (msg.method === "showPermissionsExplanation") {
+                            if (msg.itemType === "entity") {
+                                lightboxPopup.titleText = "Rez Certified Permission";
+                                lightboxPopup.bodyText = "You don't have permission to rez certified items in this domain.<br><br>" +
+                                    "Use the <b>GOTO app</b> to visit another domain or <b>go to your own sandbox.</b>";
+                                lightboxPopup.button2text = "OPEN GOTO";
+                                lightboxPopup.button2method = "sendToParent({method: 'purchases_openGoTo'});";
+                            } else if (msg.itemType === "contentSet") {
+                                lightboxPopup.titleText = "Replace Content Permission";
+                                lightboxPopup.bodyText = "You do not have the permission 'Replace Content' in this <b>domain's server settings</b>. The domain owner " +
+                                    "must enable it for you before you can replace content sets in this domain.";
+                            }
+                            lightboxPopup.button1text = "CLOSE";
+                            lightboxPopup.button1method = "root.visible = false;"
+                            lightboxPopup.visible = true;
                         } else if (msg.method === "setFilterText") {
                             filterBar.text = msg.filterText;
+                        } else if (msg.method === "openPermissionExplanationCard") {
+                            for (var i = 0; i < filteredPurchasesModel.count; i++) {
+                                if (i !== index || msg.closeAll) {
+                                    filteredPurchasesModel.setProperty(i, "permissionExplanationCardVisible", false);
+                                } else {
+                                    filteredPurchasesModel.setProperty(i, "permissionExplanationCardVisible", true);
+                                }
+                            }
                         }
                     }
                 }
@@ -683,8 +684,13 @@ Rectangle {
 
         if (sameItemCount !== tempPurchasesModel.count || filterBar.text !== filterBar.previousText) {
             filteredPurchasesModel.clear();
+            var currentId;
             for (var i = 0; i < tempPurchasesModel.count; i++) {
+                currentId = tempPurchasesModel.get(i).id;
+
                 filteredPurchasesModel.append(tempPurchasesModel.get(i));
+                filteredPurchasesModel.setProperty(i, 'permissionExplanationCardVisible', false);
+                filteredPurchasesModel.setProperty(i, 'isInstalled', ((root.installedApps).indexOf(currentId) > -1));
             }
 
             populateDisplayedItemCounts();

interface/resources/qml/hifi/commerce/wallet/Help.qml

@@ -60,48 +60,88 @@ Item {
 
         ListElement {
             isExpanded: false;
-            question: "How can I get HFC?"
-            answer: qsTr("High Fidelity commerce is in closed beta right now.<br><br>To request entry and get free HFC, <b>please contact info@highfidelity.com with your High Fidelity account username and the email address registered to that account.</b>");
+            question: "How can I get HFC?";
+            answer: "High Fidelity commerce is in open beta right now. Want more HFC? \
+Get it by going to <br><br><b><font color='#0093C5'><a href='#bank'>BankOfHighFidelity.</a></font></b> and meeting with the banker!";
         }
         ListElement {
             isExpanded: false;
-            question: "What are private keys?"
-            answer: qsTr("A private key is a secret piece of text that is used to prove ownership, unlock confidential information, and sign transactions.<br><br>In High Fidelity, <b>your private keys are used to securely access the contents of your Wallet and Purchases.</b>");
+            question: "What are private keys and where are they stored?";
+            answer: 
+                "A private key is a secret piece of text that is used to prove ownership, unlock confidential information, and sign transactions. \
+In High Fidelity, your private key is used to securely access the contents of your Wallet and Purchases. \
+After wallet setup, a hifikey file is stored on your computer in High Fidelity Interface's AppData directory. \
+Your hifikey file contains your private key and is protected by your wallet passphrase. \
+<br><br>It is very important to back up your hifikey file! \
+<b><font color='#0093C5'><a href='#privateKeyPath'>Tap here to open the folder where your HifiKeys are stored on your main display.</a></font></b>"
         }
         ListElement {
             isExpanded: false;
-            question: "Where are my private keys stored?"
-            answer: qsTr('By default, your private keys are <b>only stored on your hard drive</b> in High Fidelity Interface\'s AppData directory.<br><br><b><font color="#0093C5"><a href="#privateKeyPath">Tap here to open the folder where your HifiKeys are stored on your main display.</a></font></b>');
+            question: "How do I back up my private keys?";
+            answer: "You can back up your hifikey file (which contains your private key and is encrypted using your wallet passphrase) by copying it to a USB flash drive, or to a service like Dropbox or Google Drive. \
+Restore your hifikey file by replacing the file in Interface's AppData directory with your backup copy. \
+Others with access to your back up should not be able to spend your HFC without your passphrase. \
+<b><font color='#0093C5'><a href='#privateKeyPath'>Tap here to open the folder where your HifiKeys are stored on your main display.</a></font></b>";
         }
         ListElement {
             isExpanded: false;
-            question: "How can I backup my private keys?"
-            answer: qsTr('You may backup the file containing your private keys by copying it to a USB flash drive, or to a service like Dropbox or Google Drive.<br><br>Restore your backup by replacing the file in Interface\'s AppData directory with your backed-up copy.<br><br><b><font color="#0093C5"><a href="#privateKeyPath">Tap here to open the folder where your HifiKeys are stored on your main display.</a></font></b>');
+            question: "What happens if I lose my private keys?";
+            answer: "We cannot stress enough that you should keep a backup! For security reasons, High Fidelity does not keep a copy, and cannot restore it for you. \
+If you lose your private key, you will no longer have access to the contents of your Wallet or My Purchases. \
+Here are some things to try:<ul>\
+<li>If you have backed up your hifikey file before, search your backup location</li>\
+<li>Search your AppData directory in the last machine you used to set up the Wallet</li>\
+<li>If you are a developer and have installed multiple builds of High Fidelity, your hifikey file might be in another folder</li>\
+</ul><br><br>As a last resort, you can set up your Wallet again and generate a new hifikey file. \
+Unfortunately, this means you will start with 0 HFC and your purchased items will not be transferred over.";
         }
         ListElement {
             isExpanded: false;
-            question: "What happens if I lose my passphrase?"
-            answer: qsTr("Your passphrase is used to encrypt your private keys. If you lose your passphrase, you will no longer be able to decrypt your private key file. You will also no longer have access to the contents of your Wallet or My Purchases.<br><br><b>Nobody can help you recover your passphrase, including High Fidelity.</b> Please write it down and store it securely.");
+            question: "What if I forget my wallet passphrase?";
+            answer: "Your wallet passphrase is used to encrypt your private keys. Please write it down and store it securely! \
+<br><br>If you forget your passphrase, you will no longer be able to decrypt the hifikey file that the passphrase protects. \
+You will also no longer have access to the contents of your Wallet or My Purchases. \
+For security reasons, High Fidelity does not keep a copy of your passphrase, and can't restore it for you. \
+<br><br>If you still cannot remember your wallet passphrase, you can set up your Wallet again and generate a new hifikey file. \
+Unfortunately, this means you will start with 0 HFC and your purchased items will not be transferred over.";
         }
         ListElement {
             isExpanded: false;
-            question: "What is a 'Security Pic'?"
-            answer: qsTr("Your Security Pic is an encrypted image that you selected during Wallet Setup. <b>It acts as an extra layer of Wallet security.</b><br><br>When you see your Security Pic, you know that your actions and data are securely making use of your private keys.<br><br><b>If you don't see your Security Pic on a page that is asking you for your Wallet passphrase, someone untrustworthy may be trying to gain access to your Wallet.</b><br><br>The encrypted Pic is stored on your hard drive inside the same file as your private keys.");
+            question: "How do I send HFC to other people?";
+            answer: "You can send HFC to a High Fidelity connection (someone you've shaken hands with in-world) or somebody Nearby (currently in the same domain as you). \
+In your Wallet's Send Money tab, choose from your list of connections, or choose Nearby and select the glowing sphere of the person's avatar.";
         }
         ListElement {
             isExpanded: false;
-            question: "My HFC balance isn't what I expect it to be. Why?"
-            answer: qsTr('High Fidelity Coin (HFC) transactions are backed by a <b>blockchain</b>, which takes time to update. The status of a transaction usually updates within a few seconds.<br><br><b><font color="#0093C5"><a href="#blockchain">Tap here to learn more about the blockchain.</a></font></b>');
+            question: "What is a Security Pic?"
+            answer: "Your Security Pic is an encrypted image that you select during Wallet Setup. \
+It acts as an extra layer of Wallet security. \
+When you see your Security Pic, you know that your actions and data are securely making use of your private keys.\
+<br><br>Don't enter your passphrase anywhere that doesn't display your Security Pic! \
+If you don't see your Security Pic on a page that requests your Wallet passphrase, someone untrustworthy may be trying to access your Wallet.";
         }
         ListElement {
             isExpanded: false;
-            question: "Do I get charged money if a transaction fails?"
-            answer: qsTr("<b>No.</b> Your HFC balance only changes after a transaction is confirmed.");
+            question: "Why does my HFC balance not update instantly?";
+            answer: "HFC transations sometimes takes a few seconds to update as they are backed by a blockchain. \
+<br><br><b><font color='#0093C5'><a href='#blockchain'>Tap here to learn more about the blockchain.</a></font></b>";
+        }
+        ListElement {
+            isExpanded: false;
+            question: "Do I get charged money if a transaction fails?";
+            answer: "<b>No.</b> Your HFC balance only changes after a transaction is confirmed.";
         }
         ListElement {
             isExpanded: false;
             question: "How do I convert HFC to other currencies?"
-            answer: qsTr("We are still building the tools needed to support a vibrant economy in High Fidelity. <b>There is currently no way to convert HFC to other currencies.</b>");
+            answer: "We are hard at work building the tools needed to support a vibrant economy in High Fidelity. \
+At the moment, there is currently no way to convert HFC to other currencies. Stay tuned...";
+        }
+        ListElement {
+            isExpanded: false;
+            question: "Who can I reach out to with questions?";
+            answer: "Please email us if you have any issues or questions: \
+<b><font color='#0093C5'><a href='#support'>support@highfidelity.com</a></font></b>";
         }
     }
 
@@ -212,6 +252,10 @@ Item {
                             Qt.openUrlExternally("file:///" + root.keyFilePath.substring(0, root.keyFilePath.lastIndexOf('/')));
                         } else if (link === "#blockchain") {
                             Qt.openUrlExternally("https://docs.highfidelity.com/high-fidelity-commerce");
+                        } else if (link === "#bank") {
+                            Qt.openUrlExternally("hifi://BankOfHighFidelity");
+                        } else if (link === "#support") {
+                            Qt.openUrlExternally("mailto:support@highfidelity.com");
                         }
                     }
                 }

interface/resources/qml/hifi/commerce/wallet/PassphraseModal.qml

@@ -75,8 +75,6 @@ Item {
     // TODO: Fix this unlikely bug
     onVisibleChanged: {
         if (visible) {
-            passphraseField.error = false;
-            passphraseField.focus = true;
             sendSignalToParent({method: 'disableHmdPreview'});
         } else {
             sendSignalToParent({method: 'maybeEnableHmdPreview'});
@@ -206,6 +204,14 @@ Item {
             placeholderText: "passphrase";
             activeFocusOnPress: true;
             activeFocusOnTab: true;
+            
+            onVisibleChanged: {
+                if (visible) {
+                    error = false;
+                    focus = true;
+                    forceActiveFocus();
+                }
+            }
 
             onFocusChanged: {
                 root.keyboardRaised = focus;

interface/resources/qml/hifi/commerce/wallet/Wallet.qml

@@ -47,20 +47,22 @@ Rectangle {
                 }
             } else if (walletStatus === 1) {
                 if (root.activeView !== "walletSetup") {
-                    root.activeView = "walletSetup";
-                    Commerce.resetLocalWalletOnly();
-                    var timestamp = new Date();
-                    walletSetup.startingTimestamp = timestamp;
-                    walletSetup.setupAttemptID = generateUUID();
-                    UserActivityLogger.commerceWalletSetupStarted(timestamp, setupAttemptID, walletSetup.setupFlowVersion, walletSetup.referrer ? walletSetup.referrer : "wallet app",
-                        (AddressManager.placename || AddressManager.hostname || '') + (AddressManager.pathname ? AddressManager.pathname.match(/\/[^\/]+/)[0] : ''));
+                    walletResetSetup();
                 }
             } else if (walletStatus === 2) {
+                if (root.activeView != "preexisting") {
+                    root.activeView = "preexisting";
+                }
+            } else if (walletStatus === 3) {
+                if (root.activeView != "conflicting") {
+                    root.activeView = "conflicting";
+                }
+            } else if (walletStatus === 4) {
                 if (root.activeView !== "passphraseModal") {
                     root.activeView = "passphraseModal";
                     UserActivityLogger.commercePassphraseEntry("wallet app");
                 }
-            } else if (walletStatus === 3) {
+            } else if (walletStatus === 5) {
                 if (root.activeView !== "walletSetup") {
                     root.activeView = "walletHome";
                     Commerce.getSecurityImage();
@@ -169,6 +171,25 @@ Rectangle {
     // TITLE BAR END
     //
 
+    WalletChoice {
+        id: walletChoice;
+        proceedFunction: function (isReset) {
+            console.log(isReset ? "Reset wallet." : "Trying again with new wallet.");
+            Commerce.setSoftReset();
+            if (isReset) {
+                walletResetSetup();
+            } else {
+                var msg = { referrer: walletChoice.referrer }
+                followReferrer(msg);
+            }
+        }
+        copyFunction: Commerce.copyKeyFileFrom;
+        z: 997;
+        visible: (root.activeView === "preexisting") || (root.activeView === "conflicting");
+        activeView: root.activeView;
+        anchors.fill: parent;
+    }
+
     WalletSetup {
         id: walletSetup;
         visible: root.activeView === "walletSetup";
@@ -178,14 +199,7 @@ Rectangle {
         Connections {
             onSendSignalToWallet: {
                 if (msg.method === 'walletSetup_finished') {
-                    if (msg.referrer === '' || msg.referrer === 'marketplace cta') {
-                        root.activeView = "initialize";
-                        Commerce.getWalletStatus();
-                    } else if (msg.referrer === 'purchases') {
-                        sendToScript({method: 'goToPurchases'});
-                    } else {
-                        sendToScript({method: 'goToMarketplaceItemPage', itemId: msg.referrer});
-                    }
+                    followReferrer(msg);
                 } else if (msg.method === 'walletSetup_raiseKeyboard') {
                     root.keyboardRaised = true;
                     root.isPassword = msg.isPasswordField;
@@ -738,6 +752,7 @@ Rectangle {
         switch (message.method) {
             case 'updateWalletReferrer':
                 walletSetup.referrer = message.referrer;
+                walletChoice.referrer = message.referrer;
             break;
             case 'inspectionCertificate_resetCert':
                 // NOP
@@ -768,6 +783,28 @@ Rectangle {
             return (c === 'x' ? r : (r & 0x3 | 0x8)).toString(16);
         });
     }
+
+    function walletResetSetup() {
+        root.activeView = "walletSetup";
+        var timestamp = new Date();
+        walletSetup.startingTimestamp = timestamp;
+        walletSetup.setupAttemptID = generateUUID();
+        UserActivityLogger.commerceWalletSetupStarted(timestamp, walletSetup.setupAttemptID, walletSetup.setupFlowVersion, walletSetup.referrer ? walletSetup.referrer : "wallet app",
+            (AddressManager.placename || AddressManager.hostname || '') + (AddressManager.pathname ? AddressManager.pathname.match(/\/[^\/]+/)[0] : ''));
+    }
+
+    function followReferrer(msg) {
+        if (msg.referrer === '' || msg.referrer === 'marketplace cta') {
+            root.activeView = "initialize";
+            Commerce.getWalletStatus();
+        } else if (msg.referrer === 'purchases') {
+            sendToScript({method: 'goToPurchases'});
+        } else if (msg.referrer === 'marketplace cta' || msg.referrer === 'mainPage') {
+            sendToScript({method: 'goToMarketplaceMainPage', itemId: msg.referrer});
+        } else {
+            sendToScript({method: 'goToMarketplaceItemPage', itemId: msg.referrer});
+        }
+    }
     //
     // FUNCTION DEFINITIONS END
     //

interface/resources/qml/hifi/commerce/wallet/WalletChoice.qml

@@ -0,0 +1,297 @@
+//
+//  WalletChoice.qml
+//  qml/hifi/commerce/wallet
+//
+//  WalletChoice
+//
+//  Created by Howard Stearns
+//  Copyright 2018 High Fidelity, Inc.
+//
+//  Distributed under the Apache License, Version 2.0.
+//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
+//
+
+import Hifi 1.0 as Hifi
+import QtQuick 2.5
+import "../common" as HifiCommerceCommon
+import "../../../styles-uit"
+import "../../../controls-uit" as HifiControlsUit
+
+
+Item {
+    HifiConstants { id: hifi; }
+
+    id: root;
+    property string activeView: "conflict";
+    property var proceedFunction: nil;
+    property var copyFunction: nil;
+    property string referrer: "";
+
+    Image {
+        anchors.fill: parent;
+        source: "images/wallet-bg.jpg";
+    }
+
+    HifiCommerceCommon.CommerceLightbox {
+        id: lightboxPopup;
+        visible: false;
+        anchors.fill: parent;
+    }
+
+    // This object is always used in a popup.
+    // This MouseArea is used to prevent a user from being
+    //     able to click on a button/mouseArea underneath the popup.
+    MouseArea {
+        anchors.fill: parent;
+        propagateComposedEvents: false;
+        hoverEnabled: true;
+    }
+
+    //
+    // TITLE BAR START
+    //
+    Item {
+        id: titleBarContainer;
+        // Size
+        height: 50;
+        // Anchors
+        anchors.left: parent.left;
+        anchors.top: parent.top;
+        anchors.right: parent.right;
+
+        // Wallet icon
+        HiFiGlyphs {
+            id: walletIcon;
+            text: hifi.glyphs.wallet;
+            // Size
+            size: parent.height * 0.8;
+            // Anchors
+            anchors.left: parent.left;
+            anchors.leftMargin: 8;
+            anchors.verticalCenter: parent.verticalCenter;
+            // Style
+            color: hifi.colors.blueHighlight;
+        }
+
+        // Title Bar text
+        RalewayRegular {
+            id: titleBarText;
+            text: "Wallet Setup";
+            // Text size
+            size: hifi.fontSizes.overlayTitle;
+            // Anchors
+            anchors.top: parent.top;
+            anchors.left: walletIcon.right;
+            anchors.leftMargin: 8;
+            anchors.bottom: parent.bottom;
+            width: paintedWidth;
+            // Style
+            color: hifi.colors.white;
+            // Alignment
+            horizontalAlignment: Text.AlignHLeft;
+            verticalAlignment: Text.AlignVCenter;
+        }
+    }
+    //
+    // TITLE BAR END
+    //
+
+    //
+    // MAIN PAGE START
+    //
+    Item {
+        id: preexistingContainer;
+        // Anchors
+        anchors.top: titleBarContainer.bottom;
+        anchors.bottom: parent.bottom;
+        anchors.left: parent.left;
+        anchors.right: parent.right;
+
+        HiFiGlyphs {
+            id: bigKeyIcon;
+            text: hifi.glyphs.walletKey;
+            // Size
+            size: 180;
+            // Anchors
+            anchors.top: parent.top;
+            anchors.topMargin: 40;
+            anchors.horizontalCenter: parent.horizontalCenter;
+            // Style
+            color: hifi.colors.white;
+        }
+
+        RalewayRegular {
+            id: text01;
+            text: root.activeView === "preexisting" ? 
+                "Where are your private keys?" :
+                "Hmm, your keys are different"
+            // Text size
+            size: 26;
+            // Anchors
+            anchors.top: bigKeyIcon.bottom;
+            anchors.left: parent.left;
+            anchors.leftMargin: 16;
+            anchors.right: parent.right;
+            anchors.rightMargin: 16;
+            height: paintedHeight;
+            width: paintedWidth;
+            // Style
+            color: hifi.colors.white;
+            wrapMode: Text.WordWrap;
+            // Alignment
+            horizontalAlignment: Text.AlignHCenter;
+            verticalAlignment: Text.AlignVCenter;
+        }
+
+        RalewayRegular {
+            id: text02;
+            text: root.activeView === "preexisting" ? 
+                "Our records indicate that you created a wallet, but the private keys are not in the folder where we checked." :
+                "Our records indicate that you created a wallet with different keys than the keys you're providing."
+            // Text size
+            size: 18;
+            // Anchors
+            anchors.top: text01.bottom;
+            anchors.topMargin: 40;
+            anchors.left: parent.left;
+            anchors.leftMargin: 65;
+            anchors.right: parent.right;
+            anchors.rightMargin: 65;
+            height: paintedHeight;
+            width: paintedWidth;
+            // Style
+            color: hifi.colors.white;
+            wrapMode: Text.WordWrap;
+            // Alignment
+            horizontalAlignment: Text.AlignHCenter;
+            verticalAlignment: Text.AlignVCenter;
+        }
+
+        // "Locate" button
+        HifiControlsUit.Button {
+            id: locateButton;
+            color: hifi.buttons.blue;
+            colorScheme: hifi.colorSchemes.dark;
+            anchors.top: text02.bottom;
+            anchors.topMargin: 40;
+            anchors.horizontalCenter: parent.horizontalCenter;
+            width: parent.width/2;
+            height: 50;
+            text: root.activeView === "preexisting" ? 
+                "LOCATE MY KEYS" :
+                "LOCATE OTHER KEYS"
+            onClicked: {
+                walletChooser();
+            }
+        }
+
+        // "Create New" OR "Continue" button
+        HifiControlsUit.Button {
+            id: button02;
+            color: hifi.buttons.none;
+            colorScheme: hifi.colorSchemes.dark;
+            anchors.top: locateButton.bottom;
+            anchors.topMargin: 20;
+            anchors.horizontalCenter: parent.horizontalCenter;
+            width: parent.width/2;
+            height: 50;
+            text: root.activeView === "preexisting" ? 
+                "CREATE NEW WALLET" :
+                "CONTINUE WITH THESE KEYS"
+            onClicked: {
+                    lightboxPopup.titleText = "Are you sure?";
+                    lightboxPopup.bodyText = "Taking this step will abandon your old wallet and you will no " +
+                        "longer be able to access your money and your past purchases.<br><br>" +
+                        "This step should only be used if you cannot find your keys.<br><br>" +
+                        "This step cannot be undone.";
+                    lightboxPopup.button1color = hifi.buttons.red;
+                    lightboxPopup.button1text = "YES, CREATE NEW WALLET";
+                    lightboxPopup.button1method = "root.visible = false;proceed(true);";
+                    lightboxPopup.button2text = "CANCEL";
+                    lightboxPopup.button2method = "root.visible = false;"
+                    lightboxPopup.buttonLayout = "topbottom";
+                    lightboxPopup.visible = true;
+            }
+        }
+
+        // "What's This?" link
+        RalewayRegular {
+            id: whatsThisLink;
+            text: '<font color="#FFFFFF"><a href="#whatsthis">What\'s this?</a></font>';
+            // Anchors
+            anchors.bottom: parent.bottom;
+            anchors.bottomMargin: 48;
+            anchors.horizontalCenter: parent.horizontalCenter;
+            width: paintedWidth;
+            height: paintedHeight;
+            // Text size
+            size: 18;
+            // Style
+            color: hifi.colors.white;
+
+            MouseArea {
+                anchors.fill: parent;
+
+                onClicked: {
+                    if (root.activeView === "preexisting") {
+                        lightboxPopup.titleText = "Your wallet's private keys are not in the folder we expected";
+                        lightboxPopup.bodyText = "We see that you have created a wallet but the private keys " +
+                            "for it seem to have been moved to a different folder.<br><br>" +
+                            "To tell us where the keys are, click 'Locate My Keys'. <br><br>" +
+                            "If you'd prefer to create a new wallet (not recommended - you will lose your money and past " +
+                            "purchases), click 'Create New Wallet'.";
+                        lightboxPopup.button1text = "CLOSE";
+                        lightboxPopup.button1method = "root.visible = false;"
+                        lightboxPopup.visible = true;
+                    } else {
+                        lightboxPopup.titleText = "You may have set up more than one wallet";
+                        lightboxPopup.bodyText = "We see that the private keys stored on your computer are different " +
+                            "from the ones you used last time. This may mean that you set up more than one wallet. " +
+                            "If you would like to use these keys, click 'Continue With These Keys'.<br><br>" +
+                            "If you would prefer to use another wallet, click 'Locate Other Keys' to show us where " +
+                            "you've stored the private keys for that wallet.";
+                        lightboxPopup.button1text = "CLOSE";
+                        lightboxPopup.button1method = "root.visible = false;"
+                        lightboxPopup.visible = true;
+                    }
+                }
+            }
+        }
+    }
+    //
+    // MAIN PAGE END
+    //
+    
+    //
+    // FUNCTION DEFINITIONS START
+    //
+    function onFileOpenChanged(filename) {
+        // disconnect the event, otherwise the requests will stack up
+        try { // Not all calls to onFileOpenChanged() connect an event.
+            Window.browseChanged.disconnect(onFileOpenChanged);
+        } catch (e) {
+            console.log('WalletChoice.qml ignoring', e);
+        }
+        if (filename) {
+            if (copyFunction && copyFunction(filename)) {
+                proceed(false);
+             } else {
+                console.log("WalletChoice.qml copyFunction", copyFunction, "failed.");
+             }
+        } // Else we're still at WalletChoice
+    }
+    function walletChooser() {
+        Window.browseChanged.connect(onFileOpenChanged); 
+        Window.browseAsync("Locate your .hifikey file", "", "*.hifikey");
+    }
+    function proceed(isReset) {
+        if (!proceedFunction) {
+            console.log("Provide a function of no arguments to WalletChoice.qml.");
+        } else {
+            proceedFunction(isReset);
+        }
+    }
+    //
+    // FUNCTION DEFINITIONS END
+    //
+}

interface/resources/qml/hifi/commerce/wallet/WalletHome.qml

@@ -310,7 +310,7 @@ Item {
                 height: parent.height;
 
                 HifiControlsUit.Separator {
-                colorScheme: 1;
+                    colorScheme: 1;
                     anchors.left: parent.left;
                     anchors.right: parent.right;
                     anchors.top: parent.top;
@@ -318,20 +318,42 @@ Item {
 
                 RalewayRegular {
                     id: noActivityText;
-                text: "<b>The Wallet app is in closed Beta.</b><br><br>To request entry and <b>receive free HFC</b>, please contact " +
-                "<b>info@highfidelity.com</b> with your High Fidelity account username and the email address registered to that account.";
-                // Text size
-                size: 24;
-                // Style
-                color: hifi.colors.blueAccent;
-                anchors.left: parent.left;
-                anchors.leftMargin: 12;
-                anchors.right: parent.right;
-                anchors.rightMargin: 12;
-                anchors.verticalCenter: parent.verticalCenter;
-                height: paintedHeight;
-                wrapMode: Text.WordWrap;
-                horizontalAlignment: Text.AlignHCenter;
+                    text: "Congrats! Your wallet is all set!<br><br>" +
+                        "<b>Where's my HFC?</b><br>" +
+                        "High Fidelity commerce is in open beta right now. Want more HFC? Get it by meeting with a banker at " +
+                        "<a href='#goToBank'>BankOfHighFidelity</a>!"
+                    // Text size
+                    size: 22;
+                    // Style
+                    color: hifi.colors.blueAccent;
+                    anchors.top: parent.top;
+                    anchors.topMargin: 36;
+                    anchors.left: parent.left;
+                    anchors.leftMargin: 12;
+                    anchors.right: parent.right;
+                    anchors.rightMargin: 12;
+                    height: paintedHeight;
+                    wrapMode: Text.WordWrap;
+                    horizontalAlignment: Text.AlignHCenter;
+
+                    onLinkActivated: {
+                        sendSignalToWallet({ method: "transactionHistory_goToBank" });
+                    }
+                }
+
+                HifiControlsUit.Button {
+                    id: bankButton;
+                    color: hifi.buttons.blue;
+                    colorScheme: hifi.colorSchemes.dark;
+                    anchors.top: noActivityText.bottom;
+                    anchors.topMargin: 30;
+                    anchors.horizontalCenter: parent.horizontalCenter;
+                    width: parent.width/2;
+                    height: 50;
+                    text: "VISIT BANK OF HIGH FIDELITY";
+                    onClicked: {
+                        sendSignalToWallet({ method: "transactionHistory_goToBank" });
+                    }
                 }
             }
             

interface/resources/qml/hifi/commerce/wallet/WalletSetup.qml

@@ -441,7 +441,7 @@ Item {
     }
     Item {
         id: choosePassphraseContainer;
-        visible: root.activeView === "step_3";
+        visible: root.hasShownSecurityImageTip && root.activeView === "step_3";
         // Anchors
         anchors.top: titleBarContainer.bottom;
         anchors.topMargin: 30;
@@ -451,7 +451,10 @@ Item {
 
         onVisibleChanged: {
             if (visible) {
+                sendSignalToWallet({method: 'disableHmdPreview'});
                 Commerce.getWalletAuthenticatedStatus();
+            } else {
+                sendSignalToWallet({method: 'maybeEnableHmdPreview'});
             }
         }
 

interface/resources/qml/hifi/commerce/wallet/sendMoney/SendMoney.qml

@@ -923,7 +923,7 @@ Item {
             anchors.leftMargin: 20;
             anchors.right: parent.right;
             anchors.rightMargin: 20;
-            height: 140;
+            height: 95;
                 
             FontLoader { id: firaSansSemiBold; source: "../../../../../fonts/FiraSans-SemiBold.ttf"; }
             TextArea {
@@ -947,8 +947,14 @@ Item {
                 wrapMode: TextEdit.Wrap;
                 activeFocusOnPress: true;
                 activeFocusOnTab: true;
-                // Workaround for no max length on TextAreas
                 onTextChanged: {
+                    // Don't allow tabs or newlines
+                    if ((/[\n\r\t]/g).test(text)) {
+                        var cursor = cursorPosition;
+                        text = text.replace(/[\n\r\t]/g, '');
+                        cursorPosition = cursor-1;
+                    }
+                    // Workaround for no max length on TextAreas
                     if (text.length > maximumLength) {
                         var cursor = cursorPosition;
                         text = previousText;

interface/resources/qml/hifi/dialogs/TabletAssetServer.qml

@@ -199,7 +199,7 @@ Rectangle {
             var SHAPE_TYPE_BOX = 4;
             var SHAPE_TYPE_SPHERE = 5;
         
-            var SHAPE_TYPES = [];ww
+            var SHAPE_TYPES = [];
             SHAPE_TYPES[SHAPE_TYPE_NONE] = "No Collision";
             SHAPE_TYPES[SHAPE_TYPE_SIMPLE_HULL] = "Basic - Whole model";
             SHAPE_TYPES[SHAPE_TYPE_SIMPLE_COMPOUND] = "Good - Sub-meshes";
@@ -207,7 +207,7 @@ Rectangle {
             SHAPE_TYPES[SHAPE_TYPE_BOX] = "Box";
             SHAPE_TYPES[SHAPE_TYPE_SPHERE] = "Sphere";
         
-            var SHAPE_TYPE_DEFAULT = SHAPE_TYPE_STATIC_MESH;
+            var SHAPE_TYPE_DEFAULT = SHAPE_TYPE_SIMPLE_COMPOUND;
             var DYNAMIC_DEFAULT = false;
             var prompt = tabletRoot.customInputDialog({
                 textInput: {

interface/resources/qml/hifi/tablet/EditTabView.qml

@@ -1,5 +1,6 @@
 import QtQuick 2.5
 import QtQuick.Controls 1.4
+import QtQuick.Controls 2.2 // Need both for short-term fix
 import QtWebEngine 1.1
 import QtWebChannel 1.0
 import QtQuick.Controls.Styles 1.4
@@ -10,6 +11,7 @@ import "../../controls-uit" as HifiControls
 import "../../styles-uit"
 
 
+
 TabView {
     id: editTabView
     // anchors.fill: parent
@@ -23,8 +25,27 @@ TabView {
 
         Rectangle {
             color: "#404040"
+            id: container
+
+            Flickable {
+            height: parent.height
+            width: parent.width
+
+            contentHeight: createEntitiesFlow.height +  importButton.height + assetServerButton.height +
+                           header.anchors.topMargin + createEntitiesFlow.anchors.topMargin +
+                           assetServerButton.anchors.topMargin + importButton.anchors.topMargin
+            contentWidth: width
+
+            ScrollBar.vertical : ScrollBar {
+                visible: parent.contentHeight > parent.height
+                width: 20
+                background: Rectangle {
+                    color: hifi.colors.tableScrollBackgroundDark
+                }
+            }
 
             Text {
+                id: header
                 color: "#ffffff"
                 text: "Choose an Entity Type to Create:"
                 font.pixelSize: 14
@@ -176,6 +197,7 @@ TabView {
             }
 
             HifiControls.Button {
+                id: importButton
                 text: "Import Entities (.json)"
                 color: hifi.buttons.black
                 colorScheme: hifi.colorSchemes.dark
@@ -192,6 +214,7 @@ TabView {
                 }
             }
         }
+      } // Flickable
     }
 
     Tab {

interface/resources/qml/hifi/tablet/NewMaterialDialog.qml

@@ -131,7 +131,7 @@ Rectangle {
                     spacing: 5
 
                     anchors.horizontalCenter: column3.horizontalCenter
-                    anchors.horizontalCenterOffset: -20
+                    anchors.horizontalCenterOffset: 0
 
                     Button {
                         id: button1

interface/resources/qml/hifi/tablet/TabletHome.qml

@@ -130,6 +130,7 @@ Item {
                         flickableDirection: Flickable.AutoFlickIfNeeded
                         keyNavigationEnabled: false
                         highlightFollowsCurrentItem: false
+                        interactive: false
 
                         property int previousGridIndex: -1
 

interface/resources/qml/styles-uit/HifiConstants.qml

@@ -218,7 +218,7 @@ Item {
         readonly property var colorStart: [ colors.white, colors.primaryHighlight, "#d42043", "#343434", Qt.rgba(0, 0, 0, 0), Qt.rgba(0, 0, 0, 0), Qt.rgba(0, 0, 0, 0), Qt.rgba(0, 0, 0, 0) ]
         readonly property var colorFinish: [ colors.lightGrayText, colors.blueAccent, "#94132e", colors.black, Qt.rgba(0, 0, 0, 0), Qt.rgba(0, 0, 0, 0), Qt.rgba(0, 0, 0, 0), Qt.rgba(0, 0, 0, 0) ]
         readonly property var hoveredColor: [ colorStart[white], colorStart[blue], colorStart[red], colorFinish[black], colorStart[none], colorStart[noneBorderless], colorStart[noneBorderlessWhite], colorStart[noneBorderlessGray] ]
-        readonly property var pressedColor: [ colorFinish[white], colorFinish[blue], colorFinish[red], colorStart[black], colorStart[none], colorStart[noneBorderless], colorStart[noneBorderlessWhite], colorStart[noneBorderlessGray] ]
+        readonly property var pressedColor: [ colorFinish[white], colorFinish[blue], colorFinish[red], colorStart[black], colorStart[none], colorStart[noneBorderless], colorStart[noneBorderlessWhite], colors.lightGrayText ]
         readonly property var focusedColor: [ colors.lightGray50, colors.blueAccent, colors.redAccent, colors.darkGray, colorStart[none], colorStart[noneBorderless], colorStart[noneBorderlessWhite], colorStart[noneBorderlessGray] ]
         readonly property var disabledColorStart: [ colorStart[white], colors.baseGrayHighlight]
         readonly property var disabledColorFinish: [ colorFinish[white], colors.baseGrayShadow]
@@ -354,5 +354,9 @@ Item {
         readonly property string wallet: "\ue027"
         readonly property string paperPlane: "\ue028"
         readonly property string passphrase: "\ue029"
+        readonly property string globe: "\ue02c"
+        readonly property string wand: "\ue02d"
+        readonly property string hat: "\ue02e"
+        readonly property string install: "\ue02f"
     }
 }

interface/src/Application.cpp

@@ -68,7 +68,6 @@
 #include <Midi.h>
 #include <AudioInjectorManager.h>
 #include <AvatarBookmarks.h>
-#include <AvatarEntitiesBookmarks.h>
 #include <CursorManager.h>
 #include <DebugDraw.h>
 #include <DeferredLightingEffect.h>
@@ -166,6 +165,7 @@
 #include "scripting/AccountServicesScriptingInterface.h"
 #include "scripting/HMDScriptingInterface.h"
 #include "scripting/MenuScriptingInterface.h"
+#include "graphics-scripting/GraphicsScriptingInterface.h"
 #include "scripting/SettingsScriptingInterface.h"
 #include "scripting/WindowScriptingInterface.h"
 #include "scripting/ControllerScriptingInterface.h"
@@ -392,7 +392,7 @@ const QHash<QString, Application::AcceptURLMethod> Application::_acceptedExtensi
 class DeadlockWatchdogThread : public QThread {
 public:
     static const unsigned long HEARTBEAT_UPDATE_INTERVAL_SECS = 1;
-    static const unsigned long MAX_HEARTBEAT_AGE_USECS = 30 * USECS_PER_SECOND;
+    static const unsigned long MAX_HEARTBEAT_AGE_USECS = 120 * USECS_PER_SECOND; // 2 mins with no checkin probably a deadlock
     static const int WARNING_ELAPSED_HEARTBEAT = 500 * USECS_PER_MSEC; // warn if elapsed heartbeat average is large
     static const int HEARTBEAT_SAMPLES = 100000; // ~5 seconds worth of samples
 
@@ -575,10 +575,7 @@ void messageHandler(QtMsgType type, const QMessageLogContext& context, const QSt
     QString logMessage = LogHandler::getInstance().printMessage((LogMsgType) type, context, message);
 
     if (!logMessage.isEmpty()) {
-#ifdef Q_OS_WIN
-        OutputDebugStringA(logMessage.toLocal8Bit().constData());
-        OutputDebugStringA("\n");
-#elif defined Q_OS_ANDROID
+#ifdef Q_OS_ANDROID
         const char * local=logMessage.toStdString().c_str();
         switch (type) {
             case QtDebugMsg:
@@ -599,10 +596,75 @@ void messageHandler(QtMsgType type, const QMessageLogContext& context, const QSt
                 abort();
         }
 #endif
-        qApp->getLogger()->addMessage(qPrintable(logMessage + "\n"));
+        qApp->getLogger()->addMessage(qPrintable(logMessage));
     }
 }
 
+
+class ApplicationMeshProvider : public scriptable::ModelProviderFactory  {
+public:
+    virtual scriptable::ModelProviderPointer lookupModelProvider(const QUuid& uuid) override {
+        bool success;
+        if (auto nestable = DependencyManager::get<SpatialParentFinder>()->find(uuid, success).lock()) {
+            auto type = nestable->getNestableType();
+#ifdef SCRIPTABLE_MESH_DEBUG
+            qCDebug(interfaceapp) << "ApplicationMeshProvider::lookupModelProvider" << uuid << SpatiallyNestable::nestableTypeToString(type);
+#endif
+            switch (type) {
+            case NestableType::Entity:
+                return getEntityModelProvider(static_cast<EntityItemID>(uuid));
+            case NestableType::Overlay:
+                return getOverlayModelProvider(static_cast<OverlayID>(uuid));
+            case NestableType::Avatar:
+                return getAvatarModelProvider(uuid);
+            }
+        }
+        return nullptr;
+    }
+
+private:
+    scriptable::ModelProviderPointer getEntityModelProvider(EntityItemID entityID) {
+        scriptable::ModelProviderPointer provider;
+        auto entityTreeRenderer = qApp->getEntities();
+        auto entityTree = entityTreeRenderer->getTree();
+        if (auto entity = entityTree->findEntityByID(entityID)) {
+            if (auto renderer = entityTreeRenderer->renderableForEntityId(entityID)) {
+                provider = std::dynamic_pointer_cast<scriptable::ModelProvider>(renderer);
+                provider->modelProviderType = NestableType::Entity;
+            } else {
+                qCWarning(interfaceapp) << "no renderer for entity ID" << entityID.toString();
+            }
+        }
+        return provider;
+    }
+
+    scriptable::ModelProviderPointer getOverlayModelProvider(OverlayID overlayID) {
+        scriptable::ModelProviderPointer provider;
+        auto &overlays = qApp->getOverlays();
+        if (auto overlay = overlays.getOverlay(overlayID)) {
+            if (auto base3d = std::dynamic_pointer_cast<Base3DOverlay>(overlay)) {
+                provider = std::dynamic_pointer_cast<scriptable::ModelProvider>(base3d);
+                provider->modelProviderType = NestableType::Overlay;
+            } else {
+                qCWarning(interfaceapp) << "no renderer for overlay ID" << overlayID.toString();
+            }
+        } else {
+            qCWarning(interfaceapp) << "overlay not found" << overlayID.toString();
+        }
+        return provider;
+    }
+
+    scriptable::ModelProviderPointer getAvatarModelProvider(QUuid sessionUUID) {
+        scriptable::ModelProviderPointer provider;
+        auto avatarManager = DependencyManager::get<AvatarManager>();
+        if (auto avatar = avatarManager->getAvatarBySessionID(sessionUUID)) {
+            provider = std::dynamic_pointer_cast<scriptable::ModelProvider>(avatar);
+            provider->modelProviderType = NestableType::Avatar;
+        }
+        return provider;
+    }
+};
+
 static const QString STATE_IN_HMD = "InHMD";
 static const QString STATE_CAMERA_FULL_SCREEN_MIRROR = "CameraFSM";
 static const QString STATE_CAMERA_FIRST_PERSON = "CameraFirstPerson";
@@ -719,6 +781,7 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
     DependencyManager::set<AccountManager>(std::bind(&Application::getUserAgent, qApp));
     DependencyManager::set<StatTracker>();
     DependencyManager::set<ScriptEngines>(ScriptEngine::CLIENT_SCRIPT);
+    DependencyManager::set<ScriptInitializerMixin, NativeScriptInitializers>();
     DependencyManager::set<Preferences>();
     DependencyManager::set<recording::Deck>();
     DependencyManager::set<recording::Recorder>();
@@ -747,6 +810,9 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
     DependencyManager::set<ResourceCacheSharedItems>();
     DependencyManager::set<DesktopScriptingInterface>();
     DependencyManager::set<EntityScriptingInterface>(true);
+    DependencyManager::set<GraphicsScriptingInterface>();
+    DependencyManager::registerInheritance<scriptable::ModelProviderFactory, ApplicationMeshProvider>();
+    DependencyManager::set<ApplicationMeshProvider>();
     DependencyManager::set<RecordingScriptingInterface>();
     DependencyManager::set<WindowScriptingInterface>();
     DependencyManager::set<HMDScriptingInterface>();
@@ -785,7 +851,6 @@ bool setupEssentials(int& argc, char** argv, bool runningMarkerExisted) {
     DependencyManager::set<GooglePolyScriptingInterface>();
     DependencyManager::set<OctreeStatsProvider>(nullptr, qApp->getOcteeSceneStats());
     DependencyManager::set<AvatarBookmarks>();
-    DependencyManager::set<AvatarEntitiesBookmarks>();
     DependencyManager::set<LocationBookmarks>();
     DependencyManager::set<Snapshot>();
     DependencyManager::set<CloseEventSender>();
@@ -1515,6 +1580,7 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         settingsTimer->setSingleShot(false);
         settingsTimer->setInterval(SAVE_SETTINGS_INTERVAL); // 10s, Qt::CoarseTimer acceptable
         QObject::connect(settingsTimer, &QTimer::timeout, this, &Application::saveSettings);
+        settingsTimer->start();
     }, QThread::LowestPriority);
 
     if (Menu::getInstance()->isOptionChecked(MenuOption::FirstPerson)) {
@@ -1719,6 +1785,10 @@ Application::Application(int& argc, char** argv, QElapsedTimer& startupTimer, bo
         properties["has_async_reprojection"] = displayPlugin->hasAsyncReprojection();
         properties["hardware_stats"] = displayPlugin->getHardwareStats();
 
+        // deadlock watchdog related stats
+        properties["deadlock_watchdog_maxElapsed"] = (int)DeadlockWatchdogThread::_maxElapsed;
+        properties["deadlock_watchdog_maxElapsedAverage"] = (int)DeadlockWatchdogThread::_maxElapsedAverage;
+
         auto bandwidthRecorder = DependencyManager::get<BandwidthRecorder>();
         properties["packet_rate_in"] = bandwidthRecorder->getCachedTotalAverageInputPacketsPerSecond();
         properties["packet_rate_out"] = bandwidthRecorder->getCachedTotalAverageOutputPacketsPerSecond();
@@ -2610,7 +2680,6 @@ void Application::onDesktopRootContextCreated(QQmlContext* surfaceContext) {
     surfaceContext->setContextProperty("Settings", SettingsScriptingInterface::getInstance());
     surfaceContext->setContextProperty("ScriptDiscoveryService", DependencyManager::get<ScriptEngines>().data());
     surfaceContext->setContextProperty("AvatarBookmarks", DependencyManager::get<AvatarBookmarks>().data());
-    surfaceContext->setContextProperty("AvatarEntitiesBookmarks", DependencyManager::get<AvatarEntitiesBookmarks>().data());
     surfaceContext->setContextProperty("LocationBookmarks", DependencyManager::get<LocationBookmarks>().data());
 
     // Caches
@@ -4844,7 +4913,6 @@ void Application::setKeyboardFocusHighlight(const glm::vec3& position, const glm
     if (_keyboardFocusHighlightID == UNKNOWN_OVERLAY_ID || !getOverlays().isAddedOverlay(_keyboardFocusHighlightID)) {
         _keyboardFocusHighlight = std::make_shared<Cube3DOverlay>();
         _keyboardFocusHighlight->setAlpha(1.0f);
-        _keyboardFocusHighlight->setBorderSize(1.0f);
         _keyboardFocusHighlight->setColor({ 0xFF, 0xEF, 0x00 });
         _keyboardFocusHighlight->setIsSolid(false);
         _keyboardFocusHighlight->setPulseMin(0.5);
@@ -5986,7 +6054,6 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEnginePointe
     DependencyManager::get<TabletScriptingInterface>().data()->setToolbarScriptingInterface(toolbarScriptingInterface);
 
     scriptEngine->registerGlobalObject("Window", DependencyManager::get<WindowScriptingInterface>().data());
-    qScriptRegisterMetaType(scriptEngine.data(), CustomPromptResultToScriptValue, CustomPromptResultFromScriptValue);
     scriptEngine->registerGetterSetter("location", LocationScriptingInterface::locationGetter,
                         LocationScriptingInterface::locationSetter, "Window");
     // register `location` on the global object.
@@ -6006,7 +6073,6 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEnginePointe
     scriptEngine->registerGlobalObject("AudioStats", DependencyManager::get<AudioClient>()->getStats().data());
     scriptEngine->registerGlobalObject("AudioScope", DependencyManager::get<AudioScope>().data());
     scriptEngine->registerGlobalObject("AvatarBookmarks", DependencyManager::get<AvatarBookmarks>().data());
-    scriptEngine->registerGlobalObject("AvatarEntitiesBookmarks", DependencyManager::get<AvatarEntitiesBookmarks>().data());
     scriptEngine->registerGlobalObject("LocationBookmarks", DependencyManager::get<LocationBookmarks>().data());
 
     scriptEngine->registerGlobalObject("RayPick", DependencyManager::get<RayPickScriptingInterface>().data());
@@ -6044,6 +6110,9 @@ void Application::registerScriptEngineWithApplicationServices(ScriptEnginePointe
     scriptEngine->registerGlobalObject("Scene", DependencyManager::get<SceneScriptingInterface>().data());
     scriptEngine->registerGlobalObject("Render", _renderEngine->getConfiguration().get());
 
+    GraphicsScriptingInterface::registerMetaTypes(scriptEngine.data());
+    scriptEngine->registerGlobalObject("Graphics", DependencyManager::get<GraphicsScriptingInterface>().data());
+
     scriptEngine->registerGlobalObject("ScriptDiscoveryService", DependencyManager::get<ScriptEngines>().data());
     scriptEngine->registerGlobalObject("Reticle", getApplicationCompositor().getReticleInterface());
 
@@ -6303,6 +6372,25 @@ bool Application::askToWearAvatarAttachmentUrl(const QString& url) {
     return true;
 }
 
+void Application::replaceDomainContent(const QString& url) {
+    qCDebug(interfaceapp) << "Attempting to replace domain content: " << url;
+    QByteArray urlData(url.toUtf8());
+    auto limitedNodeList = DependencyManager::get<NodeList>();
+    const auto& domainHandler = limitedNodeList->getDomainHandler();
+    limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
+        return node->getType() == NodeType::EntityServer && node->getActiveSocket();
+    }, [&urlData, limitedNodeList, &domainHandler](const SharedNodePointer& octreeNode) {
+        auto octreeFilePacket = NLPacket::create(PacketType::OctreeFileReplacementFromUrl, urlData.size(), true);
+        octreeFilePacket->write(urlData);
+        limitedNodeList->sendPacket(std::move(octreeFilePacket), domainHandler.getSockAddr());
+    });
+    auto addressManager = DependencyManager::get<AddressManager>();
+    addressManager->handleLookupString(DOMAIN_SPAWNING_POINT);
+    QString newHomeAddress = addressManager->getHost() + DOMAIN_SPAWNING_POINT;
+    qCDebug(interfaceapp) << "Setting new home bookmark to: " << newHomeAddress;
+    DependencyManager::get<LocationBookmarks>()->setHomeLocationToAddress(newHomeAddress);
+}
+
 bool Application::askToReplaceDomainContent(const QString& url) {
     QString methodDetails;
     const int MAX_CHARACTERS_PER_LINE = 90;
@@ -6322,21 +6410,7 @@ bool Application::askToReplaceDomainContent(const QString& url) {
                 QString details;
                 if (static_cast<QMessageBox::StandardButton>(answer.toInt()) == QMessageBox::Yes) {
                     // Given confirmation, send request to domain server to replace content
-                    qCDebug(interfaceapp) << "Attempting to replace domain content: " << url;
-                    QByteArray urlData(url.toUtf8());
-                    auto limitedNodeList = DependencyManager::get<LimitedNodeList>();
-                    limitedNodeList->eachMatchingNode([](const SharedNodePointer& node) {
-                            return node->getType() == NodeType::EntityServer && node->getActiveSocket();
-                        }, [&urlData, limitedNodeList](const SharedNodePointer& octreeNode) {
-                            auto octreeFilePacket = NLPacket::create(PacketType::OctreeFileReplacementFromUrl, urlData.size(), true);
-                            octreeFilePacket->write(urlData);
-                            limitedNodeList->sendPacket(std::move(octreeFilePacket), *octreeNode);
-                        });
-                    auto addressManager = DependencyManager::get<AddressManager>();
-                    addressManager->handleLookupString(DOMAIN_SPAWNING_POINT);
-                    QString newHomeAddress = addressManager->getHost() + DOMAIN_SPAWNING_POINT;
-                    qCDebug(interfaceapp) << "Setting new home bookmark to: " << newHomeAddress;
-                    DependencyManager::get<LocationBookmarks>()->setHomeLocationToAddress(newHomeAddress);
+                    replaceDomainContent(url);
                     details = "SuccessfulRequestToReplaceContent";
                 } else {
                     details = "UserDeclinedToReplaceContent";
@@ -7557,6 +7631,18 @@ void Application::deadlockApplication() {
     }
 }
 
+// cause main thread to be unresponsive for 35 seconds
+void Application::unresponsiveApplication() {
+    // to avoid compiler warnings about a loop that will never exit
+    uint64_t start = usecTimestampNow();
+    uint64_t UNRESPONSIVE_FOR_SECONDS = 35;
+    uint64_t UNRESPONSIVE_FOR_USECS = UNRESPONSIVE_FOR_SECONDS * USECS_PER_SECOND;
+    qCDebug(interfaceapp) << "Intentionally cause Interface to be unresponsive for " << UNRESPONSIVE_FOR_SECONDS << " seconds";
+    while (usecTimestampNow() - start < UNRESPONSIVE_FOR_USECS) {
+        QThread::sleep(1);
+    }
+}
+
 void Application::setActiveDisplayPlugin(const QString& pluginName) {
     auto menu = Menu::getInstance();
     foreach(DisplayPluginPointer displayPlugin, PluginManager::getInstance()->getDisplayPlugins()) {

interface/src/Application.h

@@ -268,9 +268,8 @@ public:
 
     float getGameLoopRate() const { return _gameLoopCounter.rate(); }
 
-    // Note that takeSnapshot has a default value, as this method is used internally.
     void takeSnapshot(bool notify, bool includeAnimated = false, float aspectRatio = 0.0f, const QString& filename = QString());
-    void takeSecondaryCameraSnapshot(const QString& filename);
+    void takeSecondaryCameraSnapshot(const QString& filename = QString());
 
     void shareSnapshot(const QString& filename, const QUrl& href = QUrl(""));
 
@@ -288,6 +287,8 @@ public:
     bool getSaveAvatarOverrideUrl() { return _saveAvatarOverrideUrl; }
     void saveNextPhysicsStats(QString filename);
 
+    void replaceDomainContent(const QString& url);
+
 signals:
     void svoImportRequested(const QString& url);
 
@@ -369,6 +370,7 @@ public slots:
     void updateHeartbeat() const;
 
     static void deadlockApplication();
+    static void unresponsiveApplication(); // cause main thread to be unresponsive for 35 seconds
 
     void rotationModeChanged() const;
 

interface/src/AvatarBookmarks.cpp

@@ -19,6 +19,11 @@
 #include <Application.h>
 #include <OffscreenUi.h>
 #include <avatar/AvatarManager.h>
+#include <EntityItemID.h>
+#include <EntityTree.h>
+#include <PhysicalEntitySimulation.h>
+#include <EntityEditPacketSender.h>
+#include <VariantMapToScriptValue.h>
 
 #include "MainWindow.h"
 #include "Menu.h"
@@ -29,6 +34,62 @@
 
 #include <QtQuick/QQuickWindow>
 
+
+void addAvatarEntities(const QVariantList& avatarEntities) {
+    auto nodeList = DependencyManager::get<NodeList>();
+    const QUuid myNodeID = nodeList->getSessionUUID();
+    EntityTreePointer entityTree = DependencyManager::get<EntityTreeRenderer>()->getTree();
+    if (!entityTree) {
+        return;
+    }
+    EntitySimulationPointer entitySimulation = entityTree->getSimulation();
+    PhysicalEntitySimulationPointer physicalEntitySimulation = std::static_pointer_cast<PhysicalEntitySimulation>(entitySimulation);
+    EntityEditPacketSender* entityPacketSender = physicalEntitySimulation->getPacketSender();
+    QScriptEngine scriptEngine;
+    for (int index = 0; index < avatarEntities.count(); index++) {
+        const QVariantMap& avatarEntityProperties = avatarEntities.at(index).toMap();
+        QVariant variantProperties = avatarEntityProperties["properties"];
+        QVariantMap asMap = variantProperties.toMap();
+        QScriptValue scriptProperties = variantMapToScriptValue(asMap, scriptEngine);
+        EntityItemProperties entityProperties;
+        EntityItemPropertiesFromScriptValueHonorReadOnly(scriptProperties, entityProperties);
+
+        entityProperties.setParentID(myNodeID);
+        entityProperties.setClientOnly(true);
+        entityProperties.setOwningAvatarID(myNodeID);
+        entityProperties.setSimulationOwner(myNodeID, AVATAR_ENTITY_SIMULATION_PRIORITY);
+        entityProperties.markAllChanged();
+
+        EntityItemID id = EntityItemID(QUuid::createUuid());
+        bool success = true;
+        entityTree->withWriteLock([&] {
+            EntityItemPointer entity = entityTree->addEntity(id, entityProperties);
+            if (entity) {
+                if (entityProperties.queryAACubeRelatedPropertyChanged()) {
+                    // due to parenting, the server may not know where something is in world-space, so include the bounding cube.
+                    bool success;
+                    AACube queryAACube = entity->getQueryAACube(success);
+                    if (success) {
+                        entityProperties.setQueryAACube(queryAACube);
+                    }
+                }
+
+                entity->setLastBroadcast(usecTimestampNow());
+                // since we're creating this object we will immediately volunteer to own its simulation
+                entity->flagForOwnershipBid(VOLUNTEER_SIMULATION_PRIORITY);
+                entityProperties.setLastEdited(entity->getLastEdited());
+            } else {
+                qCDebug(entities) << "AvatarEntitiesBookmark failed to add new Entity to local Octree";
+                success = false;
+            }
+        });
+
+        if (success) {
+            entityPacketSender->queueEditEntityMessage(PacketType::EntityAdd, entityTree, id, entityProperties);
+        }
+    }
+}
+
 AvatarBookmarks::AvatarBookmarks() {
     _bookmarksFilename = PathUtils::getAppDataPath() + "/" + AVATARBOOKMARKS_FILENAME;
     readFromFile();
@@ -38,7 +99,7 @@ void AvatarBookmarks::readFromFile() {
     // migrate old avatarbookmarks.json, used to be in 'local' folder on windows
     QString oldConfigPath = QStandardPaths::writableLocation(QStandardPaths::DataLocation) + "/" + AVATARBOOKMARKS_FILENAME;
     QFile oldConfig(oldConfigPath);
-    
+
     // I imagine that in a year from now, this code for migrating (as well as the two lines above)
     // may be removed since all bookmarks should have been migrated by then
     // - Robbie Uvanni (6.8.2017)
@@ -48,9 +109,9 @@ void AvatarBookmarks::readFromFile() {
         } else {
             qCDebug(interfaceapp) << "Failed to migrate" << AVATARBOOKMARKS_FILENAME;
         }
-    } 
-    
-    Bookmarks::readFromFile();     
+    }
+
+    Bookmarks::readFromFile();
 }
 
 void AvatarBookmarks::setupMenus(Menu* menubar, MenuWrapper* menu) {
@@ -81,23 +142,27 @@ void AvatarBookmarks::changeToBookmarkedAvatar() {
         myAvatar->useFullAvatarURL(action->data().toString());
         qCDebug(interfaceapp) << " Using Legacy V1 Avatar Bookmark ";
     } else {
-        
+
         const QMap<QString, QVariant> bookmark = action->data().toMap();
-        // Not magic value. This is the current made version, and if it changes this interpreter should be updated to 
+        // Not magic value. This is the current made version, and if it changes this interpreter should be updated to
         // handle the new one separately.
         // This is where the avatar bookmark entry is parsed. If adding new Value, make sure to have backward compatability with previous
         if (bookmark.value(ENTRY_VERSION) == 3) {
-                const QString& avatarUrl = bookmark.value(ENTRY_AVATAR_URL, "").toString();
-                myAvatar->useFullAvatarURL(avatarUrl);
-                qCDebug(interfaceapp) << "Avatar On " << avatarUrl;
-                const QList<QVariant>& attachments = bookmark.value(ENTRY_AVATAR_ATTACHMENTS, QList<QVariant>()).toList();
+            myAvatar->removeAvatarEntities();
+            const QString& avatarUrl = bookmark.value(ENTRY_AVATAR_URL, "").toString();
+            myAvatar->useFullAvatarURL(avatarUrl);
+            qCDebug(interfaceapp) << "Avatar On " << avatarUrl;
+            const QList<QVariant>& attachments = bookmark.value(ENTRY_AVATAR_ATTACHMENTS, QList<QVariant>()).toList();
 
-                qCDebug(interfaceapp) << "Attach " << attachments;
-                myAvatar->setAttachmentsVariant(attachments);
+            qCDebug(interfaceapp) << "Attach " << attachments;
+            myAvatar->setAttachmentsVariant(attachments);
+
+            const float& qScale = bookmark.value(ENTRY_AVATAR_SCALE, 1.0f).toFloat();
+            myAvatar->setAvatarScale(qScale);
+
+            const QVariantList& avatarEntities = bookmark.value(ENTRY_AVATAR_ENTITIES, QVariantList()).toList();
+            addAvatarEntities(avatarEntities);
 
-                const float& qScale = bookmark.value(ENTRY_AVATAR_SCALE, 1.0f).toFloat();
-                myAvatar->setAvatarScale(qScale);
-     
         } else {
             qCDebug(interfaceapp) << " Bookmark entry does not match client version, make sure client has a handler for the new AvatarBookmark";
         }
@@ -126,6 +191,7 @@ void AvatarBookmarks::addBookmark() {
         bookmark.insert(ENTRY_AVATAR_URL, avatarUrl);
         bookmark.insert(ENTRY_AVATAR_SCALE, avatarScale);
         bookmark.insert(ENTRY_AVATAR_ATTACHMENTS, myAvatar->getAttachmentsVariant());
+        bookmark.insert(ENTRY_AVATAR_ENTITIES, myAvatar->getAvatarEntitiesVariant());
 
         Bookmarks::addBookmarkToFile(bookmarkName, bookmark);
     });

interface/src/AvatarBookmarks.h

@@ -34,6 +34,7 @@ private:
     const QString AVATARBOOKMARKS_FILENAME = "avatarbookmarks.json";
     const QString ENTRY_AVATAR_URL = "avatarUrl";
     const QString ENTRY_AVATAR_ATTACHMENTS = "attachments";
+    const QString ENTRY_AVATAR_ENTITIES = "avatarEntites";
     const QString ENTRY_AVATAR_SCALE = "avatarScale";
     const QString ENTRY_VERSION = "version";
 

interface/src/AvatarEntitiesBookmarks.cpp

@@ -1,168 +0,0 @@
-//
-//  AvatarEntitiesBookmarks.cpp
-//  interface/src
-//
-//  Created by Dante Ruiz on 15/01/18.
-//  Copyright 2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#include <QAction>
-#include <QInputDialog>
-#include <QMessageBox>
-#include <QStandardPaths>
-#include <QQmlContext>
-#include <QList>
-
-#include <Application.h>
-#include <OffscreenUi.h>
-#include <EntityItemProperties.h>
-#include <GLMHelpers.h>
-#include <avatar/AvatarManager.h>
-#include <EntityItemID.h>
-#include <EntityTree.h>
-#include <PhysicalEntitySimulation.h>
-#include <EntityEditPacketSender.h>
-#include <VariantMapToScriptValue.h>
-
-#include "MainWindow.h"
-#include "Menu.h"
-#include "AvatarEntitiesBookmarks.h"
-#include "InterfaceLogging.h"
-
-#include "QVariantGLM.h"
-
-#include <QtQuick/QQuickWindow>
-
-void addAvatarEntities(const QVariantList& avatarEntities) {
-    auto nodeList = DependencyManager::get<NodeList>();
-    const QUuid myNodeID = nodeList->getSessionUUID();
-    EntityTreePointer entityTree = DependencyManager::get<EntityTreeRenderer>()->getTree();
-    if (!entityTree) {
-        return;
-    }
-    EntitySimulationPointer entitySimulation = entityTree->getSimulation();
-    PhysicalEntitySimulationPointer physicalEntitySimulation = std::static_pointer_cast<PhysicalEntitySimulation>(entitySimulation);
-    EntityEditPacketSender* entityPacketSender = physicalEntitySimulation->getPacketSender();
-    QScriptEngine scriptEngine;
-    for (int index = 0; index < avatarEntities.count(); index++) {
-        const QVariantMap& avatarEntityProperties = avatarEntities.at(index).toMap();
-        QVariant variantProperties = avatarEntityProperties["properties"];
-        QVariantMap asMap = variantProperties.toMap();
-        QScriptValue scriptProperties = variantMapToScriptValue(asMap, scriptEngine);
-        EntityItemProperties entityProperties;
-        EntityItemPropertiesFromScriptValueHonorReadOnly(scriptProperties, entityProperties);
-
-        entityProperties.setParentID(myNodeID);
-        entityProperties.setClientOnly(true);
-        entityProperties.setOwningAvatarID(myNodeID);
-        entityProperties.setSimulationOwner(myNodeID, AVATAR_ENTITY_SIMULATION_PRIORITY);
-        entityProperties.markAllChanged();
-
-        EntityItemID id = EntityItemID(QUuid::createUuid());
-        bool success = true;
-        entityTree->withWriteLock([&] {
-            EntityItemPointer entity = entityTree->addEntity(id, entityProperties);
-            if (entity) {
-                if (entityProperties.queryAACubeRelatedPropertyChanged()) {
-                    // due to parenting, the server may not know where something is in world-space, so include the bounding cube.
-                    bool success;
-                    AACube queryAACube = entity->getQueryAACube(success);
-                    if (success) {
-                        entityProperties.setQueryAACube(queryAACube);
-                    }
-                }
-
-                entity->setLastBroadcast(usecTimestampNow());
-                // since we're creating this object we will immediately volunteer to own its simulation
-                entity->flagForOwnershipBid(VOLUNTEER_SIMULATION_PRIORITY);
-                entityProperties.setLastEdited(entity->getLastEdited());
-            } else {
-                qCDebug(entities) << "AvatarEntitiesBookmark failed to add new Entity to local Octree";
-                success = false;
-            }
-        });
-
-        if (success) {
-            entityPacketSender->queueEditEntityMessage(PacketType::EntityAdd, entityTree, id, entityProperties);
-        }
-    }
-}
-
-AvatarEntitiesBookmarks::AvatarEntitiesBookmarks() {
-    _bookmarksFilename = PathUtils::getAppDataPath() + "/" + AVATAR_ENTITIES_BOOKMARKS_FILENAME;
-    Bookmarks::readFromFile();
-}
-
-void AvatarEntitiesBookmarks::setupMenus(Menu* menubar, MenuWrapper* menu) {
-    auto bookmarkAction = menubar->addActionToQMenuAndActionHash(menu, MenuOption::BookmarkAvatarEntities);
-    QObject::connect(bookmarkAction, SIGNAL(triggered()), this, SLOT(addBookmark()), Qt::QueuedConnection);
-    _bookmarksMenu = menu->addMenu(MenuOption::AvatarEntitiesBookmarks);
-    _deleteBookmarksAction = menubar->addActionToQMenuAndActionHash(menu, MenuOption::DeleteAvatarEntitiesBookmark);
-    QObject::connect(_deleteBookmarksAction, SIGNAL(triggered()), this, SLOT(deleteBookmark()), Qt::QueuedConnection);
-
-    for (auto it = _bookmarks.begin(); it != _bookmarks.end(); ++it) {
-        addBookmarkToMenu(menubar, it.key(), it.value());
-    }
-
-    Bookmarks::sortActions(menubar, _bookmarksMenu);
-}
-
-void AvatarEntitiesBookmarks::applyBookmarkedAvatarEntities() {
-    QAction* action = qobject_cast<QAction*>(sender());
-    auto myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
-
-    const QMap<QString, QVariant> bookmark = action->data().toMap();
-
-    if (bookmark.value(ENTRY_VERSION) == AVATAR_ENTITIES_BOOKMARK_VERSION) {
-        myAvatar->removeAvatarEntities();
-        const QString& avatarUrl = bookmark.value(ENTRY_AVATAR_URL, "").toString();
-        myAvatar->useFullAvatarURL(avatarUrl);
-        const QVariantList& avatarEntities = bookmark.value(ENTRY_AVATAR_ENTITIES, QVariantList()).toList();
-        addAvatarEntities(avatarEntities);
-        const float& avatarScale = bookmark.value(ENTRY_AVATAR_SCALE, 1.0f).toFloat();
-        myAvatar->setAvatarScale(avatarScale);
-    } else {
-        qCDebug(interfaceapp) << " Bookmark entry does not match client version, make sure client has a handler for the new AvatarEntitiesBookmark";
-    }
-}
-
-void AvatarEntitiesBookmarks::addBookmark() {
-    ModalDialogListener* dlg = OffscreenUi::getTextAsync(OffscreenUi::ICON_PLACEMARK, "Bookmark Avatar Entities", "Name", QString());
-    connect(dlg, &ModalDialogListener::response, this, [=] (QVariant response) {
-        disconnect(dlg, &ModalDialogListener::response, this, nullptr);
-        auto bookmarkName = response.toString();
-        bookmarkName = bookmarkName.trimmed().replace(QRegExp("(\r\n|[\r\n\t\v ])+"), " ");
-        if (bookmarkName.length() == 0) {
-            return;
-        }
-
-        auto myAvatar = DependencyManager::get<AvatarManager>()->getMyAvatar();
-
-        const QString& avatarUrl = myAvatar->getSkeletonModelURL().toString();
-        const QVariant& avatarScale = myAvatar->getAvatarScale();
-
-        QVariantMap bookmark;
-        bookmark.insert(ENTRY_VERSION, AVATAR_ENTITIES_BOOKMARK_VERSION);
-        bookmark.insert(ENTRY_AVATAR_URL, avatarUrl);
-        bookmark.insert(ENTRY_AVATAR_SCALE, avatarScale);
-        bookmark.insert(ENTRY_AVATAR_ENTITIES, myAvatar->getAvatarEntitiesVariant());
-
-        Bookmarks::addBookmarkToFile(bookmarkName, bookmark);
-    });
-}
-
-void AvatarEntitiesBookmarks::addBookmarkToMenu(Menu* menubar, const QString& name, const QVariant& bookmark) {
-    QAction* changeAction = _bookmarksMenu->newAction();
-    changeAction->setData(bookmark);
-    connect(changeAction, SIGNAL(triggered()), this, SLOT(applyBookmarkedAvatarEntities()));
-    if (!_isMenuSorted) {
-        menubar->addActionToQMenuAndActionHash(_bookmarksMenu, changeAction, name, 0, QAction::NoRole);
-    } else {
-        // TODO: this is aggressive but other alternatives have proved less fruitful so far.
-        menubar->addActionToQMenuAndActionHash(_bookmarksMenu, changeAction, name, 0, QAction::NoRole);
-        Bookmarks::sortActions(menubar, _bookmarksMenu);
-    }
-}

interface/src/AvatarEntitiesBookmarks.h

@@ -1,45 +0,0 @@
-//
-//  AvatarEntitiesBookmarks.h
-//  interface/src
-//
-//  Created by Dante Ruiz on 15/01/18.
-//  Copyright 2018 High Fidelity, Inc.
-//
-//  Distributed under the Apache License, Version 2.0.
-//  See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html
-//
-
-#ifndef hifi_AvatarEntitiesBookmarks_h
-#define hifi_AvatarEntitiesBookmarks_h
-
-#include <DependencyManager.h>
-#include "Bookmarks.h"
-
-class AvatarEntitiesBookmarks: public Bookmarks, public Dependency {
-  Q_OBJECT
-  SINGLETON_DEPENDENCY
-
-public:
-    AvatarEntitiesBookmarks();
-    void setupMenus(Menu* menubar, MenuWrapper* menu) override;
-
-public slots:
-    void addBookmark();
-
-protected:
-    void addBookmarkToMenu(Menu* menubar, const QString& name, const QVariant& bookmark) override;
-
-private:
-    const QString AVATAR_ENTITIES_BOOKMARKS_FILENAME = "AvatarEntitiesBookmarks.json";
-    const QString ENTRY_AVATAR_URL = "AvatarUrl";
-    const QString ENTRY_AVATAR_SCALE = "AvatarScale";
-    const QString ENTRY_AVATAR_ENTITIES = "AvatarEntities";
-    const QString ENTRY_VERSION = "version";
-
-    const int AVATAR_ENTITIES_BOOKMARK_VERSION = 1;
-
-private slots:
-  void applyBookmarkedAvatarEntities();
-};
-
-#endif

interface/src/Menu.cpp

@@ -34,7 +34,6 @@
 #include "audio/AudioScope.h"
 #include "avatar/AvatarManager.h"
 #include "AvatarBookmarks.h"
-#include "AvatarEntitiesBookmarks.h"
 #include "devices/DdeFaceTracker.h"
 #include "MainWindow.h"
 #include "render/DrawStatus.h"
@@ -207,9 +206,6 @@ Menu::Menu() {
     auto avatarBookmarks = DependencyManager::get<AvatarBookmarks>();
     avatarBookmarks->setupMenus(this, avatarMenu);
 
-    auto avatarEntitiesBookmarks = DependencyManager::get<AvatarEntitiesBookmarks>();
-    avatarEntitiesBookmarks->setupMenus(this, avatarMenu);
-
     // Display menu ----------------------------------
     // FIXME - this is not yet matching Alan's spec because it doesn't have
     // menus for "2D"/"3D" - we need to add support for detecting the appropriate
@@ -712,6 +708,7 @@ Menu::Menu() {
     MenuWrapper* crashMenu = developerMenu->addMenu("Crash");
 
     addActionToQMenuAndActionHash(crashMenu, MenuOption::DeadlockInterface, 0, qApp, SLOT(deadlockApplication()));
+    addActionToQMenuAndActionHash(crashMenu, MenuOption::UnresponsiveInterface, 0, qApp, SLOT(unresponsiveApplication()));
 
     action = addActionToQMenuAndActionHash(crashMenu, MenuOption::CrashPureVirtualFunction);
     connect(action, &QAction::triggered, qApp, []() { crash::pureVirtualCall(); });

interface/src/Menu.h

@@ -46,7 +46,6 @@ namespace MenuOption {
     const QString AutoMuteAudio = "Auto Mute Microphone";
     const QString AvatarReceiveStats = "Show Receive Stats";
     const QString AvatarBookmarks = "Avatar Bookmarks";
-    const QString AvatarEntitiesBookmarks = "Avatar Entities  Bookmarks";
     const QString Back = "Back";
     const QString BinaryEyelidControl = "Binary Eyelid Control";
     const QString BookmarkAvatar = "Bookmark Avatar";
@@ -77,6 +76,7 @@ namespace MenuOption {
     const QString CrashNewFault = "New Fault";
     const QString CrashNewFaultThreaded = "New Fault (threaded)";
     const QString DeadlockInterface = "Deadlock Interface";
+    const QString UnresponsiveInterface = "Unresponsive Interface";
     const QString DecreaseAvatarSize = "Decrease Avatar Size";
     const QString DefaultSkybox = "Default Skybox";
     const QString DeleteAvatarBookmark = "Delete Avatar Bookmark...";

interface/src/avatar/MyAvatar.cpp

@@ -79,6 +79,8 @@ float DEFAULT_SCRIPTED_MOTOR_TIMESCALE = 1.0e6f;
 const int SCRIPTED_MOTOR_CAMERA_FRAME = 0;
 const int SCRIPTED_MOTOR_AVATAR_FRAME = 1;
 const int SCRIPTED_MOTOR_WORLD_FRAME = 2;
+const int SCRIPTED_MOTOR_SIMPLE_MODE = 0;
+const int SCRIPTED_MOTOR_DYNAMIC_MODE = 1;
 const QString& DEFAULT_AVATAR_COLLISION_SOUND_URL = "https://hifi-public.s3.amazonaws.com/sounds/Collisions-otherorganic/Body_Hits_Impact.wav";
 
 const float MyAvatar::ZOOM_MIN = 0.5f;
@@ -92,6 +94,7 @@ MyAvatar::MyAvatar(QThread* thread) :
     _pitchSpeed(PITCH_SPEED_DEFAULT),
     _scriptedMotorTimescale(DEFAULT_SCRIPTED_MOTOR_TIMESCALE),
     _scriptedMotorFrame(SCRIPTED_MOTOR_CAMERA_FRAME),
+    _scriptedMotorMode(SCRIPTED_MOTOR_SIMPLE_MODE),
     _motionBehaviors(AVATAR_MOTION_DEFAULTS),
     _characterController(this),
     _eyeContactTarget(LEFT_EYE),
@@ -1479,7 +1482,7 @@ void MyAvatar::setSkeletonModelURL(const QUrl& skeletonModelURL) {
     });
     saveAvatarUrl();
     emit skeletonChanged();
-
+    emit skeletonModelURLChanged();
 }
 
 void MyAvatar::removeAvatarEntities() {
@@ -1623,20 +1626,27 @@ controller::Pose MyAvatar::getControllerPoseInAvatarFrame(controller::Action act
 void MyAvatar::updateMotors() {
     _characterController.clearMotors();
     glm::quat motorRotation;
+
+    const float FLYING_MOTOR_TIMESCALE = 0.05f;
+    const float WALKING_MOTOR_TIMESCALE = 0.2f;
+    const float INVALID_MOTOR_TIMESCALE = 1.0e6f;
+
+    float horizontalMotorTimescale;
+    float verticalMotorTimescale;
+
+    if (_characterController.getState() == CharacterController::State::Hover ||
+            _characterController.computeCollisionGroup() == BULLET_COLLISION_GROUP_COLLISIONLESS) {
+        horizontalMotorTimescale = FLYING_MOTOR_TIMESCALE;
+        verticalMotorTimescale = FLYING_MOTOR_TIMESCALE;
+    } else {
+        horizontalMotorTimescale = WALKING_MOTOR_TIMESCALE;
+        verticalMotorTimescale = INVALID_MOTOR_TIMESCALE;
+    }
+
     if (_motionBehaviors & AVATAR_MOTION_ACTION_MOTOR_ENABLED) {
-
-        const float FLYING_MOTOR_TIMESCALE = 0.05f;
-        const float WALKING_MOTOR_TIMESCALE = 0.2f;
-        const float INVALID_MOTOR_TIMESCALE = 1.0e6f;
-
-        float horizontalMotorTimescale;
-        float verticalMotorTimescale;
-
         if (_characterController.getState() == CharacterController::State::Hover ||
                 _characterController.computeCollisionGroup() == BULLET_COLLISION_GROUP_COLLISIONLESS) {
             motorRotation = getMyHead()->getHeadOrientation();
-            horizontalMotorTimescale = FLYING_MOTOR_TIMESCALE;
-            verticalMotorTimescale = FLYING_MOTOR_TIMESCALE;
         } else {
             // non-hovering = walking: follow camera twist about vertical but not lift
             // we decompose camera's rotation and store the twist part in motorRotation
@@ -1647,8 +1657,6 @@ void MyAvatar::updateMotors() {
             glm::quat liftRotation;
             swingTwistDecomposition(headOrientation, Vectors::UNIT_Y, liftRotation, motorRotation);
             motorRotation = orientation * motorRotation;
-            horizontalMotorTimescale = WALKING_MOTOR_TIMESCALE;
-            verticalMotorTimescale = INVALID_MOTOR_TIMESCALE;
         }
 
         if (_isPushing || _isBraking || !_isBeingPushed) {
@@ -1668,7 +1676,12 @@ void MyAvatar::updateMotors() {
             // world-frame
             motorRotation = glm::quat();
         }
-        _characterController.addMotor(_scriptedMotorVelocity, motorRotation, _scriptedMotorTimescale);
+        if (_scriptedMotorMode == SCRIPTED_MOTOR_SIMPLE_MODE) {
+            _characterController.addMotor(_scriptedMotorVelocity, motorRotation, _scriptedMotorTimescale);
+        } else {
+            // dynamic mode
+            _characterController.addMotor(_scriptedMotorVelocity, motorRotation, horizontalMotorTimescale, verticalMotorTimescale);
+        }
     }
 
     // legacy support for 'MyAvatar::applyThrust()', which has always been implemented as a
@@ -1752,6 +1765,14 @@ QString MyAvatar::getScriptedMotorFrame() const {
     return frame;
 }
 
+QString MyAvatar::getScriptedMotorMode() const {
+    QString mode = "simple";
+    if (_scriptedMotorMode == SCRIPTED_MOTOR_DYNAMIC_MODE) {
+        mode = "dynamic";
+    }
+    return mode;
+}
+
 void MyAvatar::setScriptedMotorVelocity(const glm::vec3& velocity) {
     float MAX_SCRIPTED_MOTOR_SPEED = 500.0f;
     _scriptedMotorVelocity = velocity;
@@ -1778,6 +1799,14 @@ void MyAvatar::setScriptedMotorFrame(QString frame) {
     }
 }
 
+void MyAvatar::setScriptedMotorMode(QString mode) {
+    if (mode.toLower() == "simple") {
+        _scriptedMotorMode = SCRIPTED_MOTOR_SIMPLE_MODE;
+    } else if (mode.toLower() == "dynamic") {
+        _scriptedMotorMode = SCRIPTED_MOTOR_DYNAMIC_MODE;
+    }
+}
+
 void MyAvatar::clearScriptableSettings() {
     _scriptedMotorVelocity = Vectors::ZERO;
     _scriptedMotorTimescale = DEFAULT_SCRIPTED_MOTOR_TIMESCALE;

interface/src/avatar/MyAvatar.h

@@ -69,6 +69,7 @@ class MyAvatar : public Avatar {
      * @property motorTimescale {float} Specifies how quickly the avatar should accelerate to meet the motorVelocity,
      *   smaller values will result in higher acceleration.
      * @property motorReferenceFrame {string} Reference frame of the motorVelocity, must be one of the following: "avatar", "camera", "world"
+     * @property motorMode {string} Type of scripted motor behavior, "simple" = use motorTimescale property (default mode) and "dynamic" = use action motor's timescales
      * @property collisionSoundURL {string} Specifies the sound to play when the avatar experiences a collision.
      *   You can provide a mono or stereo 16-bit WAV file running at either 24 Khz or 48 Khz.
      *   The latter is downsampled by the audio mixer, so all audio effectively plays back at a 24 Khz sample rate.
@@ -124,6 +125,7 @@ class MyAvatar : public Avatar {
     Q_PROPERTY(glm::vec3 motorVelocity READ getScriptedMotorVelocity WRITE setScriptedMotorVelocity)
     Q_PROPERTY(float motorTimescale READ getScriptedMotorTimescale WRITE setScriptedMotorTimescale)
     Q_PROPERTY(QString motorReferenceFrame READ getScriptedMotorFrame WRITE setScriptedMotorFrame)
+    Q_PROPERTY(QString motorMode READ getScriptedMotorMode WRITE setScriptedMotorMode)
     Q_PROPERTY(QString collisionSoundURL READ getCollisionSoundURL WRITE setCollisionSoundURL)
     Q_PROPERTY(AudioListenerMode audioListenerMode READ getAudioListenerMode WRITE setAudioListenerMode)
     Q_PROPERTY(glm::vec3 customListenPosition READ getCustomListenPosition WRITE setCustomListenPosition)
@@ -662,9 +664,11 @@ private:
     glm::vec3 getScriptedMotorVelocity() const { return _scriptedMotorVelocity; }
     float getScriptedMotorTimescale() const { return _scriptedMotorTimescale; }
     QString getScriptedMotorFrame() const;
+    QString getScriptedMotorMode() const;
     void setScriptedMotorVelocity(const glm::vec3& velocity);
     void setScriptedMotorTimescale(float timescale);
     void setScriptedMotorFrame(QString frame);
+    void setScriptedMotorMode(QString mode);
     virtual void attach(const QString& modelURL, const QString& jointName = QString(),
                         const glm::vec3& translation = glm::vec3(), const glm::quat& rotation = glm::quat(),
                         float scale = 1.0f, bool isSoft = false,
@@ -706,6 +710,7 @@ private:
     glm::vec3 _scriptedMotorVelocity; // target local-frame velocity of avatar (analog script)
     float _scriptedMotorTimescale; // timescale for avatar to achieve its target velocity
     int _scriptedMotorFrame;
+    int _scriptedMotorMode;
     quint32 _motionBehaviors;
     QString _collisionSoundURL;
 

interface/src/commerce/Ledger.cpp

@@ -22,6 +22,8 @@
 // inventory answers {status: 'success', data: {assets: [{id: "guid", title: "name", preview: "url"}....]}}
 // balance answers {status: 'success', data: {balance: integer}}
 // buy and receive_at answer {status: 'success'}
+// account synthesizes a result {status: 'success', data: {keyStatus: "preexisting"|"conflicting"|"ok"}}
+
 
 QJsonObject Ledger::apiResponse(const QString& label, QNetworkReply& reply) {
     QByteArray response = reply.readAll();
@@ -49,6 +51,7 @@ Handler(balance)
 Handler(inventory)
 Handler(transferHfcToNode)
 Handler(transferHfcToUsername)
+Handler(alreadyOwned)
 
 void Ledger::send(const QString& endpoint, const QString& success, const QString& fail, QNetworkAccessManager::Operation method, AccountManagerAuth::Type authType, QJsonObject request) {
     auto accountManager = DependencyManager::get<AccountManager>();
@@ -98,7 +101,7 @@ void Ledger::buy(const QString& hfc_key, int cost, const QString& asset_id, cons
     signedSend("transaction", transactionString, hfc_key, "buy", "buySuccess", "buyFailure", controlled_failure);
 }
 
-bool Ledger::receiveAt(const QString& hfc_key, const QString& old_key) {
+bool Ledger::receiveAt(const QString& hfc_key, const QString& signing_key) {
     auto accountManager = DependencyManager::get<AccountManager>();
     if (!accountManager->isLoggedIn()) {
         qCWarning(commerce) << "Cannot set receiveAt when not logged in.";
@@ -107,7 +110,7 @@ bool Ledger::receiveAt(const QString& hfc_key, const QString& old_key) {
         return false; // We know right away that we will fail, so tell the caller.
     }
 
-    signedSend("public_key", hfc_key.toUtf8(), old_key, "receive_at", "receiveAtSuccess", "receiveAtFailure");
+    signedSend("public_key", hfc_key.toUtf8(), signing_key, "receive_at", "receiveAtSuccess", "receiveAtFailure");
     return true; // Note that there may still be an asynchronous signal of failure that callers might be interested in.
 }
 
@@ -178,7 +181,7 @@ QString transactionString(const QJsonObject& valueObject) {
     } else {
         result += valueObject["message"].toString();
     }
-    
+
     // no matter what we append a smaller date to the bottom of this...
     result += QString("<br><font size='-2' color='#1080B8'>%1").arg(createdAt.toLocalTime().toString(Qt::DefaultLocaleShortDate));
     return result;
@@ -245,18 +248,33 @@ void Ledger::accountSuccess(QNetworkReply& reply) {
     auto iv = QByteArray::fromBase64(data["iv"].toString().toUtf8());
     auto ckey = QByteArray::fromBase64(data["ckey"].toString().toUtf8());
     QString remotePublicKey = data["public_key"].toString();
+    bool isOverride = wallet->wasSoftReset();
 
     wallet->setSalt(salt);
     wallet->setIv(iv);
     wallet->setCKey(ckey);
 
+    QString keyStatus = "ok";
     QStringList localPublicKeys = wallet->listPublicKeys();
-    if (remotePublicKey.isEmpty() && !localPublicKeys.isEmpty()) {
-        receiveAt(localPublicKeys.first(), "");
+    if (remotePublicKey.isEmpty() || isOverride) {
+        if (!localPublicKeys.isEmpty()) {
+            QString key = localPublicKeys.first();
+            receiveAt(key, key);
+        }
+    } else {
+        if (localPublicKeys.isEmpty()) {
+            keyStatus = "preexisting";
+        } else if (localPublicKeys.first() != remotePublicKey) {
+            keyStatus = "conflicting";
+        }
     }
 
     // none of the hfc account info should be emitted
-    emit accountResult(QJsonObject{ {"status", "success"} });
+    QJsonObject json;
+    QJsonObject responseData{ { "status", "success"} };
+    json["keyStatus"] = keyStatus;
+    responseData["data"] = json;
+    emit accountResult(responseData);
 }
 
 void Ledger::accountFailure(QNetworkReply& reply) {
@@ -336,3 +354,12 @@ void Ledger::transferHfcToUsername(const QString& hfc_key, const QString& userna
     auto transactionString = transactionDoc.toJson(QJsonDocument::Compact);
     signedSend("transaction", transactionString, hfc_key, "transfer_hfc_to_user", "transferHfcToUsernameSuccess", "transferHfcToUsernameFailure");
 }
+
+void Ledger::alreadyOwned(const QString& marketplaceId) {
+    auto wallet = DependencyManager::get<Wallet>();
+    QString endpoint = "already_owned";
+    QJsonObject request;
+    request["public_keys"] = QJsonArray::fromStringList(wallet->listPublicKeys());
+    request["marketplace_item_id"] = marketplaceId;
+    send(endpoint, "alreadyOwnedSuccess", "alreadyOwnedFailure", QNetworkAccessManager::PutOperation, AccountManagerAuth::Required, request);
+}

interface/src/commerce/Ledger.h

@@ -26,7 +26,7 @@ class Ledger : public QObject, public Dependency {
 
 public:
     void buy(const QString& hfc_key, int cost, const QString& asset_id, const QString& inventory_key, const bool controlled_failure = false);
-    bool receiveAt(const QString& hfc_key, const QString& old_key);
+    bool receiveAt(const QString& hfc_key, const QString& signing_key);
     void balance(const QStringList& keys);
     void inventory(const QStringList& keys);
     void history(const QStringList& keys, const int& pageNumber);
@@ -35,6 +35,7 @@ public:
     void certificateInfo(const QString& certificateId);
     void transferHfcToNode(const QString& hfc_key, const QString& nodeID, const int& amount, const QString& optionalMessage);
     void transferHfcToUsername(const QString& hfc_key, const QString& username, const int& amount, const QString& optionalMessage);
+    void alreadyOwned(const QString& marketplaceId);
 
     enum CertificateStatus {
         CERTIFICATE_STATUS_UNKNOWN = 0,
@@ -55,6 +56,7 @@ signals:
     void certificateInfoResult(QJsonObject result);
     void transferHfcToNodeResult(QJsonObject result);
     void transferHfcToUsernameResult(QJsonObject result);
+    void alreadyOwnedResult(QJsonObject result);
 
     void updateCertificateStatus(const QString& certID, uint certStatus);
 
@@ -79,6 +81,8 @@ public slots:
     void transferHfcToNodeFailure(QNetworkReply& reply);
     void transferHfcToUsernameSuccess(QNetworkReply& reply);
     void transferHfcToUsernameFailure(QNetworkReply& reply);
+    void alreadyOwnedSuccess(QNetworkReply& reply);
+    void alreadyOwnedFailure(QNetworkReply& reply);
 
 private:
     QJsonObject apiResponse(const QString& label, QNetworkReply& reply);

interface/src/commerce/QmlCommerce.cpp

@@ -10,11 +10,17 @@
 //
 
 #include "QmlCommerce.h"
+#include "CommerceLogging.h"
 #include "Application.h"
 #include "DependencyManager.h"
 #include "Ledger.h"
 #include "Wallet.h"
 #include <AccountManager.h>
+#include <Application.h>
+#include <UserActivityLogger.h>
+#include <ScriptEngines.h>
+#include <ui/TabletScriptingInterface.h>
+#include "scripting/HMDScriptingInterface.h"
 
 QmlCommerce::QmlCommerce() {
     auto ledger = DependencyManager::get<Ledger>();
@@ -28,14 +34,18 @@ QmlCommerce::QmlCommerce() {
     connect(ledger.data(), &Ledger::accountResult, this, &QmlCommerce::accountResult);
     connect(wallet.data(), &Wallet::walletStatusResult, this, &QmlCommerce::walletStatusResult);
     connect(ledger.data(), &Ledger::certificateInfoResult, this, &QmlCommerce::certificateInfoResult);
+    connect(ledger.data(), &Ledger::alreadyOwnedResult, this, &QmlCommerce::alreadyOwnedResult);
     connect(ledger.data(), &Ledger::updateCertificateStatus, this, &QmlCommerce::updateCertificateStatus);
     connect(ledger.data(), &Ledger::transferHfcToNodeResult, this, &QmlCommerce::transferHfcToNodeResult);
     connect(ledger.data(), &Ledger::transferHfcToUsernameResult, this, &QmlCommerce::transferHfcToUsernameResult);
+    connect(ledger.data(), &Ledger::transferHfcToUsernameResult, this, &QmlCommerce::transferHfcToUsernameResult);
     
     auto accountManager = DependencyManager::get<AccountManager>();
     connect(accountManager.data(), &AccountManager::usernameChanged, this, [&]() {
         setPassphrase("");
     });
+
+    _appsPath = PathUtils::getAppDataPath() + "Apps/";
 }
 
 void QmlCommerce::getWalletStatus() {
@@ -52,6 +62,11 @@ void QmlCommerce::getKeyFilePathIfExists() {
     emit keyFilePathIfExistsResult(wallet->getKeyFilePath());
 }
 
+bool QmlCommerce::copyKeyFileFrom(const QString& pathname) {
+    auto wallet = DependencyManager::get<Wallet>();
+    return wallet->copyKeyFileFrom(pathname);
+}
+
 void QmlCommerce::getWalletAuthenticatedStatus() {
     auto wallet = DependencyManager::get<Wallet>();
     emit walletAuthenticatedStatusResult(wallet->walletIsAuthenticatedWithPassphrase());
@@ -118,6 +133,11 @@ void QmlCommerce::changePassphrase(const QString& oldPassphrase, const QString&
     }
 }
 
+void QmlCommerce::setSoftReset() {
+    auto wallet = DependencyManager::get<Wallet>();
+    wallet->setSoftReset();
+}
+
 void QmlCommerce::setPassphrase(const QString& passphrase) {
     auto wallet = DependencyManager::get<Wallet>();
     wallet->setPassphrase(passphrase);
@@ -163,3 +183,164 @@ void QmlCommerce::transferHfcToUsername(const QString& username, const int& amou
     QString key = keys[0];
     ledger->transferHfcToUsername(key, username, amount, optionalMessage);
 }
+
+void QmlCommerce::replaceContentSet(const QString& itemHref) {
+    qApp->replaceDomainContent(itemHref);
+    QJsonObject messageProperties = {
+        { "status", "SuccessfulRequestToReplaceContent" },
+        { "content_set_url", itemHref }
+    };
+    UserActivityLogger::getInstance().logAction("replace_domain_content", messageProperties);
+
+    emit contentSetChanged(itemHref);
+}
+
+void QmlCommerce::alreadyOwned(const QString& marketplaceId) {
+    auto ledger = DependencyManager::get<Ledger>();
+    ledger->alreadyOwned(marketplaceId);
+}
+
+QString QmlCommerce::getInstalledApps() {
+    QString installedAppsFromMarketplace;
+    QStringList runningScripts = DependencyManager::get<ScriptEngines>()->getRunningScripts();
+
+    QDir directory(_appsPath);
+    QStringList apps = directory.entryList(QStringList("*.app.json"));
+    foreach(QString appFileName, apps) {
+        installedAppsFromMarketplace += appFileName;
+        installedAppsFromMarketplace += ",";
+        QFile appFile(_appsPath + appFileName);
+        if (appFile.open(QIODevice::ReadOnly)) {
+            QJsonDocument appFileJsonDocument = QJsonDocument::fromJson(appFile.readAll());
+
+            appFile.close();
+
+            QJsonObject appFileJsonObject = appFileJsonDocument.object();
+            QString scriptURL = appFileJsonObject["scriptURL"].toString();
+
+            // If the script .app.json is on the user's local disk but the associated script isn't running
+            // for some reason, start that script again.
+            if (!runningScripts.contains(scriptURL)) {
+                if ((DependencyManager::get<ScriptEngines>()->loadScript(scriptURL.trimmed())).isNull()) {
+                    qCDebug(commerce) << "Couldn't start script while checking installed apps.";
+                }
+            }
+        } else {
+            qCDebug(commerce) << "Couldn't open local .app.json file for reading.";
+        }
+    }
+
+    return installedAppsFromMarketplace;
+}
+
+bool QmlCommerce::installApp(const QString& itemHref) {
+    if (!QDir(_appsPath).exists()) {
+        if (!QDir().mkdir(_appsPath)) {
+            qCDebug(commerce) << "Couldn't make _appsPath directory.";
+            return false;
+        }
+    }
+
+    QUrl appHref(itemHref);
+
+    auto request = DependencyManager::get<ResourceManager>()->createResourceRequest(this, appHref);
+
+    if (!request) {
+        qCDebug(commerce) << "Couldn't create resource request for app.";
+        return false;
+    }
+
+    connect(request, &ResourceRequest::finished, this, [=]() {
+        if (request->getResult() != ResourceRequest::Success) {
+            qCDebug(commerce) << "Failed to get .app.json file from remote.";
+            return false;
+        }
+
+        // Copy the .app.json to the apps directory inside %AppData%/High Fidelity/Interface
+        auto requestData = request->getData();
+        QFile appFile(_appsPath + "/" + appHref.fileName());
+        if (!appFile.open(QIODevice::WriteOnly)) {
+            qCDebug(commerce) << "Couldn't open local .app.json file for creation.";
+            return false;
+        }
+        if (appFile.write(requestData) == -1) {
+            qCDebug(commerce) << "Couldn't write to local .app.json file.";
+            return false;
+        }
+        // Close the file
+        appFile.close();
+
+        // Read from the returned datastream to know what .js to add to Running Scripts
+        QJsonDocument appFileJsonDocument = QJsonDocument::fromJson(requestData);
+        QJsonObject appFileJsonObject = appFileJsonDocument.object();
+        QString scriptUrl = appFileJsonObject["scriptURL"].toString();
+
+        if ((DependencyManager::get<ScriptEngines>()->loadScript(scriptUrl.trimmed())).isNull()) {
+            qCDebug(commerce) << "Couldn't load script.";
+            return false;
+        }
+
+        emit appInstalled(itemHref);
+        return true;
+    });
+    request->send();
+    return true;
+}
+
+bool QmlCommerce::uninstallApp(const QString& itemHref) {
+    QUrl appHref(itemHref);
+
+    // Read from the file to know what .js script to stop
+    QFile appFile(_appsPath + "/" + appHref.fileName());
+    if (!appFile.open(QIODevice::ReadOnly)) {
+        qCDebug(commerce) << "Couldn't open local .app.json file for deletion.";
+        return false;
+    }
+    QJsonDocument appFileJsonDocument = QJsonDocument::fromJson(appFile.readAll());
+    QJsonObject appFileJsonObject = appFileJsonDocument.object();
+    QString scriptUrl = appFileJsonObject["scriptURL"].toString();
+
+    if (!DependencyManager::get<ScriptEngines>()->stopScript(scriptUrl.trimmed(), false)) {
+        qCDebug(commerce) << "Couldn't stop script.";
+        return false;
+    }
+
+    // Delete the .app.json from the filesystem
+    // remove() closes the file first.
+    if (!appFile.remove()) {
+        qCDebug(commerce) << "Couldn't delete local .app.json file.";
+        return false;
+    }
+
+    emit appUninstalled(itemHref);
+    return true;
+}
+
+bool QmlCommerce::openApp(const QString& itemHref) {
+    QUrl appHref(itemHref);
+
+    // Read from the file to know what .html or .qml document to open
+    QFile appFile(_appsPath + "/" + appHref.fileName());
+    if (!appFile.open(QIODevice::ReadOnly)) {
+        qCDebug(commerce) << "Couldn't open local .app.json file.";
+        return false;
+    }
+    QJsonDocument appFileJsonDocument = QJsonDocument::fromJson(appFile.readAll());
+    QJsonObject appFileJsonObject = appFileJsonDocument.object();
+    QString homeUrl = appFileJsonObject["homeURL"].toString();
+
+    auto tabletScriptingInterface = DependencyManager::get<TabletScriptingInterface>();
+    auto tablet = dynamic_cast<TabletProxy*>(tabletScriptingInterface->getTablet("com.highfidelity.interface.tablet.system"));
+    if (homeUrl.contains(".qml", Qt::CaseInsensitive)) {
+        tablet->loadQMLSource(homeUrl);
+    } else if (homeUrl.contains(".html", Qt::CaseInsensitive)) {
+        tablet->gotoWebScreen(homeUrl);
+    } else {
+        qCDebug(commerce) << "Attempted to open unknown type of homeURL!";
+        return false;
+    }
+
+    DependencyManager::get<HMDScriptingInterface>()->openTablet();
+
+    return true;
+}

interface/src/commerce/QmlCommerce.h

@@ -42,12 +42,18 @@ signals:
     void historyResult(QJsonObject result);
     void accountResult(QJsonObject result);
     void certificateInfoResult(QJsonObject result);
+    void alreadyOwnedResult(QJsonObject result);
 
     void updateCertificateStatus(const QString& certID, uint certStatus);
 
     void transferHfcToNodeResult(QJsonObject result);
     void transferHfcToUsernameResult(QJsonObject result);
 
+    void contentSetChanged(const QString& contentSetHref);
+
+    void appInstalled(const QString& appHref);
+    void appUninstalled(const QString& appHref);
+
 protected:
     Q_INVOKABLE void getWalletStatus();
 
@@ -55,10 +61,12 @@ protected:
     Q_INVOKABLE void getKeyFilePathIfExists();
     Q_INVOKABLE void getSecurityImage();
     Q_INVOKABLE void getWalletAuthenticatedStatus();
+    Q_INVOKABLE bool copyKeyFileFrom(const QString& pathname);
 
     Q_INVOKABLE void chooseSecurityImage(const QString& imageFile);
     Q_INVOKABLE void setPassphrase(const QString& passphrase);
     Q_INVOKABLE void changePassphrase(const QString& oldPassphrase, const QString& newPassphrase);
+    Q_INVOKABLE void setSoftReset();
 
     Q_INVOKABLE void buy(const QString& assetId, int cost, const bool controlledFailure = false);
     Q_INVOKABLE void balance();
@@ -68,9 +76,20 @@ protected:
     Q_INVOKABLE void account();
 
     Q_INVOKABLE void certificateInfo(const QString& certificateId);
+    Q_INVOKABLE void alreadyOwned(const QString& marketplaceId);
 
     Q_INVOKABLE void transferHfcToNode(const QString& nodeID, const int& amount, const QString& optionalMessage);
     Q_INVOKABLE void transferHfcToUsername(const QString& username, const int& amount, const QString& optionalMessage);
+
+    Q_INVOKABLE void replaceContentSet(const QString& itemHref);
+
+    Q_INVOKABLE QString getInstalledApps();
+    Q_INVOKABLE bool installApp(const QString& appHref);
+    Q_INVOKABLE bool uninstallApp(const QString& appHref);
+    Q_INVOKABLE bool openApp(const QString& appHref);
+
+private:
+    QString _appsPath;
 };
 
 #endif // hifi_QmlCommerce_h

interface/src/commerce/Wallet.cpp

@@ -59,6 +59,23 @@ QString keyFilePath() {
     auto accountManager = DependencyManager::get<AccountManager>();
     return PathUtils::getAppDataFilePath(QString("%1.%2").arg(accountManager->getAccountInfo().getUsername(), KEY_FILE));
 }
+bool Wallet::copyKeyFileFrom(const QString& pathname) {
+    QString existing = getKeyFilePath();
+    qCDebug(commerce) << "Old keyfile" << existing;
+    if (!existing.isEmpty()) {
+        QString backup = QString(existing).insert(existing.indexOf(KEY_FILE) - 1,
+            QDateTime::currentDateTime().toString(Qt::ISODate).replace(":", ""));
+        qCDebug(commerce) << "Renaming old keyfile to" << backup;
+        if (!QFile::rename(existing, backup)) {
+            qCCritical(commerce) << "Unable to backup" << existing << "to" << backup;
+            return false;
+        }
+    }
+    QString destination = keyFilePath();
+    bool result = QFile::copy(pathname, destination);
+    qCDebug(commerce) << "copy" << pathname << "to" << destination << "=>" << result;
+    return result;
+}
 
 // use the cached _passphrase if it exists, otherwise we need to prompt
 int passwordCallback(char* password, int maxPasswordSize, int rwFlag, void* u) {
@@ -300,17 +317,24 @@ Wallet::Wallet() {
     packetReceiver.registerListener(PacketType::ChallengeOwnership, this, "handleChallengeOwnershipPacket");
     packetReceiver.registerListener(PacketType::ChallengeOwnershipRequest, this, "handleChallengeOwnershipPacket");
 
-    connect(ledger.data(), &Ledger::accountResult, this, [&]() {
+    connect(ledger.data(), &Ledger::accountResult, this, [&](QJsonObject result) {
         auto wallet = DependencyManager::get<Wallet>();
         auto walletScriptingInterface = DependencyManager::get<WalletScriptingInterface>();
         uint status;
+        QString keyStatus = result.contains("data") ? result["data"].toObject()["keyStatus"].toString() : "";
 
         if (wallet->getKeyFilePath() == "" || !wallet->getSecurityImage()) {
-            status = (uint)WalletStatus::WALLET_STATUS_NOT_SET_UP;
+            if (keyStatus == "preexisting") {
+                status = (uint) WalletStatus::WALLET_STATUS_PREEXISTING;
+            } else{
+                status = (uint) WalletStatus::WALLET_STATUS_NOT_SET_UP;
+            }
         } else if (!wallet->walletIsAuthenticatedWithPassphrase()) {
-            status = (uint)WalletStatus::WALLET_STATUS_NOT_AUTHENTICATED;
+            status = (uint) WalletStatus::WALLET_STATUS_NOT_AUTHENTICATED;
+        } else if (keyStatus == "conflicting") {
+            status = (uint) WalletStatus::WALLET_STATUS_CONFLICTING;
         } else {
-            status = (uint)WalletStatus::WALLET_STATUS_READY;
+            status = (uint) WalletStatus::WALLET_STATUS_READY;
         }
 
         walletScriptingInterface->setWalletStatus(status);
@@ -524,17 +548,17 @@ bool Wallet::generateKeyPair() {
 
     // TODO: redo this soon -- need error checking and so on
     writeSecurityImage(_securityImage, keyFilePath());
-    QString oldKey = _publicKeys.count() == 0 ? "" : _publicKeys.last();
     QString key = keyPair.first->toBase64();
     _publicKeys.push_back(key);
     qCDebug(commerce) << "public key:" << key;
+    _isOverridingServer = false;
 
     // It's arguable whether we want to change the receiveAt every time, but:
     // 1. It's certainly needed the first time, when createIfNeeded answers true.
     // 2. It is maximally private, and we can step back from that later if desired.
     // 3. It maximally exercises all the machinery, so we are most likely to surface issues now.
     auto ledger = DependencyManager::get<Ledger>();
-    return ledger->receiveAt(key, oldKey);
+    return ledger->receiveAt(key, key);
 }
 
 QStringList Wallet::listPublicKeys() {

interface/src/commerce/Wallet.h

@@ -35,6 +35,7 @@ public:
     void chooseSecurityImage(const QString& imageFile);
     bool getSecurityImage();
     QString getKeyFilePath();
+    bool copyKeyFileFrom(const QString& pathname);
 
     void setSalt(const QByteArray& salt) { _salt = salt; }
     QByteArray getSalt() { return _salt; }
@@ -48,11 +49,15 @@ public:
     bool getPassphraseIsCached() { return !(_passphrase->isEmpty()); }
     bool walletIsAuthenticatedWithPassphrase();
     bool changePassphrase(const QString& newPassphrase);
+    void setSoftReset() { _isOverridingServer = true;  }
+    bool wasSoftReset() { bool was = _isOverridingServer; _isOverridingServer = false; return was; }
 
     void getWalletStatus();
     enum WalletStatus {
         WALLET_STATUS_NOT_LOGGED_IN = 0,
         WALLET_STATUS_NOT_SET_UP,
+        WALLET_STATUS_PREEXISTING,
+        WALLET_STATUS_CONFLICTING,
         WALLET_STATUS_NOT_AUTHENTICATED,
         WALLET_STATUS_READY
     };
@@ -73,6 +78,7 @@ private:
     QByteArray _iv;
     QByteArray _ckey;
     QString* _passphrase { new QString("") };
+    bool _isOverridingServer { false };
 
     bool writeWallet(const QString& newPassphrase = QString(""));
     void updateImageProvider();

interface/src/main.cpp

@@ -38,6 +38,7 @@ extern "C" {
 #endif
 
 int main(int argc, const char* argv[]) {
+    setupHifiApplication(BuildInfo::INTERFACE_NAME);
 
 #ifdef Q_OS_LINUX
     QApplication::setAttribute(Qt::AA_DontUseNativeMenuBar);
@@ -51,17 +52,9 @@ int main(int argc, const char* argv[]) {
     QCoreApplication::setAttribute(Qt::AA_UseOpenGLES);
 #endif
 
-    disableQtBearerPoll(); // Fixes wifi ping spikes
-
     QElapsedTimer startupTime;
     startupTime.start();
 
-    // Set application infos
-    QCoreApplication::setApplicationName(BuildInfo::INTERFACE_NAME);
-    QCoreApplication::setOrganizationName(BuildInfo::MODIFIED_ORGANIZATION);
-    QCoreApplication::setOrganizationDomain(BuildInfo::ORGANIZATION_DOMAIN);
-    QCoreApplication::setApplicationVersion(BuildInfo::VERSION);
-
     Setting::init();
 
     // Instance UserActivityLogger now that the settings are loaded

interface/src/octree/OctreePacketProcessor.cpp

@@ -17,6 +17,8 @@
 #include "SceneScriptingInterface.h"
 
 OctreePacketProcessor::OctreePacketProcessor() {
+    setObjectName("Octree Packet Processor");
+
     auto& packetReceiver = DependencyManager::get<NodeList>()->getPacketReceiver();
     
     packetReceiver.registerDirectListenerForTypes({ PacketType::OctreeStats, PacketType::EntityData, PacketType::EntityErase },

interface/src/scripting/AccountServicesScriptingInterface.h

@@ -38,7 +38,7 @@ class AccountServicesScriptingInterface : public QObject {
     Q_PROPERTY(QString username READ getUsername NOTIFY myUsernameChanged)
     Q_PROPERTY(bool loggedIn READ loggedIn NOTIFY loggedInChanged)
     Q_PROPERTY(QString findableBy READ getFindableBy WRITE setFindableBy NOTIFY findableByChanged)
-    Q_PROPERTY(QUrl metaverseServerURL READ getMetaverseServerURL)
+    Q_PROPERTY(QUrl metaverseServerURL READ getMetaverseServerURL CONSTANT)
     
 public:
     static AccountServicesScriptingInterface* getInstance();

interface/src/scripting/MenuScriptingInterface.cpp

@@ -94,22 +94,6 @@ bool MenuScriptingInterface::menuItemExists(const QString& menu, const QString&
     return result;
 }
 
-void MenuScriptingInterface::addActionGroup(const QString& groupName, const QStringList& actionList,
-                                            const QString& selected) {
-    static const char* slot = SLOT(menuItemTriggered());
-    QMetaObject::invokeMethod(Menu::getInstance(), "addActionGroup",
-                              Q_ARG(const QString&, groupName),
-                              Q_ARG(const QStringList&, actionList),
-                              Q_ARG(const QString&, selected),
-                              Q_ARG(QObject*, this),
-                              Q_ARG(const char*, slot));
-}
-
-void MenuScriptingInterface::removeActionGroup(const QString& groupName) {
-    QMetaObject::invokeMethod(Menu::getInstance(), "removeActionGroup",
-                              Q_ARG(const QString&, groupName));
-}
-
 bool MenuScriptingInterface::isOptionChecked(const QString& menuOption) {
     if (QThread::currentThread() == qApp->thread()) {
         return Menu::getInstance()->isOptionChecked(menuOption);
@@ -147,19 +131,3 @@ void MenuScriptingInterface::setMenuEnabled(const QString& menuOption, bool isCh
 void MenuScriptingInterface::triggerOption(const QString& menuOption) {
     QMetaObject::invokeMethod(Menu::getInstance(), "triggerOption", Q_ARG(const QString&, menuOption));    
 }
-
-void MenuScriptingInterface::closeInfoView(const QString& path) {
-    QMetaObject::invokeMethod(Menu::getInstance(), "closeInfoView", Q_ARG(const QString&, path));
-}
-
-bool MenuScriptingInterface::isInfoViewVisible(const QString& path) {
-    if (QThread::currentThread() == qApp->thread()) {
-        return Menu::getInstance()->isInfoViewVisible(path);
-    }
-        
-    bool result;
-    BLOCKING_INVOKE_METHOD(Menu::getInstance(), "isInfoViewVisible",
-        Q_RETURN_ARG(bool, result), Q_ARG(const QString&, path));
-    return result;
-}
-